Beispiel #1
0
 def _post(self):
     request = _get_json(self.request.body)
     uid = request['uid']
     _logger.info("Proceeding with unlink request for '%(uid)s'..." % {
      'uid': uid,
     })
     
     record = database.get_record(uid)
     if not record:
         self.send_error(404)
         return
         
     trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
     if not trust.write:
         self.send_error(403)
         return
     
     fs = state.get_filesystem(record['physical']['family'])
     try:
         fs.unlink(record)
     except filesystem.FileNotFoundError as e:
         _logger.error("Database record exists for '%(uid)s', but filesystem entry does not" % {
          'uid': uid,
         })
         self.send_error(404)
         return
     else:
         database.drop_record(uid)
Beispiel #2
0
 def run(self):
     """
     Cycles through every database record in order, removing any records associated
     with files that do not exist.
     """
     ctime = -1.0
     while True:
         while not self._within_window(DATABASE_WINDOWS):
             _logger.debug("Not in execution window; sleeping" % {
              'name': self.name,
             })
             time.sleep(60)
             
         records_retrieved = False
         for record in database.enumerate_all(ctime):
             ctime = record['physical']['ctime']
             records_retrieved = True
             
             filesystem = state.get_filesystem(record['physical']['family'])
             if not filesystem.file_exists(record):
                 _logger.warn("Discovered database record for '%(uid)s' without matching file; dropping record..." % {
                  'uid': record['_id'],
                 })
                 database.drop_record(record['_id'])
                 
         if not records_retrieved: #Cycle complete
             _logger.debug("All records processed; sleeping")
             time.sleep(CONFIG.maintainer_database_sleep)
             ctime = -1.0
Beispiel #3
0
 def _post(self):
     request = _get_json(self.request.body)
     uid = request['uid']
     _logger.info("Proceeding with retrieval request for '%(uid)s'..." % {
      'uid': uid,
     })
     
     record = database.get_record(uid)
     if not record:
         self.send_error(404)
         return
         
     trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
     if not trust.read:
         self.send_error(403)
         return
         
     current_time = int(time.time())
     record['physical']['atime'] = current_time
     record['stats']['accesses'] += 1
     for policy in ('delete', 'compress'):
         if 'stale' in record['policy'][policy]:
             record['policy'][policy]['staleTime'] = current_time + record['policy'][policy]['stale']
     database.update_record(record)
     
     fs = state.get_filesystem(record['physical']['family'])
     try:
         data = fs.get(record)
     except filesystem.FileNotFoundError as e:
         _logger.error("Database record exists for '%(uid)s', but filesystem entry does not" % {
          'uid': uid,
         })
         self.send_error(404)
         return
     else:
         _logger.debug("Evaluating decompression requirements...")
         applied_compression = record['physical']['format'].get('comp')
         supported_compressions = (c.strip() for c in (self.request.headers.get('Media-Storage-Supported-Compression') or '').split(';'))
         if applied_compression and not applied_compression in supported_compressions: #Must be decompressed first
             data = compression.get_decompressor(applied_compression)(data)
             applied_compression = None
             
         _logger.debug("Returning entity...")
         self.set_header('Content-Type', record['physical']['format']['mime'])
         if applied_compression:
             self.set_header('Media-Storage-Applied-Compression', applied_compression)
         while True:
             chunk = data.read(_CHUNK_SIZE)
             if chunk:
                 self.write(chunk)
             else:
                 break
Beispiel #4
0
 def run(self):
     """
     Cycles through every filesystem entry in order, removing any files associated
     with database records that do not exist.
     """
     while True:
         for family in state.get_families():
             _logger.info("Processing family %(family)r..." % {
              'family': family,
             })
             filesystem = state.get_filesystem(family)
             self._walk(filesystem.walk())
             
         _logger.debug("All records processed; sleeping")
         time.sleep(CONFIG.maintainer_filesystem_sleep)
Beispiel #5
0
 def _post(self):
     (header, data) = self._get_payload()
     
     current_time = time.time()
     try:
         _logger.debug("Assembling database record...")
         record = {
          '_id': header.get('uid') or uuid.uuid1().hex,
          'keys': self._build_keys(header),
          'physical': {
           'family': header['physical'].get('family'),
           'ctime': current_time,
           'minRes': CONFIG.storage_minute_resolution,
           'atime': int(current_time),
           'format': self._build_format(header),
          },
          'policy': self._build_policy(header),
          'stats': {
           'accesses': 0,
          },
          'meta': header.get('meta') or {},
         }
     except (KeyError, TypeError, AttributeError) as e:
         _logger.error("Request received did not adhere to expected structure: %(error)s" % {
          'error': str(e),
         })
         self.send_error(409)
         return
     else:
         _logger.info("Proceeding with storage request for '%(uid)s'..." % {
          'uid': record['_id'],
         })
         
     _logger.debug("Evaluating compression requirements...")
     target_compression = record['physical']['format'].get('comp')
     if target_compression and self.request.headers.get('Media-Storage-Compress-On-Server') == 'yes':
         _logger.info("Compressing file...")
         data = compression.get_compressor(target_compression)(data)
         
     _logger.debug("Storing entity...")
     database.add_record(record)
     fs = state.get_filesystem(record['physical']['family'])
     fs.put(record, data)
     
     return {
      'uid': record['_id'],
      'keys': record['keys'],
     }
Beispiel #6
0
 def _process_record(self, record):
     """
     Determines whether the given `record` is a candidate for deletion, removing it and the
     associated file if it is.
     """
     _logger.info("Unlinking record...")
     filesystem = state.get_filesystem(record['physical']['family'])
     try:
         filesystem.unlink(record)
     except Exception as e:
         _logger.warn("Unable to unlink record: %(error)s" % {
          'error': str(e),
         })
         return False
     else:
         database.drop_record(record['_id'])
         return True
Beispiel #7
0
 def _post(self):
     request = _get_json(self.request.body)
     uid = request['uid']
     _logger.info("Proceeding with description request for '%(uid)s'..." % {
      'uid': uid,
     })
     
     record = database.get_record(uid)
     if not record:
         self.send_error(404)
         return
         
     trust = _get_trust(record, request.get('keys'), self.request.remote_ip)
     if not trust.read:
         self.send_error(403)
         return
         
     _logger.debug("Describing entity...")
     record['physical']['exists'] = state.get_filesystem(record['physical']['family']).file_exists(record)
     record['uid'] = record['_id']
     del record['_id']
     del record['physical']['minRes']
     del record['keys']
     return record
Beispiel #8
0
 def _post(self):
     request = _get_json(self.request.body)
     
     query = {}
     trust = _get_trust(None, None, self.request.remote_ip)
     if not trust.read:
         query['keys.read'] = None #Anonymous records only
         
     def _assemble_range_block(name, attribute):
         attribute_block = {}
         _min = request[name]['min']
         _max = request[name]['max']
         _block = {}
         if _min:
             attribute_block['$gte'] = _min
         if _max:
             attribute_block['$lte'] = _max
         if attribute_block:
             query[attribute] = attribute_block
     _assemble_range_block('ctime', 'physical.ctime')
     _assemble_range_block('atime', 'physical.atime')
     _assemble_range_block('accesses', 'stats.accesses')
     
     query['physical.family'] = request['family']
     
     mime = request['mime']
     if mime:
         if '/' in mime:
             query['physical.mime'] = mime
         else:
             query['physical.mime'] = {'$regex': '^' + mime}
             
     for (key, value) in request['meta'].items():
         key = 'meta.' + key
         
         if type(value) in types.StringTypes:
             if value.startswith('::'):
                 value = value[1:]
             else:
                 match = _FILTER_RE.match(value)
                 if match:
                     filter = match.group('filter')
                     query = match.group('query')
                     if filter == 'range':
                         (_min, _max) = (float(v) for v in query.split(':', 1))
                         value = {'$gte': _min, '$lte': _max}
                     elif filter == 'lte':
                         value = {'$lte': float(query)}
                     elif filter == 'gte':
                         value = {'$gte': float(query)}
                     elif filter == 're':
                         value = {'$regex': query}
                     elif filter == 're':
                         value = {'$regex': query}
                     elif filter == 're.i':
                         value = {'$regex': query, '$options': 'i'}
                     elif filter == 'like':
                         if query.count('%') == 1 and query.endswith('%'):
                             value = {'$regex': '^' + query[:-1]}
                         else:
                             value = {'$regex': '^' + query.replace('%', '.*') + '$'}
                     elif filter == 'ilike':
                         value = {'$regex': '^' + query.replace('%', '.*') + '$', '$options': 'i'}
                     else:
                         raise ValueError("Unrecognised filter: %(filter)s" % {
                          'filter': filter,
                         })
         query[key] = value
         
     records = []
     for record in database.enumerate_where(query):
         record['physical']['exists'] = state.get_filesystem(record['physical']['family']).file_exists(record)
         if not trust.read:
             del record['keys']
         else:
             record['physical']['path'] = state.get_filesystem(request['family']).resolve_path(record)
         #Has to happen after the filesystem search
         record['uid'] = record['_id']
         del record['_id']
         del record['physical']['minRes']
         records.append(record)
     return {
      'records': records,
     }
Beispiel #9
0
 def _process_record(self, record):
     """
     Determines whether the given `record` is a candidate for compression, compressing the
     associated file and updating the record if it is.
     """
     _logger.info("Compressing record '%(uid)s'..." % {
      'uid': record['_id'],
     })
     current_compression = record['physical']['format'].get('comp')
     target_compression = record['policy']['compress'].get('comp')
     if current_compression == target_compression:
         _logger.debug("File already compressed in target format")
         record['policy']['compress'].clear() #Drop the compression policy
         try:
             database.update_record(record)
         except Exception as e:
             _logger.error("Unable to update record to reflect already-applied compression; compression routine will retry later: %(error)s" % {
              'error': str(e),
             })
             return False
         else:
             return True
             
     filesystem = state.get_filesystem(record['physical']['family'])
     data = filesystem.get(record)
     if current_compression: #Must be decompressed first
         _logger.info("Decompressing file...")
         data = compression.get_decompressor(current_compression)(data)
     data = compression.get_compressor(target_compression)(data)
     
     _logger.info("Updating entity...")
     old_format = record['physical']['format'].copy()
     record['physical']['format']['comp'] = target_compression
     try:
         filesystem.put(record, data, tempfile=True)
     except Exception as e: #Harmless backout point
         _logger.warn("Unable to write compressed file to disk; backing out with no consequences")
         return False
     else:
         old_compression_policy = record['policy']['compress'].copy()
         record['policy']['compress'].clear() #Drop the compression policy
         try:
             database.update_record(record)
         except Exception as e: #Results in wasted space until the next attempt
             _logger.error("Unable to update record; old file will be served, and new file will be replaced on a subsequent compression attempt: %(error)s" % {
              'error': str(e),
             })
             return False
         else:
             try:
                 filesystem.make_permanent(record)
             except Exception as e:
                 _logger.error("Unable to update on-disk file; rolling back database update: %(error)s" % {
                  'error': str(e),
                 })
                 record['policy']['compress'] = old_compression_policy
                 record['physical']['format'] = old_format
                 try:
                     database.update_record(record)
                 except Exception as e:
                     _logger.error("Unable to roll back database update; '%(uid)s' is inaccessible and must be manually decompressed from '%(comp)s' format: %(error)s" % {
                      'error': str(e),
                      'uid': record['_id'],
                      'comp': target_compression,
                     })
                 return False
                 
             record['physical']['format'] = old_format
             try:
                 filesystem.unlink(record)
             except Exception as e: #Results in wasted space, but non-fatal
                 _logger.error("Unable to unlink old file; space occupied by '%(uid)s' non-recoverable unless unlinked manually: %(family)r | %(file)s : %(error)s" % {
                  'family': record['physical']['family'],
                  'file': filesystem.resolve_path(record),
                  'uid': record['_id'],
                  'error': str(e),
                 })
             return True