示例#1
0
 def put(self, data, mime, family=None,
  comp=compression.COMPRESS_NONE, compress_on_server=False,
  deletion_policy=None, compression_policy=None,
  meta=None,
  uid=None, keys=None,
  timeout=10.0
 ):
     """
     Stores data on a server, returning a dictionary containing the keys 'uid', which points to
     the UID of the stored data, and 'keys', which is another dictionary containing 'read' and
     'write', the keys needed to perform either type of action on the stored data.
     
     `data` is a string or file-like object containing the payload to be stored and `mime` is the
     MIME-type of the data.
     
     `timeout` defaults to 10.0s, but should be adjusted depending on your needs.
     
     All other arguments are the same as in ``media_storage.interfaces.ControlConstruct.put``.
     """
     description = {
      'uid': uid,
      'keys': keys,
      'physical': {
       'family': family,
       'format': {
        'mime': mime,
        'comp': comp,
       },
      },
      'policy': {
       'delete': deletion_policy,
       'compress': compression_policy,
      },
      'meta': meta,
     }
     
     headers = {}
     if comp:
         if not compress_on_server:
             try:
                 if type(data) in types.StringTypes: #The compressors expect file-like objects
                     data = StringIO.StringIO(data)
                 data = compression.get_compressor(comp)(data)
             except ValueError:
                 headers[common.HEADER_COMPRESS_ON_SERVER] = common.HEADER_COMPRESS_ON_SERVER_TRUE
         else:
             headers[common.HEADER_COMPRESS_ON_SERVER] = common.HEADER_COMPRESS_ON_SERVER_TRUE
             
     request = common.assemble_request(self._server.get_host() + common.SERVER_PUT, description, headers=headers, data=data)
     (properties, response) = common.send_request(request, timeout=timeout)
     return json.loads(response)
示例#2
0
 def _post(self):
     (header, data) = self._get_payload()
     
     current_time = time.time()
     try:
         _logger.debug("Assembling database record...")
         record = {
          '_id': header.get('uid') or uuid.uuid1().hex,
          'keys': self._build_keys(header),
          'physical': {
           'family': header['physical'].get('family'),
           'ctime': current_time,
           'minRes': CONFIG.storage_minute_resolution,
           'atime': int(current_time),
           'format': self._build_format(header),
          },
          'policy': self._build_policy(header),
          'stats': {
           'accesses': 0,
          },
          'meta': header.get('meta') or {},
         }
     except (KeyError, TypeError, AttributeError) as e:
         _logger.error("Request received did not adhere to expected structure: %(error)s" % {
          'error': str(e),
         })
         self.send_error(409)
         return
     else:
         _logger.info("Proceeding with storage request for '%(uid)s'..." % {
          'uid': record['_id'],
         })
         
     _logger.debug("Evaluating compression requirements...")
     target_compression = record['physical']['format'].get('comp')
     if target_compression and self.request.headers.get('Media-Storage-Compress-On-Server') == 'yes':
         _logger.info("Compressing file...")
         data = compression.get_compressor(target_compression)(data)
         
     _logger.debug("Storing entity...")
     database.add_record(record)
     fs = state.get_filesystem(record['physical']['family'])
     fs.put(record, data)
     
     return {
      'uid': record['_id'],
      'keys': record['keys'],
     }
示例#3
0
 def _process_record(self, record):
     """
     Determines whether the given `record` is a candidate for compression, compressing the
     associated file and updating the record if it is.
     """
     _logger.info("Compressing record '%(uid)s'..." % {
      'uid': record['_id'],
     })
     current_compression = record['physical']['format'].get('comp')
     target_compression = record['policy']['compress'].get('comp')
     if current_compression == target_compression:
         _logger.debug("File already compressed in target format")
         record['policy']['compress'].clear() #Drop the compression policy
         try:
             database.update_record(record)
         except Exception as e:
             _logger.error("Unable to update record to reflect already-applied compression; compression routine will retry later: %(error)s" % {
              'error': str(e),
             })
             return False
         else:
             return True
             
     filesystem = state.get_filesystem(record['physical']['family'])
     data = filesystem.get(record)
     if current_compression: #Must be decompressed first
         _logger.info("Decompressing file...")
         data = compression.get_decompressor(current_compression)(data)
     data = compression.get_compressor(target_compression)(data)
     
     _logger.info("Updating entity...")
     old_format = record['physical']['format'].copy()
     record['physical']['format']['comp'] = target_compression
     try:
         filesystem.put(record, data, tempfile=True)
     except Exception as e: #Harmless backout point
         _logger.warn("Unable to write compressed file to disk; backing out with no consequences")
         return False
     else:
         old_compression_policy = record['policy']['compress'].copy()
         record['policy']['compress'].clear() #Drop the compression policy
         try:
             database.update_record(record)
         except Exception as e: #Results in wasted space until the next attempt
             _logger.error("Unable to update record; old file will be served, and new file will be replaced on a subsequent compression attempt: %(error)s" % {
              'error': str(e),
             })
             return False
         else:
             try:
                 filesystem.make_permanent(record)
             except Exception as e:
                 _logger.error("Unable to update on-disk file; rolling back database update: %(error)s" % {
                  'error': str(e),
                 })
                 record['policy']['compress'] = old_compression_policy
                 record['physical']['format'] = old_format
                 try:
                     database.update_record(record)
                 except Exception as e:
                     _logger.error("Unable to roll back database update; '%(uid)s' is inaccessible and must be manually decompressed from '%(comp)s' format: %(error)s" % {
                      'error': str(e),
                      'uid': record['_id'],
                      'comp': target_compression,
                     })
                 return False
                 
             record['physical']['format'] = old_format
             try:
                 filesystem.unlink(record)
             except Exception as e: #Results in wasted space, but non-fatal
                 _logger.error("Unable to unlink old file; space occupied by '%(uid)s' non-recoverable unless unlinked manually: %(family)r | %(file)s : %(error)s" % {
                  'family': record['physical']['family'],
                  'file': filesystem.resolve_path(record),
                  'uid': record['_id'],
                  'error': str(e),
                 })
             return True