def update(self, uid, write_key,
new={}, removed=(),
deletion_policy=None, compression_policy=None,
timeout=2.5
):
"""
Updates attributes of an existing record on a server.
`timeout` defaults to 2.5s.
All other arguments are the same as in ``media_storage.interfaces.ControlConstruct.update``.
"""
request = common.assemble_request(self._server.get_host() + common.SERVER_UPDATE, {
'uid': uid,
'keys': {
'write': write_key,
},
'policy': {
'delete': deletion_policy,
'compress': compression_policy,
},
'meta': {
'new': new,
'removed': removed,
},
})
common.send_request(request, timeout=timeout)
def get(self, uid, read_key, output_file=None, decompress_on_server=False, timeout=5.0):
"""
Retrieves the requested data from the local proxy, returning its MIME and the decompressed
content as an open filehandle in a tuple.
`output_file` is the path of the file to which the response is written; an anonymous
tempfile is used by default. If supplied, the caller is responsble for cleaning it up.
`decompress_on_server` is ignored; the caching proxy is responsible for that.
`timeout` defaults to 5.0s.
All other arguments are the same as in ``media_storage.interfaces.RetrievalConstruct.get``.
"""
request = common.assemble_request(self._proxy + common.SERVER_GET, {
'uid': uid,
'keys': {
'read': read_key,
},
'proxy': {
'server': self._server.to_dictionary(),
},
})
(properties, response) = common.send_request(request, timeout=timeout)
if not output_file:
output = tempfile.SpooledTemporaryFile(_TEMPFILE_SIZE)
else:
output = open(output_file, 'wb')
output.write(response)
output.seek(0)
return (properties.get(common.PROPERTY_CONTENT_TYPE), output)
def list_families(self, timeout=2.5):
"""
Enumerates all families currently defined on the server, returning a sorted list of strings.
`timeout` is the number of seconds to allow for retrieval to complete, defaulting to 2.5s.
"""
request = common.assemble_request(self._server.get_host() + common.SERVER_LIST_FAMILIES, {})
(properties, response) = common.send_request(request, timeout=timeout)
return json.loads(response)['families']
def ping(self, timeout=1.0):
"""
Indicates whether the server is online or not, raising an exception in case of failure.
`timeout` is the number of seconds to allow for pinging to complete, defaulting to 1.0s.
"""
request = common.assemble_request(self._server.get_host() + common.SERVER_PING, {})
(properties, response) = common.send_request(request, timeout=timeout)
return json.loads(response)
def query(self, query, timeout=5.0):
"""
Returns a list of matching records, up to the server's limit.
`timeout` defaults to 5.0s.
All other arguments are the same as in ``media_storage.interfaces.ControlConstruct.query``.
"""
request = common.assemble_request(self._server.get_host() + common.SERVER_QUERY, query.to_dict())
(properties, response) = common.send_request(request, timeout=timeout)
return json.loads(response)['records']
def put(self, data, mime, family=None,
comp=compression.COMPRESS_NONE, compress_on_server=False,
deletion_policy=None, compression_policy=None,
meta=None,
uid=None, keys=None,
timeout=10.0
):
"""
Stores data on a server, returning a dictionary containing the keys 'uid', which points to
the UID of the stored data, and 'keys', which is another dictionary containing 'read' and
'write', the keys needed to perform either type of action on the stored data.
`data` is a string or file-like object containing the payload to be stored and `mime` is the
MIME-type of the data.
`timeout` defaults to 10.0s, but should be adjusted depending on your needs.
All other arguments are the same as in ``media_storage.interfaces.ControlConstruct.put``.
"""
description = {
'uid': uid,
'keys': keys,
'physical': {
'family': family,
'format': {
'mime': mime,
'comp': comp,
},
},
'policy': {
'delete': deletion_policy,
'compress': compression_policy,
},
'meta': meta,
}
headers = {}
if comp:
if not compress_on_server:
try:
if type(data) in types.StringTypes: #The compressors expect file-like objects
data = StringIO.StringIO(data)
data = compression.get_compressor(comp)(data)
except ValueError:
headers[common.HEADER_COMPRESS_ON_SERVER] = common.HEADER_COMPRESS_ON_SERVER_TRUE
else:
headers[common.HEADER_COMPRESS_ON_SERVER] = common.HEADER_COMPRESS_ON_SERVER_TRUE
request = common.assemble_request(self._server.get_host() + common.SERVER_PUT, description, headers=headers, data=data)
(properties, response) = common.send_request(request, timeout=timeout)
return json.loads(response)
def unlink(self, uid, write_key, timeout=2.5):
"""
Unlinks the identified data on the server.
`timeout` defaults to 2.5s.
All other arguments are the same as in ``media_storage.interfaces.ControlConstruct.unlink``.
"""
request = common.assemble_request(self._server.get_host() + common.SERVER_UNLINK, {
'uid': uid,
'keys': {
'write': write_key,
},
})
common.send_request(request, timeout=timeout)
def put(self, data, mime, family=None,
comp=compression.COMPRESS_NONE, compress_on_server=False,
deletion_policy=None, compression_policy=None,
meta=None,
uid=None, keys=None,
timeout=3.0
):
"""
Stores data in the proxy's buffers, immediately returning a dictionary containing the keys
'uid', which points to the eventual UID of the stored data, and 'keys', which is another
dictionary containing 'read' and 'write', the keys needed to perform either type of action
on the stored data.
It is important to note that the data is NOT actually stored when this pointer is returned,
but rather that the pointer will be valid at some point in the future (typically very soon,
but not within a predictable timeframe).
`data` is a string containing the local filesystem path of the data to store and `mime` is
the MIME-type of the data.
`timeout` defaults to 3.0s, but should be adjusted depending on your server's performance.
All other arguments are the same as in ``media_storage.interfaces.ControlConstruct.put``.
"""
description = {
'uid': uid,
'keys': keys,
'physical': {
'family': family,
'format': {
'mime': mime,
'comp': comp,
},
},
'policy': {
'delete': deletion_policy,
'compress': compression_policy,
},
'meta': meta,
'proxy': {
'server': self._server.to_dictionary(),
'data': data,
},
}
request = common.assemble_request(self._proxy + common.SERVER_PUT, description)
(properties, response) = common.send_request(request, timeout=timeout)
return json.loads(response)
def status(self, timeout=2.5):
"""
Yields a dictionary of load data from the server::
'process': {
'cpu': {'percent': 0.1,},
'memory': {'percent': 1.2, 'rss': 8220392,},
'threads': 4,
},
'system': {
'load': {'t1': 0.2, 't5': 0.5, 't15': 0.1,},
}
`timeout` is the number of seconds to allow for retrieval to complete, defaulting to 2.5s.
"""
request = common.assemble_request(self._server.get_host() + common.SERVER_STATUS, {})
(properties, response) = common.send_request(request, timeout=timeout)
return json.loads(response)
def describe(self, uid, read_key, timeout=2.5):
"""
Retrieves the requested record from the server as a dictionary.
`timeout` defaults to 2.5s.
All other arguments are the same as in
``media_storage.interfaces.ControlConstruct.describe``.
"""
request = common.assemble_request(self._server.get_host() + common.SERVER_DESCRIBE, {
'uid': uid,
'keys': {
'read': read_key,
},
})
(properties, response) = common.send_request(request, timeout=timeout)
response = json.loads(response)
if not response['physical']['exists']:
raise common.NotPresentError(response)
return response
def describe(self, uid, read_key, timeout=2.5):
"""
Retrieves the requested record from the local proxy as a dictionary.
`timeout` defaults to 2.5s.
All other arguments are the same as in
``media_storage.interfaces.ControlConstruct.describe``.
"""
request = common.assemble_request(self._proxy + common.SERVER_DESCRIBE, {
'uid': uid,
'keys': {
'read': read_key,
},
'proxy': {
'server': self._server.to_dictionary(),
},
})
(properties, response) = common.send_request(request, timeout=timeout)
return json.loads(response)
def get(self, uid, read_key, output_file=None, decompress_on_server=False, timeout=5.0):
"""
Retrieves the requested data from the server, returning its MIME and the decompressed
content as a file-like object (optionally that supplied as `output_file`) in a tuple; the
file-like object has a ``length`` parameter that contains its length in bytes.
`output_file` is an optional file-like object to which data should be written (a spooled
tempfile is used by default).
`timeout` defaults to 5.0s.
All other arguments are the same as in ``media_storage.interfaces.ControlConstruct.get``.
"""
headers = {}
if not decompress_on_server: #Tell the server what the client supports
headers[common.HEADER_SUPPORTED_COMPRESSION] = common.HEADER_SUPPORTED_COMPRESSION_DELIMITER.join(compression.SUPPORTED_FORMATS)
request = common.assemble_request(self._server.get_host() + common.SERVER_GET, {
'uid': uid,
'keys': {
'read': read_key,
},
}, headers=headers)
if not output_file:
output = tempfile.SpooledTemporaryFile(_TEMPFILE_SIZE)
else:
output = output_file
properties = common.send_request(request, output=output, timeout=timeout)
length = properties.get(common.PROPERTY_CONTENT_LENGTH)
if properties.get(common.PROPERTY_APPLIED_COMPRESSION):
output = compression.get_decompressor(properties.get(common.PROPERTY_APPLIED_COMPRESSION))(output)
if output_file: #The decompression process returns a tempfile
output_file.seek(0)
output_file.truncate()
length = common.transfer_data(output, output_file)
output_file.seek(0)
output = output_file
output.length = length
return (properties.get(common.PROPERTY_CONTENT_TYPE), output)