def store_resource(self, content, meta):
id = ObjectId() # Generate one so I can use it to assemble the url
resc = GridFSProxy()
kwargs = {k: meta[k] for k in self.resource_meta if meta.get(k)}
kwargs.update({'url': reverse('resource', args=(id,)), '_id': id})
resc.put(content, **kwargs)
return resc
def store_resoure(content, filename=None, content_type=None,
content_disposition=None, **kwargs):
"""A helper to create Resource instance"""
http_meta = {}
if content_type:
http_meta['content_type'] = content_type
if content_disposition:
http_meta['content_disposition'] = content_disposition
resc = GridFSProxy()
# Store there for resource view
resc.put(content, **http_meta)
return Resource(gridfs=resc,
filename=filename, **kwargs)
def put_model(self,
obj,
name,
attributes=None,
serving_input_fn=None,
strip_default_attrs=None,
**kwargs):
export_dir_base = self._make_savedmodel(
obj,
serving_input_receiver_fn=serving_input_fn,
strip_default_attrs=strip_default_attrs)
zipfname = self._package_savedmodel(export_dir_base, name)
with open(zipfname, 'rb') as fzip:
fileid = self.model_store.fs.put(
fzip,
filename=self.model_store._get_obj_store_key(
name, self._model_ext))
gridfile = GridFSProxy(grid_id=fileid,
db_alias='omega',
collection_name=self.model_store.bucket)
rmtree(export_dir_base)
os.remove(zipfname)
return self.model_store._make_metadata(name=name,
prefix=self.model_store.prefix,
bucket=self.model_store.bucket,
kind=self.KIND,
attributes=attributes,
gridfile=gridfile).save()
def put(self,
obj,
name,
attributes=None,
prefix=None,
bucket=None,
encoding=None,
**kwargs):
self.data_store.drop(name, force=True)
fn = self.data_store._get_obj_store_key(name, 'file')
if self._is_path(obj):
with open(obj, 'rb') as fin:
fileid = self.data_store.fs.put(fin, filename=fn)
else:
fileid = self.data_store.fs.put(obj,
filename=fn,
encoding=encoding)
gridfile = GridFSProxy(grid_id=fileid,
db_alias='omega',
collection_name=self.data_store.bucket)
return self.data_store._make_metadata(name=name,
prefix=prefix
or self.data_store.prefix,
bucket=bucket
or self.data_store.bucket,
kind=self.KIND,
attributes=attributes,
gridfile=gridfile).save()
def put(self, obj, name, attributes=None, **kwargs):
"""
save a python package
:param obj: full path to package file or directory, sytanx as
pkg://path/to/dist.tar.gz or pkg://path/to/setup.py
:param name: name of package. must be the actual name of the package
as specified in setup.py
:return: the Metadata object
"""
pkgsrc = pkgdist = obj.split('//')[1]
if not 'tar.gz' in os.path.basename(pkgdist):
distdir = os.path.join(pkgsrc, 'dist')
sdist = build_sdist(pkgsrc, distdir)
version = sdist.metadata.version
pkgname = sdist.metadata.name
pkgdist = os.path.join(
distdir, '{pkgname}-{version}.tar.gz'.format(**locals()))
with open(pkgdist, 'rb') as fzip:
fileid = self.data_store.fs.put(
fzip, filename=self.data_store._get_obj_store_key(name, 'pkg'))
gridfile = GridFSProxy(grid_id=fileid,
db_alias='omega',
collection_name=self.data_store.bucket)
return self.data_store._make_metadata(name=name,
prefix=self.data_store.prefix,
bucket=self.data_store.bucket,
kind=PythonPackageData.KIND,
attributes=attributes,
gridfile=gridfile).save()
def get_b64_image_common(self, image: GridFSProxy):
if not image:
return None
byte_array = image.read()
if not byte_array:
return None
byte_string = bytes(byte_array)
b64_string = b64encode(byte_string)
return b64_string.decode('utf-8')
def put_model(self, obj, name, attributes=None, **kwargs):
fn = temp_filename()
self._save_model(obj, fn)
with open(fn, mode='rb') as fin:
fileid = self.model_store.fs.put(
fin, filename=self.model_store._get_obj_store_key(name, 'h5'))
gridfile = GridFSProxy(grid_id=fileid,
db_alias='omega',
collection_name=self.model_store.bucket)
remove_temp_filename(fn)
return self.model_store._make_metadata(name=name,
prefix=self.model_store.prefix,
bucket=self.model_store.bucket,
kind=self.KIND,
attributes=attributes,
gridfile=gridfile).save()
def put_model(self, obj, name, attributes=None):
# create a copy so we can reset the model dir
# this is required so the model path does not get restored on get
obj = copy(obj)
obj._model_dir = None
zipfname = self._package_model(obj, name)
with open(zipfname, 'rb') as fzip:
fileid = self.model_store.fs.put(
fzip,
filename=self.model_store._get_obj_store_key(name, '.tfm'))
gridfile = GridFSProxy(grid_id=fileid,
db_alias='omega',
collection_name=self.model_store.bucket)
return self.model_store._make_metadata(name=name,
prefix=self.model_store.prefix,
bucket=self.model_store.bucket,
kind=self.KIND,
attributes=attributes,
gridfile=gridfile).save()
def put(self, obj, name, attributes=None, **kwargs):
# TODO add obj signing so that only trustworthy sources can put functions
# ensure we have a dill'able object
# -- only instances can be dill'ed
if isinstance(obj, six.class_types):
obj = obj()
data = dill.dumps(obj)
filename = self.model_store._get_obj_store_key(name, '.dill')
fileid = self.model_store.fs.put(data, filename=filename)
gridfile = GridFSProxy(grid_id=fileid,
db_alias='omega',
collection_name=self.model_store.bucket)
return self.model_store._make_metadata(
name=name,
prefix=self.model_store.prefix,
bucket=self.model_store.bucket,
kind=self.KIND,
attributes=attributes,
gridfile=gridfile).save()
def put(self, obj, name, attributes=None, **kwargs):
serialized = self._serializer(obj).serialize(wrap=False)
bbuf = BytesIO(serialized)
bbuf.seek(0)
# see if we have a file already, if so replace the gridfile
meta = self.data_store.metadata(name)
if not meta:
filename = uuid4().hex
fileid = self._fs.put(bbuf, filename=filename)
meta = self.data_store._make_metadata(
name=name,
prefix=self.data_store.prefix,
bucket=self.data_store.bucket,
kind=DashAppBackend.KIND,
attributes=attributes,
gridfile=GridFSProxy(grid_id=fileid))
else:
meta.gridfile.replace(bbuf)
return meta.save()
def put(self, obj, name, attributes=None, **kwargs):
data = obj.SerializeToString()
fileid = self.model_store.fs.put(
data, filename=self.model_store._get_obj_store_key(name, '.pbf'))
gridfile = GridFSProxy(grid_id=fileid,
db_alias='omega',
collection_name=self.model_store.bucket)
kind_meta = {
'protobuf_type':
'{module}.{name}'.format(
module=obj.DESCRIPTOR._concrete_class.__module__,
name=obj.DESCRIPTOR._concrete_class.__name__)
}
return self.model_store._make_metadata(name=name,
prefix=self.model_store.prefix,
bucket=self.model_store.bucket,
kind=self.KIND,
kind_meta=kind_meta,
attributes=attributes,
gridfile=gridfile).save()
def _store_to_file(self,
store,
obj,
filename,
encoding=None,
replace=False):
"""
Use this method to store file-like objects to the store's gridfs
Args:
store (OmegaStore): the store whose .fs filesystem access will be used
obj (file-like): a file-like object or path, if path will be opened with mode=rb,
otherwise its obj.read() method is called to get the data as a byte stream
filename (path): the path in the store (key)
encoding (str): a valid encoding such as utf8, optional
replace (bool): if True the existing file(s) of the same name are deleted to avoid automated versioning
by gridfs. defaults to False
Returns:
gridfile (GridFSProxy), assignable to Metadata.gridfile
"""
if replace:
for fileobj in store.fs.find({'filename': filename}):
try:
store.fs.delete(fileobj._id)
except Exception as e:
warn('deleting {filename} resulted in {e}'.format(
**locals()))
pass
if self._is_path(obj):
with open(obj, 'rb') as fin:
fileid = store.fs.put(fin,
filename=filename,
encoding=encoding)
else:
fileid = store.fs.put(obj, filename=filename, encoding=encoding)
gridfile = GridFSProxy(grid_id=fileid,
db_alias=store._dbalias,
key=filename,
collection_name=store._fs_collection)
return gridfile
def download_minidump(self, minidump_id):
minidump = GridFSProxy(ObjectId(minidump_id))
file = minidump.read()
response = make_response(file)
response.mimetype = 'application/octet-stream'
return response
def read_image_to_arr(object_id):
bimage = GridFSProxy(ObjectId(object_id), collection_name=Photo.image.collection_name).get().read()
image = PIL.Image.open(BytesIO(bimage))
return preprocess_image(image)
def to_file(self):
return GridFSProxy(self)
def _add_report_object(self, file, target):
proxy = GridFSProxy()
proxy.put(file)
target[file.name] = proxy