class ExportedCompressedFile(Document):
""" Represents exported files
"""
file_name = fields.StringField()
file = fields.FileField(
blank=True, collection_name=GRIDFS_EXPORTED_COMPRESSED_FILE_COLLECTION)
is_ready = fields.BooleanField(default=False)
mime_type = fields.StringField()
@staticmethod
def get_by_id(object_id):
""" Get Exported compressed file with the given id
Args:
object_id:
Returns:
"""
try:
return ExportedCompressedFile.objects.get(pk=str(object_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(e.message)
except Exception as ex:
raise exceptions.ModelError(ex.message)
class TSNEMap(Document):
"""T-SNE map class
"""
file = fields.FileField(blank=False, collection_name=GRIDFS_TSNE_COLLECTION)
@staticmethod
def get_by_id(tsne_map_id):
""" Return the object with the given id.
Args:
tsne_map_id:
Returns:
Data (obj): T-SNE map with the given id
"""
try:
return TSNEMap.objects.get(pk=str(tsne_map_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(e.message)
except Exception as ex:
raise exceptions.ModelError(ex.message)
@staticmethod
def get_last():
""" Return latest T-SNE map
Returns:
"""
# TODO: need testing
return TSNEMap.objects().latest('id')
def clean(self):
""" Clean is called before saving
Returns:
"""
# read csv content
csv_content = self.file.read()
if not check_headers(csv_content, CSV_HEADERS):
raise ModelError("Expected CSV file should have the following headers: {}".format(','.join(CSV_HEADERS)))
class AbstractData(Document):
"""AbstractData object"""
dict_content = fields.DictField(blank=True)
title = fields.StringField(blank=False,
validation=not_empty_or_whitespaces)
xml_file = fields.FileField(blank=False,
collection_name=GRIDFS_DATA_COLLECTION)
creation_date = fields.DateTimeField(blank=True, default=None)
last_modification_date = fields.DateTimeField(blank=True, default=None)
last_change_date = fields.DateTimeField(blank=True, default=None)
_xml_content = None
meta = {
"abstract": True,
}
@property
def xml_content(self):
"""Get xml content - read from a saved file.
Returns:
"""
# private field xml_content not set yet, and reference to xml_file to read is set
if self._xml_content is None and self.xml_file is not None:
# read xml file into xml_content field
xml_content = self.xml_file.read()
try:
self._xml_content = (xml_content.decode("utf-8")
if xml_content else xml_content)
except AttributeError:
self._xml_content = xml_content
# return xml content
return self._xml_content
@xml_content.setter
def xml_content(self, value):
"""Set xml content - to be saved as a file.
Args:
value:
Returns:
"""
# update modification times
self.last_modification_date = datetime_now()
# update content
self._xml_content = value
def convert_and_save(self):
"""Save Data object and convert the xml to dict if needed.
Returns:
"""
self.convert_to_dict()
self.convert_to_file()
return self.save_object()
def convert_to_dict(self):
"""Convert the xml contained in xml_content into a dictionary.
Returns:
"""
# transform xml content into a dictionary
dict_content = xml_utils.raw_xml_to_dict(self.xml_content,
xml_utils.post_processor)
# if limit on element occurrences is set
if SEARCHABLE_DATA_OCCURRENCES_LIMIT is not None:
# Remove lists which size exceed the limit size
xml_utils.remove_lists_from_xml_dict(
dict_content, SEARCHABLE_DATA_OCCURRENCES_LIMIT)
# store dictionary
self.dict_content = dict_content
def convert_to_file(self):
"""Convert the xml string into a file.
Returns:
"""
try:
xml_file = BytesIO(self.xml_content.encode("utf-8"))
except Exception:
xml_file = BytesIO(self.xml_content)
if self.xml_file.grid_id is None:
# new file
self.xml_file.put(xml_file, content_type="application/xml")
else:
# editing (self.xml_file gets a new id)
self.xml_file.replace(xml_file, content_type="application/xml")
def save_object(self):
"""Custom save. Set the datetime fields and save.
Returns:
"""
try:
# initialize times
now = datetime_now()
# update change date every time the data is updated
self.last_change_date = now
if not self.id:
# initialize when first created
self.creation_date = now
# initialize when first saved, then only updates when content is updated
self.last_modification_date = now
return self.save()
except mongoengine_errors.NotUniqueError as e:
raise exceptions.NotUniqueError(e)
except Exception as ex:
raise exceptions.ModelError(ex)
class AbstractData(Document):
""" AbstractData object
"""
dict_content = fields.DictField(blank=True)
title = fields.StringField(blank=False, regex=NOT_EMPTY_OR_WHITESPACES)
last_modification_date = fields.DateTimeField(blank=True, default=None)
xml_file = fields.FileField(blank=False,
collection_name=GRIDFS_DATA_COLLECTION)
_xml_content = None
meta = {
'abstract': True,
}
@property
def xml_content(self):
"""Get xml content - read from a saved file.
Returns:
"""
# private field xml_content not set yet, and reference to xml_file to read is set
if self._xml_content is None and self.xml_file is not None:
# read xml file into xml_content field
self._xml_content = self.xml_file.read()
# return xml content
return self._xml_content
@xml_content.setter
def xml_content(self, value):
"""Set xml content - to be saved as a file.
Args:
value:
Returns:
"""
self._xml_content = value
def convert_and_save(self):
""" Save Data object and convert the xml to dict if needed.
Returns:
"""
self.convert_to_dict()
self.convert_to_file()
return self.save()
def convert_to_dict(self):
""" Convert the xml contained in xml_content into a dictionary.
Returns:
"""
# transform xml content into a dictionary
dict_content = xml_utils.raw_xml_to_dict(self.xml_content,
xml_utils.post_processor)
# if limit on element occurrences is set
if SEARCHABLE_DATA_OCCURRENCES_LIMIT is not None:
# Remove lists which size exceed the limit size
xml_utils.remove_lists_from_xml_dict(
dict_content, SEARCHABLE_DATA_OCCURRENCES_LIMIT)
# store dictionary
self.dict_content = dict_content
def convert_to_file(self):
""" Convert the xml string into a file.
Returns:
"""
try:
xml_file = BytesIO(self.xml_content.encode('utf-8'))
except Exception:
xml_file = BytesIO(self.xml_content)
if self.xml_file.grid_id is None:
# new file
self.xml_file.put(xml_file, content_type="application/xml")
else:
# editing (self.xml_file gets a new id)
self.xml_file.replace(xml_file, content_type="application/xml")