class ExchangeData(Document):
open_time= fields.IntField()
close_time = fields.IntField()
price = fields.IntField()
value = fields.IntField()
exchnage = fields.StringField(max_length=255)
updated = fields.DateTimeField(default=datetime.datetime.utcnow)
class HistoryQuery(Document):
""" History object to retrieve a query to a specific step
"""
user_id = fields.IntField() # User link to the history
query_id = fields.StringField() # Query link to the history
message = fields.StringField() # History message
status = fields.IntField(
) # History status (0: Query step, 1: Waiting for treatment, 2: Output files created)
@staticmethod
def get_by_id(history_query_id):
""" Return a HistoryQuery given its id.
Args:
history_query_id:
Returns:
"""
try:
return HistoryQuery.objects.get(pk=str(history_query_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
@staticmethod
def get_all():
""" Return a list of all HistoryQuery.
Returns:
"""
return HistoryQuery.objects().all()
@staticmethod
def get_all_by_user_id(user_id):
""" Return a list of HistoryQuery given their user_id.
Args:
user_id:
Returns:
"""
return HistoryQuery.objects.filter(user_id=str(user_id))
def delete_database(self, from_query_admin=False):
""" Delete function.
If the deletion come from the admin panel, Delete all the user query linked to this history.
Args:
from_query_admin: Variable to avoid circle deletion.
"""
if not from_query_admin:
temp_user_queries = TempUserQuery.objects.filter(id=self.query_id)
for temp_user_query in temp_user_queries:
temp_user_query.delete_database(from_history=True)
self.delete()
class GlobalElement(Document):
"""
Storage model for a globally defined element within a schema document.
Such an element can be used as the root element of a document.
:property name str: the local name for the element
:property namespace str: the namespace URI of the schema within which the
element is defined
:property version int: the version of the schema document that this
:property schemaname str: the unique name given to the schema document where
the element is defined
element is defined in
:property schema ref: a reference to the SchemaVersion record where
this version of the element is defined
:property annots ref: a reference to the GlobalElementAnnots that
contains user annotations.
"""
name = fields.StringField(
unique_with=["namespace", "schemaname", "version"])
namespace = fields.StringField()
schemaname = fields.StringField()
version = fields.IntField()
schema = fields.ReferenceField(SchemaVersion)
annots = fields.ReferenceField(GlobalElementAnnots)
@property
def qname(self):
"""
the type's qualified name of the form "{NS}LOCAL-NAME"
"""
return "{{{0}}}{1}".format(self.namespace, self.name)
@classmethod
def get_all_elements(cls):
"""
return all of the global elements from all the representative
namespaces that aren't marked as hidden.
:return dict: a dictionary that has namespaces as keys and
GlobalElement records as values.
"""
currents = {}
scs = SchemaCommon.objects.filter(current__gt=0)
for sc in scs:
sv = SchemaVersion.get_by_version(sc.name, sc.current)
if sv:
currents[sc.name] = sc.current
elsbyns = {}
for name in currents:
glels = GlobalElement.objects.filter(schemaname=name) \
.filter(version=currents[name])
for el in glels:
if el.annots.hide:
continue
if el.namespace not in elsbyns:
elsbyns[el.namespace] = []
elsbyns[el.namespace].append(el)
return elsbyns
class WebPage(Document):
"""Represents a WebPage"""
type = fields.IntField()
content = fields.StringField()
@staticmethod
def get_by_type(page_type):
"""Get a WebPage given its type
Parameters:
page_type (str): page type
Returns:
Web Page corresponding to the given type
"""
try:
return WebPage.objects.get(type=WEB_PAGE_TYPES[page_type])
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
@staticmethod
def delete_by_type(page_type_key):
"""Delete all WebPage with the given type key
Args:
page_type_key (int): page type key
Returns:
"""
return WebPage.objects(type=page_type_key).delete()
class Status(Document):
user_id = fields.IntField()
criteria = fields.EmbeddedDocumentField(StatusCriteria)
device_id = fields.StringField()
active = fields.BooleanField()
@property
def device(self):
return Device.objects.get(id=self.device_id)
def current_value(self):
return self.criteria.metric_value_for(self.device)
def current_check(self):
return self.criteria.operation_check(self.current_value())
def __str__(self):
return "{}{}".format(self.device_id, self.criteria)
@staticmethod
def only_triggered_for(user_id):
active = Status.objects.filter(user_id=user_id, active=True)
triggered_ids = []
for obj in active:
if obj.current_check():
triggered_ids.append(str(obj.id))
return Status.objects.filter(id__in=triggered_ids)
@staticmethod
def for_user(user_id):
return Status.objects.filter(user_id=user_id)
class User(Document):
name = fields.StringField()
username = fields.StringField(unique=True)
email = fields.EmailField(unique=True)
# these 2 fields are NOT TO BE FILLED
ratings = fields.IntField(default=0)
date_joined = fields.DateTimeField(default=timezone.now, editable=False)
gears = fields.ListField(fields.ReferenceField('Gear'),
default=[],
blank=True)
password = fields.StringField(min_length=8, max_length=128)
USERNAME_FIELD = 'username'
REQUIRED_FIELDS = ['name', 'email', 'password']
def __str__(self):
return self.username
def get_username(self):
return self.username
def get_name(self):
return self.name
def get_ratings(self):
return self.ratings
def is_active(self):
return True
def check_password(self, raw_password):
""" Checks the password if it matches the stored password """
return check_password(raw_password, self.password)
class Profile(Document):
#user = fields.ReferenceField('User')
user_id = fields.IntField()
full_name = fields.StringField(max_length=191, blank=True)
location = fields.DictField({}, blank=True)
settings = fields.DictField({}, blank=True)
has_at_least_one_social_account = fields.BooleanField(default=False)
social_accounts = fields.ListField(fields.DictField({}, blank=True),
blank=True)
associated_platforms = fields.ListField(fields.DictField(), blank=True)
channels = fields.ListField(fields.DictField(blank=True), blank=True)
authorized_devices = fields.ListField(fields.DictField(blank=True),
blank=True)
birth_date = fields.DateTimeField(blank=True)
roles = fields.ListField(fields.DictField({}, blank=True), blank=True)
permissions = fields.ListField(fields.DictField({}, blank=True),
blank=True)
signed_up_with = fields.StringField(defalut="email")
signed_up_ip_address = fields.StringField(blank=True)
singed_up_user_agent = fields.StringField(blank=True)
logged_in_ip_address = fields.StringField(blank=True)
logged_in_user_agent = fields.StringField(blank=True)
meta = {"collection": "auth_profile"}
def __str__(self):
return self.mobile_number
def check_otp(self, otp):
return check_password(otp, self.otp)
class Notifications(EmbeddedDocument):
user_id = fields.StringField(unique=False)
notification_id = fields.StringField(unique=False)
notification_received_at = fields.DateTimeField(default=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
unique=False)
notification_type = fields.IntField()
notification_details = fields.EmbeddedDocumentField(NotificationDetails, required=False,
default= None, blank=True,unique=False)
class ReportPosts(Document):
post_id = fields.StringField(max_length=36,required=True)
user_id = fields.StringField(required=True, max_length=36)
report_id = fields.StringField(max_length=36,required=True)
report_type = fields.IntField(default=0)
created_at = fields.DateTimeField(
default=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
, required=True, unique=False
)
class DynQueryStep(Document):
"""
Step object created by the administrator.
"""
# Step name
name = fields.StringField(blank=False, unique=False)
# True if the step is not query-able, False else
output_field = fields.BooleanField(blank=False, unique=False)
# Relative path to the object, in dot notation
xpath = fields.StringField(blank=False, unique=False)
# Define the part of the node concerned by the
target = fields.StringField(blank=False, unique=False)
# query, could be name, attribute, or value
value = fields.StringField(blank=True, unique=False)
# Define the name of the attribute concerned by
data_type = fields.StringField(blank=False, unique=False)
# Define if the there is a restriction to the step
restriction = fields.BooleanField(blank=True, unique=False)
# Define the minimum range for multiple choices step
data_min_range = fields.IntField(blank=True, unique=False)
# Define the maximum range for multiple choices step
data_max_range = fields.IntField(blank=True, unique=False)
# Define the minimum range to infinity
data_infinity = fields.BooleanField(blank=True, unique=False) #
# Define the number of days for date query
date_range = fields.IntField(blank=True, unique=False)
@staticmethod
def get_by_id(dyn_query_step_id):
"""Return a DynQueryStep given its id.
Args:
dyn_query_step_id:
Returns:
"""
try:
return DynQueryStep.objects.get(pk=str(dyn_query_step_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
class WebTxLocal(Document):
sender = fields.StringField(max_length=250)
page_hash = fields.StringField(max_length=250)
page_name = fields.StringField(max_length=250)
last_page_hash = fields.StringField(max_length=250)
timestamp = fields.IntField(default=0)
page_content = fields.StringField(default='', max_length=2000000)
def __unicode__(self):
return self.page_hash
class Book(Document):
id = fields.StringField(primary_key=True, default=ObjectId)
name = fields.StringField(max_length=300)
slug = fields.StringField()
pages = fields.IntField()
authors = fields.ListField(fields.ReferenceField(Author))
pubdate = fields.DateTimeField()
_meta = {"ordering": ["-pubdate"]}
def __unicode__(self):
return self.name or ""
class RhythmPosts(Document):
post_id = fields.StringField(max_length=36,required=True,unique=True)
user_id = fields.StringField(required=True, max_length=36)
poster_url = fields.StringField(
default = "http://profile.ak.fbcdn.net/static-ak/rsrc.php/v2/yo/r/UlIqmHJn-SK.gif",
unique=False, required=True
)
created_at = fields.DateTimeField(
default=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
, required=True, unique=False
)
post_likes = fields.EmbeddedDocumentListField(LikeDetails, required=False,
default= [], blank=True,unique=False)
total_likes = fields.IntField(default=0)
post_comments = fields.EmbeddedDocumentListField(CommentDetails, required=False,
default= [], blank=True,unique=False)
total_comments = fields.IntField(default=0)
is_comment_allowed = fields.BooleanField(default=True)
post_caption = fields.StringField(unique=False, required=False, null=True)
song_name = fields.StringField(unique=False, required=True)
album = fields.StringField(unique=False, required=False, null=True)
ratings = fields.DecimalField(min_value=0, max_value=5, default=0)
class Template(dme_Document):
"""Represents an XML schema template that defines the structure of data for curation"""
title = dme_fields.StringField()
filename = dme_fields.StringField()
content = dme_fields.StringField()
templateVersion = dme_fields.StringField(blank=True)
version = dme_fields.IntField(blank=True)
hash = dme_fields.StringField()
user = dme_fields.StringField(blank=True)
dependencies = dme_fields.ListField(StringField(), blank=True)
exporters = dme_fields.ListField(ReferenceField(Exporter, reverse_delete_rule=PULL), blank=True)
XSLTFiles = dme_fields.ListField(ReferenceField(ExporterXslt, reverse_delete_rule=PULL), blank=True)
ResultXsltList = dme_fields.ReferenceField(ResultXslt, reverse_delete_rule=NULLIFY, blank=True)
ResultXsltDetailed = dme_fields.ReferenceField(ResultXslt, reverse_delete_rule=NULLIFY, blank=True)
class Tmall(DynamicDocument):
title = fields.StringField()
store_id = fields.StringField()
url = fields.URLField()
price = fields.FloatField()
sale_num = fields.IntField()
image_urls = fields.ListField()
scrapy_mongodb = fields.DictField(db_field='scrapy-mongodb')
def __str__(self):
return self.title
@property
def cover(self):
return self.image_urls[0]
class DynQuery(Document):
"""
Query object created by the administrator.
"""
# Query name
name = fields.StringField(blank=False, unique=True)
# Limit the query to number_records records
is_limited = fields.BooleanField(blank=False, default=False)
# Records number to limit the query
number_records = fields.IntField(blank=True)
# Group query
group = fields.StringField(blank=False)
# Schema associated to the query
# FIXME: replace by a reference field
schema = fields.StringField(blank=False)
# List of step links to the query
steps = fields.ListField(
fields.ReferenceField(DynQueryStep,
blank=False,
reverse_delete_rule=CASCADE))
@staticmethod
def get_by_id(dyn_query_id):
"""Return a DynQuery given its id.
Args:
dyn_query_id:
Returns:
"""
try:
return DynQuery.objects.get(pk=str(dyn_query_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
@staticmethod
def get_all():
"""Return all DynQueries.
Returns:
"""
return DynQuery.objects().all()
class TemplateCommon(Document):
"""
Storage model for a document curation template. This captures
:property name str: a unique name
:property current int: the version number of the schema that should be
considered the current one.
:property root str: a namespace-qualified element name (in the form
"SCHEMANAME:ELNAME") for the root element of
conforming instance documents.
:property desc str: a brief (displayable) description of the template.
"""
name = fields.StringField(unique=True)
current = fields.IntField(blank=False)
root = fields.StringField(blank=False)
desc = fields.StringField(default="")
@classmethod
def get_by_name(self, name, allowdeleted=False):
"""
Return the TemplateCommon record having the given name (which
must be unique).
:param name str: the user-given name for the schema to look up
:param allowdeleted bool: if true, include records where current <= 0
"""
out = TemplateCommon.objects.filter(name=name)
if not allowdeleted:
out = out.filter(current__gt=0)
if len(out) > 0:
return out[0]
def get_current_version(self):
"""
return the TemplateVersion record that corresponds to the current
instance of this schema. None is returned if there is no version
currently marked as current.
"""
return TemplateVersion.get_by_version(self.name, self.current)
@classmethod
def get_names(self):
"""
return a list of all user-provided names currently registered
"""
return map(lambda s: s.name, SchemaCommon.objects.all())
class BioSubject(Feature):
species = fields.ReferenceField(Species)
subjectname = fields.StringField(max_length=255)
subjectspeciesname = fields.StringField(max_length=1024)
subjecttaxon = fields.IntField(default=None, null=True)
strain = fields.StringField(max_length=1024, default=None, null=True)
subjectdescription = fields.StringField(default="")
dob = fields.DateTimeField(default=None, null=True)
sex = fields.StringField(max_length=1, default=None, null=True)
cohort = fields.StringField(max_length=10, default=None, null=True)
changed = fields.BooleanField(default=False)
comment = fields.StringField(max_length=1024, default=None, null=True)
do_ignore = fields.BooleanField(default=False)
centre = fields.StringField(max_length=255, default=None, null=True)
def GetName(self):
return self.subjectname
class User(Document):
created_time = fields.DateTimeField(
default=datetime.datetime.now, editable=False,
)
user_name = fields.StringField(max_length=250)
phone_num = fields.StringField(default='', max_length=250)
phone_zone_code = fields.StringField(default='', max_length=250)
password = fields.StringField(default='', max_length=250)
address = fields.StringField(default='', max_length=250)
access_token = fields.StringField(default='', max_length=250)
password_site = fields.StringField(default='', max_length=250)
client_id = fields.StringField(default='', max_length=250)
client_secret = fields.StringField(default='', max_length=250)
expires_in = fields.IntField(default=7200)
def __unicode__(self):
return self.user_name
class LogFile(Document):
"""
Error reports for administrators
"""
application = fields.StringField(
blank=False) # Django application name concerned by the error
code = fields.IntField(blank=True) # Error code
message = fields.StringField(blank=False) # Error message
additionalInformation = fields.DictField(
blank=True) # Dictionary of additional information
timestamp = fields.DateTimeField(blank=True) # Error creation date
@staticmethod
def get_by_id(log_file_id):
"""Return a LogFile given its id.
Args:
log_file_id:
Returns:
"""
try:
return LogFile.objects.get(pk=str(log_file_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
@staticmethod
def get_all():
"""Return all LogFiles.
Returns:
"""
return LogFile.objects().all()
def delete_database(self):
"""
Delete function
"""
self.delete()
class WebPage(Document):
"""Represents a WebPage
"""
type = fields.IntField(choices=WEB_PAGE_TYPES.values())
content = fields.StringField()
@staticmethod
def get_by_type(page_type):
"""Get a WebPage given its type
Parameters:
page_type (str): page type of the page
Returns:
Web Page corresponding to the given type
"""
try:
return WebPage.objects.get(type=WEB_PAGE_TYPES[page_type])
except:
raise WebsiteWebPageDoesNotExistError("Web page does not exist")
class Event(Document):
title = fields.StringField(max_length=255)
description = fields.StringField(max_length=255)
location = fields.StringField(max_length=255)
landmark = fields.StringField(max_length=255)
city = fields.StringField(max_length=255)
coordinates = fields.ListField(fields.FloatField())
start_timestamp = fields.DateTimeField()
end_timestamp = fields.DateTimeField()
banner = fields.StringField()
#banner = fields.ImageField(upload_to='banners', default="images/None/no-img.jpeg")
#banner = fields.ImageField(upload_to=fs, default="images/None/no-img.jpeg")
tags = fields.ListField(fields.StringField())
recurring = fields.BooleanField(default=False)
open_event = fields.BooleanField(default=True)
full_day_event = fields.BooleanField(default=False)
created_by = fields.IntField()
user_agent = fields.StringField(blank=True)
timestamp = fields.DateTimeField(default=datetime.datetime.now)
updated_at = fields.DateTimeField(default=datetime.datetime.now)
slug = fields.StringField(blank=True)
ip_address = fields.StringField(blank=True)
channel = fields.ObjectIdField()
deleted = fields.BooleanField(default=False)
deleted_at = fields.DateTimeField(blank=True)
meta = {'collection': 'events'}
def __str__(self):
return self.title
@classmethod
def pre_save(cls, sender, document, **kwargs):
document.title = document.title.capitalize()
document.description = document.description.capitalize()
document.location = document.location.title()
document.landmark = document.landmark.title()
document.city = document.city.title()
if not document.slug:
document.slug = slugify(document.title)
class Tenant(Document):
user_id = fields.IntField()
tenant_id = fields.StringField()
@property
def user(self):
return User.objects.get(id=self.user_id)
def __str__(self):
return "{} <=> {}".format(self.user_id, self.tenant_id)
@staticmethod
def for_user(user_id):
return Tenant.objects.filter(user_id=user_id)
@staticmethod
def id_for_user(user_id):
try:
return Tenant.objects.get(user_id=user_id).tenant_id
except Tenant.DoesNotExist:
return None
class OaiRequestPage(Document):
"""Informations about a request sent by a harvester needed a paginated
response.
"""
resumption_token = fields.StringField(blank=False, unique=True)
template_id_list = fields.ListField(blank=False)
metadata_format = fields.StringField(blank=False)
oai_set = fields.StringField(blank=True, default=None)
from_date = fields.DateTimeField(blank=True, default=None)
until_date = fields.DateTimeField(blank=True, default=None)
expiration_date = fields.DateTimeField(blank=False, default=None)
page_number = fields.IntField(blank=False)
@staticmethod
def get_by_resumption_token(resumption_token):
try:
return OaiRequestPage.objects.get(
resumption_token=resumption_token)
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
class CustomResource(Document):
"""
Custom resource model
"""
template = fields.ReferenceField(Template, blank=False)
name_in_schema = fields.StringField(blank=True)
title = fields.StringField(unique_with="template")
slug = fields.StringField()
description = fields.StringField(blank=True)
type = fields.StringField(
blank=False,
choices=(CUSTOM_RESOURCE_TYPE.RESOURCE.value,
CUSTOM_RESOURCE_TYPE.ALL.value),
)
icon = fields.StringField(blank=False)
icon_color = fields.StringField(blank=True)
display_icon = fields.BooleanField(blank=True)
role_choice = fields.StringField(blank=True)
role_type = fields.StringField(blank=True)
sort = fields.IntField(unique_with="template", min_value=0)
refinements = fields.ListField(fields.StringField(), blank=True)
def __str__(self):
return f'CustomResource: name_in_schema: {self.name_in_schema} \n' + \
f' title: {self.title}\n' + \
f' slug: {self.slug}\n' + \
f' description: {self.description}\n' + \
f' type: {self.type}\n' + \
f' role_choice: {self.role_choice}\n' + \
f' role_type: {self.role_type}'
def save(self, *args, **kwargs):
if not self.slug:
self.slug = slugify(self.title)
return super(CustomResource, self).save(*args, **kwargs)
@staticmethod
def get_all_by_template(template):
"""Get all custom resource by template.
Args:
template:
Returns:
"""
return CustomResource.objects(template=template).all()
@staticmethod
def get_custom_resource_by_template_and_type(template, type):
"""Get custom resource by template and type.
Args:
template:
type:
Returns:
"""
return CustomResource.objects(template=template, type=type).all()
@staticmethod
def get_custom_resource_by_template_and_slug(template, slug):
"""Get custom resource by template and slug.
Args:
template:
slug:
Returns:
"""
try:
return CustomResource.objects.get(template=template, slug=slug)
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
@staticmethod
def get_by_role_for_current_template(template, role):
"""Get custom resource by role for current template.
Args:
template:
role:
Returns:
"""
try:
list_custom_resources = CustomResource.objects(
template=template, role_choice=role).all()
if len(list_custom_resources) > 0:
return list_custom_resources[0]
else:
raise exceptions.ModelError(
"Can't find the custom resource with the given role: " +
role)
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
@staticmethod
def delete_custom_resources_by_template(template):
"""Delete all custom resources related to a template.
Args:
template:
Returns:
"""
CustomResource.objects(template=template).delete()
class OaiRegistry(Document):
""" A registry object for Oai-Pmh Harvester"""
name = fields.StringField()
url = fields.URLField(unique=True)
harvest_rate = fields.IntField(blank=True)
description = fields.StringField(blank=True)
harvest = fields.BooleanField(default=False)
last_update = fields.DateTimeField(blank=True)
is_harvesting = fields.BooleanField(default=False)
is_updating = fields.BooleanField(default=False)
is_activated = fields.BooleanField(default=True)
is_queued = fields.BooleanField(default=False)
@staticmethod
def get_by_id(oai_registry_id):
""" Get an OaiRegistry by its id
Args:
oai_registry_id: OaiRegistry id.
Returns: The OaiRegistry instance.
Raises:
DoesNotExist: The registry doesn't exist
ModelError: Internal error during the process
"""
try:
return OaiRegistry.objects().get(pk=str(oai_registry_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as e:
raise exceptions.ModelError(str(e))
@staticmethod
def get_by_name(oai_registry_name):
""" Get an OaiRegistry by its name.
Args:
oai_registry_name: OaiRegistry name.
Returns: The OaiRegistry instance.
Raises:
DoesNotExist: The registry doesn't exist
ModelError: Internal error during the process
"""
try:
return OaiRegistry.objects().get(name=oai_registry_name)
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as e:
raise exceptions.ModelError(str(e))
@staticmethod
def get_all():
""" Return all OaiRegistry
Returns:
List of OaiRegistry
"""
return OaiRegistry.objects().all()
@staticmethod
def get_all_by_is_activated(is_activated, order_by_field=None):
""" Return all OaiRegistry by their is_activated field
Params:
is_activated: True or False.
order_by_field: Field to order on.
Returns:
List of OaiRegistry
"""
return OaiRegistry.objects(
is_activated=is_activated).order_by(order_by_field)
@staticmethod
def check_registry_url_already_exists(oai_registry_url):
""" Check if an OaiRegistry with the given url already exists.
Params:
oai_registry_url: URL to check.
Returns:
Yes or No (bool).
"""
return OaiRegistry.objects(url__exact=oai_registry_url).count() > 0
class TempUserStep(Document):
"""
Model which define the step associated to a user choice
"""
step = fields.ReferenceField(DynQueryStep, blank=True) # Base step
# Attribute for query_management
query_able = fields.BooleanField() # True if the query is query able, false else
# List of choices and files
list_choices_id_file = fields.ListField(fields.ReferenceField(TempChoiceListFile, blank=True), blank=True)
dict_choices_id_file = dict() # Dict of choices and files id
files = fields.ListField(blank=True) # List of file for date step
choices = fields.ListField(blank=True) # User choices for the step
form_type = fields.StringField() # Form step type (radio, check, date)
absolute_position = fields.IntField() # Position in the list of steps for the query
viewable_position = fields.IntField() # Position in the list of query able steps for the query
full_xpath = fields.StringField() # Full path to the query
@staticmethod
def get_all():
"""Return all TempUserStep.
Returns:
"""
return TempUserStep.objects().all()
def delete_database(self):
"""
Delete function
Delete each choices and the associated files.
"""
for choices_id_file in self.list_choices_id_file:
try:
choices_id_file_obj = TempChoiceListFile.objects.get(id=choices_id_file.id)
choices_id_file_obj.delete_database()
except:
pass
self.delete()
def initialize_step(self, step=None):
"""
Initialise the query step.
It initialize a step. If the step name is given, the information are loaded from the
step define by the admin into step.
:param step: The step to load.
:return: The object initialized.
"""
# Attribute for query_management
self.list_choices_id_file = list()
self.dict_choices_id_file = dict()
self.files = list() # List of file for date step
self.choices = list() # User choice for the step
self.form_type = ''
self.absolute_position = None
self.viewable_position = None
self.full_xpath = None
self.query_able = False
if step is not None:
self.load_from_step_admin(step=step)
def load_from_step_admin(self, step):
"""
Load the step from the admin defined step.
The information are loaded from the step define by the admin into the user's step.
It is mainly used by the __init__.
:param step: The step to load.
"""
self.step = step
self.query_able = not step.output_field
if self.step.data_type == 'data':
if self.step.data_min_range == 1 and self.step.data_max_range == 1:
self.form_type = 'radio'
else:
self.form_type = 'check'
else:
self.form_type = 'date'
def load_choices_and_xml_data(self, cursor_xml_data):
"""
Load the dictionary of choices and files from the list of input files
:param cursor_xml_data: List of input files
"""
dict_choices_file = defaultdict(set)
for file_xml_data in cursor_xml_data:
data_dict = file_xml_data.to_mongo()
self.get_choices(
[data_dict['dict_content']],
[x for x in self.full_xpath.split(".") if x],
dict_choices_file,
data_dict['_id']
)
for choices, set_files in dict_choices_file.iteritems():
self.dict_choices_id_file[choices] = list(set_files)
if len(self.dict_choices_id_file) == 0:
raise EmptyChoicesFromQuery(x_path=self.full_xpath)
def get_id_files_from_user_choices(self):
"""
Get the list of unique files from the user choices and the list of choices and files.
:return: List of unique files
"""
unique_id_files = set()
for temp_choice_list_file in self.list_choices_id_file:
if temp_choice_list_file.choice in self.choices:
for bucket in temp_choice_list_file.list_buckets:
for id_file in bucket.list_files:
unique_id_files.add(ObjectId(id_file))
return list(unique_id_files)
def get_choices_as_tuple(self):
"""
Get list of choices.
Get a list of possible choices for the user as tuple as
[(choice1, choice1), (choice2, choice2), ...].
:return: List of possible choices for the user as tuple or None if there is no choices.
"""
if len(self.dict_choices_id_file) == 0:
return None
else:
choices = set()
for keys in self.dict_choices_id_file:
choices.add((keys, keys))
return sorted(tuple(choices))
def update_choices_to_db(self):
"""
Update the user choices for the step.
"""
self.update(choices=self.choices)
def update_choices_id_files_to_db(self):
"""
Update the choices and the corresponding files to the db.
Update the choices and the corresponding files to the db. Change also the way of data
structure, from a dictionary based structure to a list based structure more suitable
for database restriction.
"""
choices_id_files_to_save = self.dict_choices_id_file
# Delete old data in the DB
for choice in self.list_choices_id_file:
for bucket in choice.list_buckets:
for files in bucket.list_files:
files.delete()
bucket.delete()
choice.delete()
# Transform and update the data
new_list_choices_list_files = list()
for choice, list_value in choices_id_files_to_save.items():
new_temp_choice_list_file = TempChoiceListFile()
new_temp_choice_list_file.choice = choice
list_bucket_temp = list()
chunks = [list_value[x:x + 300] for x in xrange(0, len(list_value), 300)]
for chunk in chunks:
new_bucket = TempBucketIdFiles()
new_bucket.list_files = chunk
new_bucket.save()
list_bucket_temp.append(new_bucket)
new_temp_choice_list_file.list_buckets = list_bucket_temp
new_temp_choice_list_file.save()
new_list_choices_list_files.append(new_temp_choice_list_file)
self.update(list_choices_id_file=new_list_choices_list_files)
def transform_choices_files_list_to_dict(self):
"""
Transform and return the possible choices-lists from db-list to use-dict
Transform the choices-files list structure into a DB to a usable choices-files dict
structure. It also delete old trace of existing choices-files dict structure.
"""
self.dict_choices_id_file = dict()
for list_choices_id_file in self.list_choices_id_file:
self.dict_choices_id_file[list_choices_id_file.choice] = list()
for buckets in list_choices_id_file.list_buckets:
self.dict_choices_id_file[list_choices_id_file.choice].extend(buckets.list_files)
def update_files_in_db(self):
"""
Update the list of files into the db
"""
self.update(files=self.files)
def clean_step(self):
"""
Clean a specific step into the database.
"""
self.update(choices=list())
self.update(files=list())
for list_choices_id_file in self.list_choices_id_file:
for bucket in list_choices_id_file.list_buckets:
bucket.delete()
list_choices_id_file.delete()
self.update(list_choices_id_file=list())
def filter_user_choice_date(self, element, time_from, time_to, translate_dt):
"""
Filter a specif element against it date value
:param element: element too filter
:param time_from: First threshold to filter
:param time_to: Second threshold to filter
:param translate_dt: Python translate for the element value
:return: Return true is the element date is after the time_from and before the time_to
"""
timestamp = str(element["@" + self.step.value])
if timestamp[-1] == 'Z':
timestamp = timestamp[:-1]
timestamp = map(int, timestamp.translate(translate_dt).split())
if (is_datetime_inf_or_equal(time_from, timestamp) and
is_datetime_inf_or_equal(timestamp, time_to)) is False:
return False
else:
return True
def choose_filter_choice(self, list_elements, element_title, translate_dt, translate_bounds):
"""
Choose a specific filter and apply it to a list of elements.
:param list_elements: List of elements to filter
:param element_title: Title of the elements
:param translate_dt: Translate object for date comparison
:param translate_bounds: Translate object for date comparison
:return: Filtered list of elements
"""
if self.query_able:
if self.step.data_type == 'data':
target = self.step.target
choices = self.choices
if target == 'attribute':
value = "@" + self.step.value
list_elements_cleaned = [x for x in list_elements if x[value] in choices]
return list_elements_cleaned
elif target == 'name':
if element_title not in choices:
list_elements_cleaned = []
else:
list_elements_cleaned = list_elements
return list_elements_cleaned
else:
# FIXME: #text was hardcoded, but elements don't have #text if the xml tag does not have any
# FIXME: attributes
list_elements_cleaned = [x for x in list_elements if isinstance(x, dict) and x['#text'] in choices
or x in choices]
return list_elements_cleaned
else:
time_from = map(int, str(self.choices[0]).
translate(translate_bounds).split())
time_to = map(int, str(self.choices[1]).
translate(translate_bounds).split())
my_filter_date = self.filter_user_choice_date
list_elements_cleaned = [
x
for x in list_elements
if my_filter_date(element=x, time_from=time_from, time_to=time_to,
translate_dt=translate_dt)
]
return list_elements_cleaned
else:
return list_elements
def get_choices(self, list_elements, list_keys_xpath, dict_choices_file, id_file):
"""
Get possibles choices from a list of elements to explore.
:param list_elements: List of elements to explore
:param list_keys_xpath: List of elements from the xpath spliced
:param dict_choices_file: Dictionary of choices, linked to a list of files
:param id_file:Current if file
:return:
"""
element_title = list_keys_xpath[-1]
while list_keys_xpath:
key_xpath = list_keys_xpath.pop(0)
if list_elements:
if isinstance(list_elements[0], list):
list_elements = flat_list(list_elements)
if key_xpath == "*":
set_possibilities = explore_star(list_elements)
for key_possible in set_possibilities:
list_key_param_star = [key_possible]
if len(list_keys_xpath) != 0:
list_key_param_star.extend(list_keys_xpath)
self.get_choices(
list_elements=list_elements,
list_keys_xpath=list_key_param_star,
dict_choices_file=dict_choices_file,
id_file=id_file
)
return
else:
list_elements = [x.get(key_xpath) for x in list_elements if x.get(key_xpath)]
target = self.step.target
if target == 'name':
dict_choices_file[element_title].add(id_file)
else:
if list_elements:
if isinstance(list_elements[0], list):
list_elements = flat_list(list_elements)
if target == 'attribute':
value = "@" + self.step.value
set_elements = (str(x.get(value)) for x in list_elements)
for element in set_elements:
dict_choices_file[element].add(id_file)
else:
# FIXME: #text was hardcoded, but elements don't have #text if the xml tag does not have any attributes
# set_elements = (x.get("#text") for x in list_elements)
set_elements = (x.get("#text") if isinstance(x, dict) else str(x) for x in list_elements)
for element in set_elements:
dict_choices_file[element].add(id_file)
return
class MongoDoc(Document):
intfield = fields.IntField()
class SwachhToilet(Document):
qci_id = fields.StringField(max_length=100, unique=True)
toilet_id = fields.StringField(max_length=100, unique=True)
address = fields.StringField()
latitude = fields.FloatField()
longitude = fields.FloatField()
location = fields.PointField()
state = fields.StringField(blank=True)
city = fields.StringField()
category = fields.StringField(blank=True)
category_code = fields.StringField(blank=True)
type = fields.StringField(blank=True)
opening_time = fields.StringField(blank=True)
closing_time = fields.StringField(blank=True)
open_days = fields.StringField(default='All Seven Days')
seats = fields.IntField(default=0)
gender = fields.StringField(default='Gents and Ladies')
child_friendly = fields.BooleanField(default=False)
differntly_abled_friedly = fields.BooleanField(default=False)
fee_type = fields.StringField(default='Free of Charge')
cost = fields.StringField(blank=True)
image = fields.StringField(blank=True)
timestamp = fields.DateTimeField(default=datetime.now)
comments = fields.ListField(fields.DictField(), blank=True, default=None)
meta = {'collection': 'swachh_toilets'}
def __str__(self):
return self.qci_id
def title(self):
return 'Swachh Public Toilet'
def get_api_url(self, request=None):
return api_reverse("api-public-toilets:detail-toilet",
kwargs={'pk': str(self.pk)},
request=request)
def get_comments_api_url(self, request=None):
return self.get_api_url(request) + '/comments'
def opened_today(self):
days = week_days_in_short_format()
open_days_from_db = self.open_days
if open_days_from_db == 'Monday to Friday':
open_days = days[:5]
if open_days_from_db == 'Monday to Saturday':
open_days = days[:6]
elif open_days_from_db == 'Tuesday,Wednesday,Thursday,\
Friday,Saturday,Sunday':
open_days = days[1:7]
else:
open_days = days
if today_name_in_short_form() in open_days:
return True
def opened_all_the_day(self):
return (self.opening_time == '00:00' and self.closing_time == '23:59')
class Template(dme_Document):
"""Represents an XML schema template that defines the structure of data for curation"""
title = dme_fields.StringField()
filename = dme_fields.StringField()
content = dme_fields.StringField()
templateVersion = dme_fields.StringField(blank=True)
version = dme_fields.IntField(blank=True)
hash = dme_fields.StringField()
user = dme_fields.StringField(blank=True)
dependencies = dme_fields.ListField(StringField(), blank=True)
exporters = dme_fields.ListField(ReferenceField(Exporter,
reverse_delete_rule=PULL),
blank=True)
XSLTFiles = dme_fields.ListField(ReferenceField(ExporterXslt,
reverse_delete_rule=PULL),
blank=True)
ResultXsltList = dme_fields.ReferenceField(ResultXslt,
reverse_delete_rule=NULLIFY,
blank=True)
ResultXsltDetailed = dme_fields.ReferenceField(ResultXslt,
reverse_delete_rule=NULLIFY,
blank=True)
def __str__(self):
return self.title
@staticmethod
#@app.route('/my_dashboard_my_records')
def getcursor(p1=None, p2=None):
# create a connection
#client = MongoClient(MONGODB_URI, document_class=OrderedDict)
# connect to the db 'mgi'
client = MongoClient(MONGODB_URI)
db = client[MGI_DB]
# get the xmldata collection
template = db['template']
# find the associated object
if (p2 is not None):
cursor = template.find({p1: p2})
else:
cursor = (template.find()) #.get('schema')
#c = cursore.get('schema')
return cursor
#@staticmethod
@register.filter
def get_templ_object_from_Id(p1):
"""
Returns the title of the template used to enter the file associated to the given id
"""
# create a connection to MongoDB
client = MongoClient(MONGODB_URI)
# connect to the db 'mgi'
db = client[MGI_DB]
# get the xmldata collection
template = db['template']
# find the associated object
template = Template.objects(pk=p1).get()
return template.title
@register.filter
def get_id_doc(postID):
"""
Returns the object with the given id
"""
# create a connection
client = MongoClient(MONGODB_URI, document_class=OrderedDict)
# connect to the db 'mgi'
db = client[MGI_DB]
# get the xmldata collection
template = db['template']
return template.find_one({'_id': ObjectId(postID)})
@staticmethod
def find(param1=None, param2=None):
# create a connection
client = MongoClient(MONGODB_URI)
# connect to the db 'mgi'
db = client[MGI_DB]
# get all templates of the db
template = db['template']
# find the templates matching the parameters
if (param2 is not None):
cursor = template.find({param1: param2})
else:
cursor = (template.find()) #.get('schema')
return cursor
