class Tmall(DynamicDocument):
title = fields.StringField()
store_id = fields.StringField()
url = fields.URLField()
price = fields.FloatField()
sale_num = fields.IntField()
image_urls = fields.ListField()
scrapy_mongodb = fields.DictField(db_field='scrapy-mongodb')
def __str__(self):
return self.title
@property
def cover(self):
return self.image_urls[0]
class OaiIdentify(Document):
"""Represents an identify object for Oai-Pmh Harvester"""
admin_email = fields.StringField(blank=True)
base_url = fields.URLField(unique=True)
repository_name = fields.StringField(blank=True)
deleted_record = fields.StringField(blank=True)
delimiter = fields.StringField(blank=True)
description = fields.StringField(blank=True)
earliest_datestamp = fields.StringField(blank=True)
granularity = fields.StringField(blank=True)
oai_identifier = fields.StringField(blank=True)
protocol_version = fields.StringField(blank=True)
repository_identifier = fields.StringField(blank=True)
sample_identifier = fields.StringField(blank=True)
scheme = fields.StringField(blank=True)
raw = fields.DictField(blank=True)
registry = fields.ReferenceField(OaiRegistry,
reverse_delete_rule=CASCADE,
unique=True)
@staticmethod
def get_by_registry_id(registry_id):
"""Return an OaiIdentify by its registry id.
Args:
registry_id: The registry id.
Returns:
OaiIdentify instance.
Raises:
DoesNotExist: The OaiIdentify doesn't exist.
ModelError: Internal error during the process.
"""
try:
return OaiIdentify.objects().get(registry=str(registry_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as e:
raise exceptions.ModelError(str(e))
class BuildingBaseShanghai(Document):
RecordID = fields.UUIDField(default=uuid.uuid1(),
binary=True,
primary_key=True,
null=False)
CurTimeStamp = fields.StringField(default=str(datetime.datetime.now()),
index=True)
project_no = fields.StringField(default='', max_length=255, null=False)
project_name = fields.StringField(default='', max_length=255, null=False)
opening_unit_no = fields.StringField(default='',
max_length=255,
null=False)
building_name = fields.StringField(default='', max_length=255, null=False)
building_no = fields.StringField(default='', max_length=255, null=False)
building_price = fields.StringField(default='', max_length=255, null=False)
building_fluctuation = fields.StringField(default='',
max_length=255,
null=False)
building_num = fields.StringField(default='', max_length=255, null=False)
building_area = fields.StringField(default='', max_length=255, null=False)
building_sts = fields.StringField(default='', max_length=255, null=False)
change_data = fields.StringField(default='', max_length=1023, null=False)
building_url = fields.URLField(default=None, null=True, blank=True)
NewCurTimeStamp = fields.StringField(default=str(datetime.datetime.now()),
index=True)
meta = {
'indexes': [
'CurTimeStamp',
'NewCurTimeStamp',
'change_data',
'project_no',
'building_sts',
'opening_unit_no',
'building_no',
]
}
class OaiRegistry(Document):
""" A registry object for Oai-Pmh Harvester"""
name = fields.StringField()
url = fields.URLField(unique=True)
harvest_rate = fields.IntField(blank=True)
description = fields.StringField(blank=True)
harvest = fields.BooleanField(default=False)
last_update = fields.DateTimeField(blank=True)
is_harvesting = fields.BooleanField(default=False)
is_updating = fields.BooleanField(default=False)
is_activated = fields.BooleanField(default=True)
is_queued = fields.BooleanField(default=False)
@staticmethod
def get_by_id(oai_registry_id):
""" Get an OaiRegistry by its id
Args:
oai_registry_id: OaiRegistry id.
Returns: The OaiRegistry instance.
Raises:
DoesNotExist: The registry doesn't exist
ModelError: Internal error during the process
"""
try:
return OaiRegistry.objects().get(pk=str(oai_registry_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as e:
raise exceptions.ModelError(str(e))
@staticmethod
def get_by_name(oai_registry_name):
""" Get an OaiRegistry by its name.
Args:
oai_registry_name: OaiRegistry name.
Returns: The OaiRegistry instance.
Raises:
DoesNotExist: The registry doesn't exist
ModelError: Internal error during the process
"""
try:
return OaiRegistry.objects().get(name=oai_registry_name)
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as e:
raise exceptions.ModelError(str(e))
@staticmethod
def get_all():
""" Return all OaiRegistry
Returns:
List of OaiRegistry
"""
return OaiRegistry.objects().all()
@staticmethod
def get_all_by_is_activated(is_activated, order_by_field=None):
""" Return all OaiRegistry by their is_activated field
Params:
is_activated: True or False.
order_by_field: Field to order on.
Returns:
List of OaiRegistry
"""
return OaiRegistry.objects(
is_activated=is_activated).order_by(order_by_field)
@staticmethod
def check_registry_url_already_exists(oai_registry_url):
""" Check if an OaiRegistry with the given url already exists.
Params:
oai_registry_url: URL to check.
Returns:
Yes or No (bool).
"""
return OaiRegistry.objects(url__exact=oai_registry_url).count() > 0
class Instance(Document):
""" Represents an instance of a remote project
"""
name = fields.StringField(blank=False,
unique=True,
regex=NOT_EMPTY_OR_WHITESPACES)
endpoint = fields.URLField(blank=False, unique=True)
access_token = fields.StringField(blank=False)
refresh_token = fields.StringField(blank=False)
expires = fields.DateTimeField(blank=False)
@staticmethod
def get_all():
""" Return all instances.
Returns:
instance collection
"""
return Instance.objects().all()
@staticmethod
def get_by_id(instance_id):
""" Return the object with the given id.
Args:
instance_id:
Returns:
Instance (obj): Instance object with the given id
"""
try:
return Instance.objects.get(pk=str(instance_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(e.message)
except Exception as ex:
raise exceptions.ModelError(ex.message)
@staticmethod
def get_by_name(instance_name):
""" Return the object with the given name.
Args:
instance_name:
Returns:
Instance (obj): Instance object with the given name
"""
try:
return Instance.objects.get(name=str(instance_name))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(e.message)
except Exception as ex:
raise exceptions.ModelError(ex.message)
def save_object(self):
""" Custom save.
Returns:
"""
try:
self.check_instance_name()
return self.save()
except mongoengine_errors.NotUniqueError as e:
raise exceptions.NotUniqueError(
"Unable to create the new repository: Not Unique")
except Exception as ex:
raise exceptions.ModelError(ex.message)
def check_instance_name(self):
""" Test if the name is 'Local'.
Returns:
"""
if self.name.upper() == "LOCAL":
raise exceptions.ModelError(
"By default, the instance named Local is the instance currently running."
)
def clean(self):
""" Clean is called before saving
Returns:
"""
self.name = self.name.strip()
class Instance(Document):
"""Represents an instance of a remote project"""
name = fields.StringField(
blank=False, unique=True, validation=not_empty_or_whitespaces
)
endpoint = fields.URLField(blank=False, unique=True)
access_token = fields.StringField(blank=False)
refresh_token = fields.StringField(blank=False)
expires = fields.DateTimeField(blank=False)
@staticmethod
def get_all():
"""Return all instances.
Returns:
instance collection
"""
return Instance.objects().all()
@staticmethod
def get_by_id(instance_id):
"""Return the object with the given id.
Args:
instance_id:
Returns:
Instance (obj): Instance object with the given id
"""
try:
return Instance.objects.get(pk=str(instance_id))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
@staticmethod
def get_by_name(instance_name):
"""Return the object with the given name.
Args:
instance_name:
Returns:
Instance (obj): Instance object with the given name
"""
try:
return Instance.objects.get(name=str(instance_name))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
@staticmethod
def get_by_endpoint_starting_with(instance_endpoint):
"""Return the object with the given endpoint.
Args:
instance_endpoint:
Returns:
Instance (obj): Instance object with the given name
"""
try:
return Instance.objects.get(endpoint__startswith=str(instance_endpoint))
except mongoengine_errors.DoesNotExist as e:
raise exceptions.DoesNotExist(str(e))
except Exception as ex:
raise exceptions.ModelError(str(ex))
def save_object(self):
"""Custom save.
Returns:
"""
try:
self.check_instance_name()
return self.save()
except mongoengine_errors.NotUniqueError as e:
raise exceptions.NotUniqueError(
"Unable to create the new repository: Not Unique"
)
except Exception as ex:
raise exceptions.ModelError(str(ex))
def check_instance_name(self):
"""Test if the name is the name of the local instance.
Returns:
"""
if self.name.upper() == settings.CUSTOM_NAME.upper():
raise exceptions.ModelError(
f'By default, the instance named "{settings.CUSTOM_NAME}" is the instance currently running.'
)
def clean(self):
"""Clean is called before saving
Returns:
"""
self.name = self.name.strip()
class User(document.Document):
SEXO_HOMBRE = 'masculino'
SEXO_MUJER = 'femenino'
SEXOS = ((None, 'No definido'), (SEXO_HOMBRE, 'Masculino'), (SEXO_MUJER,
'Femenino'))
"""A User document that aims to mirror most of the API specified by Django
at http://docs.djangoproject.com/en/dev/topics/auth/#users
"""
username = fields.StringField(
max_length=250,
verbose_name=('username'),
help_text=
("Required. 250 characters or fewer. Letters, numbers and @/./+/-/_ characters"
),
required=False)
first_name = fields.StringField(
max_length=250,
blank=True,
verbose_name=('first name'),
)
last_name = fields.StringField(max_length=250,
blank=True,
verbose_name=('last name'))
email = fields.EmailField(verbose_name=('e-mail address'), blank=False)
password = fields.StringField(
blank=True,
max_length=128,
verbose_name=('password'),
help_text=
("Use '[algo]$[iterations]$[salt]$[hexdigest]' or use the <a href=\"password/\">change password form</a>."
))
is_staff = fields.BooleanField(
default=False,
verbose_name=('staff status'),
help_text=(
"Designates whether the user can log into this admin site."))
is_active = fields.BooleanField(
default=True,
verbose_name=('active'),
help_text=
("Designates whether this user should be treated as active. Unselect this instead of deleting accounts."
))
is_superuser = fields.BooleanField(
default=False,
verbose_name=('superuser status'),
help_text=
("Designates that this user has all permissions without explicitly assigning them."
))
last_login = fields.DateTimeField(default=timezone.now,
verbose_name=('last login'))
date_joined = fields.DateTimeField(default=timezone.now,
verbose_name=('date joined'))
user_permissions = fields.ListField(
fields.ReferenceField(Permission),
verbose_name=('user permissions'),
blank=True,
help_text=('Permissions for the user.'))
birthdate = DateField(blank=True)
# image = LocalStorageFileField(upload_to='users/')
web_url = fields.URLField(blank=True)
facebook_page = fields.URLField(blank=True)
youtube_channel = fields.URLField(blank=True)
genere = fields.StringField(choices=SEXOS, required=False, blank=True)
is_complete = fields.BooleanField(default=False)
providers = fields.ListField(fields.EmbeddedDocumentField('Provider'),
blank=True)
photo_url = fields.URLField(blank=True)
uid = fields.StringField(blank=False, required=True)
display_name = fields.StringField(blank=True)
USERNAME_FIELD = 'username'
REQUIRED_FIELDS = ['email']
meta = {
'allow_inheritance': True,
'indexes': [{
'fields': ['username'],
'unique': True,
'sparse': True
}]
}
def __unicode__(self):
return self.username
def get_full_name(self):
"""Returns the users first and last names, separated by a space.
"""
full_name = u'%s %s' % (self.first_name or '', self.last_name or '')
return full_name.strip()
def is_anonymous(self):
return False
def is_authenticated(self):
return True
def set_password(self, raw_password):
"""Sets the user's password - always use this rather than directly
assigning to :attr:`~mongoengine.django.auth.User.password` as the
password is hashed before storage.
"""
self.password = make_password(raw_password)
self.save()
return self
def check_password(self, raw_password):
"""Checks the user's password against a provided password - always use
this rather than directly comparing to
:attr:`~mongoengine.django.auth.User.password` as the password is
hashed before storage.
"""
return check_password(raw_password, self.password)
@classmethod
def create_user(cls, username, password, email=None):
"""Create (and save) a new user with the given username, password and
email address.
"""
now = timezone.now()
# Normalize the address by lowercasing the domain part of the email
# address.
if email is not None:
try:
email_name, domain_part = email.strip().split('@', 1)
except ValueError:
pass
else:
email = '@'.join([email_name, domain_part.lower()])
user = cls(username=username, email=email, date_joined=now)
user.set_password(password)
user.save()
return user
def get_group_permissions(self, obj=None):
"""
Returns a list of permission strings that this user has through his/her
groups. This method queries all available auth backends. If an object
is passed in, only permissions matching this object are returned.
"""
permissions = set()
for backend in auth.get_backends():
if hasattr(backend, "get_group_permissions"):
permissions.update(backend.get_group_permissions(self, obj))
return permissions
def get_all_permissions(self, obj=None):
return _user_get_all_permissions(self, obj)
def has_perm(self, perm, obj=None):
"""
Returns True if the user has the specified permission. This method
queries all available auth backends, but returns immediately if any
backend returns True. Thus, a user who has permission from a single
auth backend is assumed to have permission in general. If an object is
provided, permissions for this specific object are checked.
"""
# Active superusers have all permissions.
if self.is_active and self.is_superuser:
return True
# Otherwise we need to check the backends.
return _user_has_perm(self, perm, obj)
def has_module_perms(self, app_label):
"""
Returns True if the user has any permissions in the given app label.
Uses pretty much the same logic as has_perm, above.
"""
# Active superusers have all permissions.
if self.is_active and self.is_superuser:
return True
return _user_has_module_perms(self, app_label)
def email_user(self, subject, message, from_email=None):
"Sends an e-mail to this User."
from django.core.mail import send_mail
send_mail(subject, message, from_email, [self.email])
class Provider(document.EmbeddedDocument):
uid = fields.StringField()
display_name = fields.StringField(blank=True)
photo_url = fields.URLField(blank=True)
email = fields.EmailField(blank=False)
provider_id = fields.StringField(blank=False)