class Structures(Document):
name = StringField(required=True, help_text="name")
lattice = DictField(required=True, help_text="lattice")
sites = ListField(DictField(), required=True, help_text="sites")
charge = FloatField(null=True, help_text="charge")
md5 = StringField(regex=r"^[a-z0-9]{32}$",
unique=True,
help_text="md5 sum")
cif = StringField(help_text="CIF string")
meta = {"collection": "structures", "indexes": ["name", "md5"]}
@queryset_manager
def objects(doc_cls, queryset):
return queryset.only("name", "md5")
@classmethod
def pre_save_post_validation(cls, sender, document, **kwargs):
from mpcontribs.api.structures.views import StructuresResource
resource = StructuresResource()
d = resource.serialize(document, fields=["lattice", "sites", "charge"])
s = json.dumps(d, sort_keys=True).encode("utf-8")
document.md5 = md5(s).hexdigest()
structure = Structure.from_dict(d)
try:
writer = CifWriter(structure, symprec=1e-10)
except TypeError:
# save CIF string without symmetry information
writer = CifWriter(structure)
document.cif = writer.__str__()
class NotificationRequest(Document):
topic = StringField(default='marketing')
type = StringField(default='data')
content = DictField(default={}, required=False)
message = StringField(required=False)
# age_group, sex, location
selector = DictField(required=False)
class Pet(Document):
id = IntField(primary_key=True)
category = DictField()
name = StringField()
photoUrls = ListField(StringField())
tags = ListField(DictField())
status = StringField()
class Structures(DynamicDocument):
contribution = LazyReferenceField(
Contributions,
passthrough=True,
reverse_delete_rule=CASCADE,
required=True,
help_text="contribution this structure belongs to",
)
is_public = BooleanField(
required=True, default=False, help_text="public/private structure"
)
name = StringField(required=True, help_text="structure name")
label = StringField(required=True, help_text="structure label")
lattice = DictField(required=True, help_text="lattice")
sites = ListField(DictField(), required=True, help_text="sites")
charge = FloatField(null=True, help_text="charge")
klass = StringField(help_text="@class")
module = StringField(help_text="@module")
meta = {
"collection": "structures",
"indexes": ["contribution", "is_public", "name", "label"],
}
@classmethod
def post_save(cls, sender, document, **kwargs):
set_root_keys = set(k.split(".", 1)[0] for k in document._delta()[0].keys())
cid = document.contribution.id
nbs = Notebooks.objects(pk=cid)
if not set_root_keys or set_root_keys == {"is_public"}:
nbs.update(set__is_public=document.is_public)
else:
nbs.delete()
document.update(unset__cif=True)
Contributions.objects(pk=cid).update(unset__structures=True)
class Structures(Document):
contribution = LazyReferenceField(
Contributions,
passthrough=True,
reverse_delete_rule=CASCADE,
required=True,
help_text="contribution this structure belongs to",
)
is_public = BooleanField(
required=True, default=False, help_text="public/private structure"
)
name = StringField(required=True, help_text="structure name")
label = StringField(required=True, help_text="structure label")
lattice = DictField(required=True, help_text="lattice")
sites = ListField(DictField(), required=True, help_text="sites")
charge = FloatField(null=True, help_text="charge")
klass = StringField(help_text="@class")
module = StringField(help_text="@module")
meta = {
"collection": "structures",
"indexes": ["contribution", "is_public", "label"],
}
@classmethod
def post_save(cls, sender, document, **kwargs):
Notebooks.objects(pk=document.contribution.id).delete()
class Metadata(DictField):
kernelspec = DictField(
Kernelspec(), required=True, help_text="kernelspec", default=Kernelspec
)
language_info = DictField(
LanguageInfo(), required=True, help_text="language info", default=LanguageInfo
)
class Task(Document):
qasm = StringField(required=True)
shots = IntField(min_value=0, default=8192)
qjob_id = StringField(null=True)
status = StringField(default="created")
config = DictField(default={})
result = DictField(default={})
@property
def id_str(self):
return str(self.id)
def to_dict(self):
d = {
"id": self.id_str,
"qasm": self.qasm,
"status": self.status,
"shots": self.shots,
"config": self.config
}
if self.result:
d["result"] = self.result
return d
def update_results(self, job: QuantumExecutionJob):
self.result = job.result_prob
def create_qjob(self):
qjob = QuantumExecutionJob(QuantumCircuit.from_qasm_str(self.qasm),
shots=self.shots,
config=self.config)
self.qjob_id = qjob.id
return qjob
class NodeConfig(Document):
version = IntField()
node_name = StringField(max_length=1000)
icon_link = URLField()
connection_method = DictField()
configures = EmbeddedDocumentListField(OperateFactor)
device_icons = DictField() # key:hid, value:URL
class FactorRegressionAnalysisResult(Document):
"""
因子回归分析结果
"""
# 因子名字
factor_name = StringField(required=True)
# 因子数据开始时间
begin_date = DateTimeField(required=True)
# 因子数据结束时间
end_date = DateTimeField(required=True)
# 因子收益率的自相关系数acf和偏自相关系数pacf,默认1-10阶,结果list len=11，取1-10个数
acf_result = DictField()
# 因子收益率，单利，复利, 日收益率
factor_return = DictField()
# 单因子检测的T值, Series, index为时间
factor_t_value = DictField()
# 单因子检测的T值的统计值，'t_value_mean': 绝对值均值, 't_value_greater_two':绝对值序列大于2的占比
factor_t_value_statistics = DictField()
# 净值分析结果
net_analysis_result = DictField()
meta = {
'indexes': ['factor_name', ('factor_name', 'begin_date', 'end_date')],
'shard_key': ('factor_name', )
}
class FactorIcAnalysisResult(Document):
"""
因子ic分析结果
"""
# 因子名字
factor_name = StringField(required=True)
# 因子数据开始时间
begin_date = DateTimeField(required=True)
# 因子数据结束时间
end_date = DateTimeField(required=True)
# IC信号衰减计算，index 是时间序列， columns是decay周期，[1, self.ic_decay], 闭区间
ic = DictField()
# p值信号衰减计算，index 是时间序列， columns是decay周期，[1, self.ic_decay], 闭区间
p_value = DictField()
# ic分析结果, index如下，column是decay周期，[1, self.ic_decay], 闭区间
# IC均值、 IC标准差、 IC_IR比率、 IC > 0 占比、 | IC | > 0.02 占比(绝对值)、 偏度、 峰度、
# 正相关显著比例、负相关显著比例、状态切换比例、同向比例
# ['ic_mean', 'ic_std', 'ic_ir', 'ic_ratio', 'ic_abs_ratio', 'ic_skewness', 'ic_kurtosis',
# 'ic_positive_ratio', 'ic_negative_ratio', 'ic_change_ratio', 'ic_unchange_ratio', ]
ic_result = DictField()
meta = {
'indexes': ['factor_name', ('factor_name', 'begin_date', 'end_date')],
'shard_key': ('factor_name', )
}
class Cell(DictField):
cell_type = StringField(required=True, default="code", help_text="cell type")
metadata = DictField(help_text="cell metadata")
source = StringField(required=True, default="print('hello')", help_text="source")
outputs = ListField(
DictField(), required=True, help_text="outputs", default=lambda: [DictField()]
)
execution_count = IntField(help_text="exec count")
class Game(EmbeddedDocument):
"""Games played in a round."""
game_type = StringField(max_length=16, required=True)
options = DictField(default=dict)
hole_data = DictField(default=dict)
leaderboard = DictField(default=dict)
scorecard = DictField(default=dict)
status = DictField(default=dict)
class DatasourceModule(EmbeddedDocument):
id = StringField(required=True)
primary = StringField(required=True)
matching = StringField(null=True)
fields = ListField(StringField())
labels = DictField()
types = DictField()
discrepencies = EmbeddedDocumentField(Discrepencies)
source_type = StringField(
choices=("datasource", "datalab"), required=True, default="datasource"
)
class User(Document):
userid = StringField(unique=True)
username = StringField(unique=True)
password = StringField()
fullname = StringField()
created = DateTimeField()
modified = DateTimeField(default=datetime.datetime.now)
phone = StringField(required=False)
homepage = StringField(required=False)
company = StringField(required=False)
organization = StringField(required=False)
tags = ListField(StringField(max_length=30))
modules = ListField(StringField(max_length=30))
upstream_credentials = DictField(required=False)
billing = DictField(required=False)
parent = StringField(required=False)
@classmethod
def assign_userid(cls, sender, document, **kwargs):
if sender is User and not document.userid:
document.userid = str(uuid.uuid4())
"""
reg. password hashing
https://github.com/passy/flask-bcrypt/blob/master/flaskext/bcrypt.py
https://github.com/maxcountryman/flask-bcrypt/blob/master/flask_bcrypt.py
https://exyr.org/2011/hashing-passwords/
https://github.com/SimonSapin/snippets/blob/master/hashing_passwords.py
https://github.com/mitsuhiko/python-pbkdf2/blob/master/pbkdf2.py
"""
@classmethod
def assign_created(cls, sender, document, **kwargs):
if sender is User and not document.created:
document.created = datetime.datetime.now()
@staticmethod
def crypt500(password):
return crypt(password, iterations=500)
@classmethod
def hash_password(cls, sender, document, **kwargs):
if sender is User and document.password and not document.password.startswith(
'$p5k2$'):
document.password = cls.crypt500(document.password)
def check_password(self, password):
pwhash = self.password
return pwhash == crypt(password, pwhash)
class Protein(Sequence):
meta = {'allow_inheritance': True, 'collection': "proteins",
'index_cls': False,
'indexes': [
{"fields": ["organism", "name"]},
{"fields": ["organism", "gene"]},
{"fields": ["seq_collection_id", "keywords"]},
{"fields": ["organism", "keywords"]},
{"fields": ["organism", "alias"]},
'keywords'
]}
organism = StringField()
gene = ListField(StringField(), required=True)
gene_id = ObjectIdField()
seq_collection_id = ReferenceField(SeqCollection)
reactions = ListField(EmbeddedDocumentField(Reaction), default=[])
reactome = ListField(DictField(), default=[])
druggability = FloatField()
tregulation = DictField(required=False)
alias = ListField(StringField(), default=[])
chembl = EmbeddedDocumentField(ChEMBL)
def __init__(self, **kwargs):
'''
'''
super(Sequence, self).__init__(**kwargs)
self._seq_init()
self.size.unit = "aa"
def add_domain(self, name, start, end, source="unknown"):
dn_feature = Feature(source=source, identifier=name, type=SO_TERMS["polypeptide_domain"],
location=Location(base=self.id, start=start, end=end))
self.features.append(dn_feature)
def domains(self):
return [feature for feature in self.features if feature.type == SO_TERMS["polypeptide_domain"]]
def domain_seqs(self):
return self.seqs_from_features(type=SO_TERMS["polypeptide_domain"])
def reaction(self, **kwargs):
filtered_reactions = [x for x in self.reactions if all([x[k] == v for k, v in kwargs.items()])]
if filtered_reactions:
return filtered_reactions[0]
return None
def __str__(self):
return self.name + " " + str(len(self.seq)) + "aa"
def homologous(self):
return [feature for feature in self.features if feature.type == SO_TERMS["homologous"]]
class Metadata(Document):
"""
Metadata stores information about objects in OmegaStore
"""
# fields
#: this is the name of the data
name = StringField(unique_with=['bucket', 'prefix'])
#: bucket
bucket = StringField()
#: prefix
prefix = StringField()
#: kind of data
kind = StringField(choices=MDREGISTRY.KINDS)
#: for PANDAS_HDF and SKLEARN_JOBLIB this is the gridfile
gridfile = FileField(
db_alias='omega',
collection_name=settings().OMEGA_MONGO_COLLECTION)
#: for PANDAS_DFROWS this is the collection
collection = StringField()
#: for PYTHON_DATA this is the actual document
objid = ObjectIdField()
#: omegaml technical attributes, e.g. column indicies
kind_meta = DictField()
#: customer-defined other meta attributes
attributes = DictField()
#: s3file attributes
s3file = DictField()
#: location URI
uri = StringField()
#: created datetime
created = DateTimeField(default=datetime.datetime.now)
# the actual db is defined in settings, OMEGA_MONGO_URL
meta = {
'db_alias': 'omega',
'indexes': [
# unique entry
{
'fields': ['bucket', 'prefix', 'name'],
},
'created', # most recent is last, i.e. [-1]
]
}
def __eq__(self, other):
return self.objid == other.objid
def __unicode__(self):
fields = ('name', 'bucket', 'prefix', 'created', 'kind')
kwargs = ('%s=%s' % (k, getattr(self, k))
for k in self._fields.keys() if k in fields)
return u"Metadata(%s)" % ','.join(kwargs)
class HostModel(DynamicDocument):
username = StringField()
levels = DictField()
blockchain = StringField()
registered_at = DateTimeField()
architect = StringField()
hoperator = StringField()
consensus_percent = FloatField()
referral_percent = FloatField()
dac_mode = IntField()
dacs = DictField()
chosts = DictField()
ahost = StringField()
non_active_chost = BooleanField()
need_switch = BooleanField()
fhosts_mode = IntField()
fhosts = DictField()
title = StringField()
purpose = StringField()
total_shares = IntField()
quote_amount = StringField()
root_token_contract = StringField()
root_token = StringField()
symbol = StringField()
precision = IntField()
to_pay = IntField()
payed = BooleanField()
cycle_start_id = IntField()
current_pool_id = IntField()
current_cycle_num = IntField()
current_pool_num = IntField()
parameters_setted = BooleanField()
activated = BooleanField()
priority_flag = BooleanField()
meta = StringField()
app = StringField()
meta = {
'collection':
'hosts',
'ordering': ['-registered_at'],
'indexes': [
'username', 'blockchain', 'ahost', 'activated', 'payed',
'parameters_setted'
'priority_flag'
'need_switch', 'non_active_chost', 'app'
],
'auto_create_index':
True,
'index_background':
True
}
class MediaAggregate(Document):
"""
Location specific amenities.
Every MediaAggregate could could contain one or more
Media sources
"""
# Type
typespec = ReferenceField('MediaAggregateType', required=False)
# Demographic properties
name = StringField()
display_name = StringField()
description = StringField()
survey_name = StringField()
address1 = StringField()
address2 = StringField()
city = StringField()
state = StringField()
country = StringField()
# Creation attributes
owner = ReferenceField('MediaUser', required=False)
created_time = DateTimeField(default=datetime.now())
updated_time = DateTimeField(required=False)
# Control
enabled = BooleanField(required=False, default=False)
# geo-enabled properties
location = GeoPointField()
poi_marker_data = DictField(required=False)
# IoT properties
internet_settings = DictField(required=False)
# Image properties
icon_image_url = StringField(default="")
icon_content = ReferenceField('JpegImageContent', required=False)
image_url = StringField(default="")
image_content = ReferenceField('JpegImageContent', required=False)
# e.g. Mall as the media-source
inhouse_source = ReferenceField('DigitalMediaSource', required=False)
# retail sources inside a MediaAggregate
digital_sourcelist = ListField(ReferenceField('DigitalMediaSource'),
default=[],
required=False)
# OOH advertisement source for MediaAggregate
ooh_sourcelist = ListField(ReferenceField('OOHMediaSource'),
default=[],
required=False)
# Radio advertisement source for MediaAggregate
radio_sourcelist = ListField(ReferenceField('RadioMediaSource'),
default=[],
required=False)
def get_absolute_url(self):
return "/mediaaggregates/%i/" % self.id
class Articles(Document):
article_id = StringField(primary_key=True)
article_title = StringField(required=True)
author = StringField()
board = StringField()
content = StringField(required=True)
date = StringField()
ip = StringField()
message_count = DictField()
messages = ListField(DictField())
url = URLField()
created_at = DateTimeField(default=datetime.now)
updated_at = DateTimeField(default=datetime.now)
meta = {"allow_inheritance": True}
class Service(Document):
"""
Represents a service instance.
"""
id = UUIDField(primary_key=True, default=uuid4)
created_at = DateTimeField(default=datetime.utcnow)
status = StringField()
descriptor = DictField(required=True)
functions = EmbeddedDocumentListField(Function, required=True)
placement = DictField()
class Tables(DynamicDocument):
contribution = LazyReferenceField(
Contributions,
passthrough=True,
reverse_delete_rule=CASCADE,
required=True,
help_text="contribution this table belongs to",
)
is_public = BooleanField(
required=True, default=False, help_text="public/private table"
)
name = StringField(required=True, help_text="table name")
label = StringField(required=True, help_text="table label")
columns = ListField(StringField(), required=True, help_text="column names")
data = ListField(ListField(StringField()), required=True, help_text="table rows")
config = DictField(help_text="graph config")
meta = {
"collection": "tables",
"indexes": ["contribution", "is_public", "name", "label", "columns"],
}
@classmethod
def post_save(cls, sender, document, **kwargs):
set_root_keys = set(k.split(".", 1)[0] for k in document._delta()[0].keys())
cid = document.contribution.id
nbs = Notebooks.objects(pk=cid)
if not set_root_keys or set_root_keys == {"is_public"}:
nbs.update(set__is_public=document.is_public)
else:
nbs.delete()
if "data" in set_root_keys:
document.update(unset__total_data_rows=True)
Contributions.objects(pk=cid).update(unset__tables=True)
class Overlay(Document):
meta = {"collection": "noc.gis.overlays", "strict": False, "auto_create_index": False}
name = StringField(required=True)
gate_id = StringField(unique=True)
is_active = BooleanField(required=True, default=True)
permission_name = StringField(required=True)
overlay = StringField(required=True)
config = DictField()
_overlay_cache = {} # name -> Overlay
def __str__(self):
return self.name
def get_overlay(self):
if self.overlay not in self._overlay_cache:
from noc.gis.overlays.base import OverlayHandler
m = __import__("noc.gis.overlays.%s" % self.overlay, {}, {}, "*")
for n in dir(m):
o = getattr(m, n)
if inspect.isclass(o) and o != OverlayHandler and issubclass(o, OverlayHandler):
self._overlay_cache[self.overlay] = o
break
h = self._overlay_cache[self.overlay]
return h(**self.config)
class StructuredQuery(Document):
userid = ObjectIdField(required=False)
query_object_type = StringField(required=True)
query_type = StringField(default='structured', required=False)
query = DictField(required=False)
creation_time = DateTimeField(default=datetime.now())
query_runtime_duration = FloatField()
class Tables(Document):
contribution = LazyReferenceField(
Contributions,
passthrough=True,
reverse_delete_rule=CASCADE,
required=True,
help_text="contribution this table belongs to",
)
is_public = BooleanField(
required=True, default=False, help_text="public/private table"
)
name = StringField(required=True, help_text="table name")
columns = ListField(StringField(), required=True, help_text="column names")
data = ListField(ListField(StringField()), required=True, help_text="table rows")
config = DictField(help_text="graph config")
meta = {
"collection": "tables",
"indexes": [
"contribution",
"is_public",
"name",
"columns",
{"fields": ("contribution", "name"), "unique": True},
],
}
@classmethod
def post_save(cls, sender, document, **kwargs):
Notebooks.objects(pk=document.contribution.id).delete()
class Certificate(Observable):
"""X509 certificate"""
value = StringField(verbose_name="Value")
hashes = ListField(DictField(), verbose_name="Hashes")
body = ReferenceField("AttachedFile")
exclude_fields = Observable.exclude_fields + ['body']
@staticmethod
def check_type(txt):
return True
@classmethod
def from_data(cls, data, hash_sha256=None):
"""Creates a Certificate observable based on raw certificate data and
its hash_sha256 value.
"""
if hash_sha256 is None:
hash_sha256 = hashlib.sha256(data).hexdigest()
body = AttachedFile.from_content(
BytesIO(data),
"cert.der",
"application/pkix-cert",
)
return cls.get_or_create(
value="CERT:{}".format(hash_sha256),
body=body,
)
class Playing(Document):
"""
At a given time, a media source could play multiple Campaigns.
The time is captured as start and end, where the relationship is valid.
This class realizes a relationship.
If the media owner decides to abrupt the campaign, he sets the flag
is_valid to False through Media dash-board. deletion_date will be updated
for records.
"""
# for e.g. Mall's default source
primary_media_source = ReferenceField('MediaSource', required=False)
playing_content = ReferenceField('Campaign', default=None, required=False)
# for e.g. VOD, OOH, Sensor etc.
source_type = StringField()
# official start-end date
start_date = DateTimeField(default=datetime.now())
end_date = DateTimeField()
# forceful setting of the play flag
pause_playing = BooleanField(default=False)
# book-keeping entries
creation_date = DateTimeField(default=datetime.now())
deletion_date = DateTimeField()
# Status
state = StringField(required=False)
# vendor book-keeping (campaign-id, sensor-id, zone-id etc.)
playing_vendor_attributes = DictField(required=False)
def get_absolute_url(self):
return "/mediacontent/playing/%i/" % self.id
class CommentModel(DynamicDocument):
blockchain = StringField()
author = StringField()
parent_author = StringField()
created = DateTimeField()
last_update = DateTimeField()
permlink = StringField()
parent_permlink = StringField()
body = StringField()
title = StringField()
meta = StringField()
json = DictField()
meta = {
'collection': 'posts',
'ordering': ['-created'],
'indexes': [
'blockchain',
'author',
'parent_author',
'permlink',
'parent_permlink',
'created',
'json.app',
'json.tags'
],
'auto_create_index': True,
'index_background': True
}
class MultiplexExtension(AmenityExtension):
"""
This extension adds a multi-plex to be part of the
amenity.
"""
ex_name = StringField(default='Multiplex', required=False)
ex_type = StringField(default='moviemultiplex')
brand_name = StringField(required=True)
brand_description = StringField(required=True)
brand_url = StringField(required=True)
# floor, shop number etc.
outlet_address1 = StringField(required=True)
outlet_address2 = StringField(required=True)
# > 400
audis = StringField()
capacity_per_audi = StringField()
# e.g. precaution etc.
service_condition = StringField()
reservation_facility = BooleanField()
reservation_number = StringField()
rservation_url = StringField()
average_price = StringField()
open_days = StringField()
# [9:30-12:30, 12:45-2:30]
show_timings = ListField()
# Internal data, Stored in ad-wise
# not to be serialized
_meta = DictField(required=False)
def get_absolute_url(self):
return "/mediacontent/extension/movie/%i/" % self.id
class Enumeration(Document):
meta = {
"collection": "noc.enumerations",
"allow_inheritance": False,
"json_collection": "fm.enumerations"
}
name = StringField(unique=True)
uuid = UUIDField(binary=True)
values = DictField() # value -> [possible combinations]
def __unicode__(self):
return self.name
def get_json_path(self):
return "%s.json" % quote_safe_path(self.name)
def to_json(self):
return to_json(
{
"name": self.name,
"$collection": self._meta["json_collection"],
"uuid": self.uuid,
"values": self.values
},
order=["name", "$collection", "uuid"])
class Sensor(MediaSource):
name = StringField(required=True)
uuid = StringField(required=True, unique=True)
# beacon, wifi, gps etc.
type = "sensor"
range = FloatField(default=10.0)
# zone_name = StringField(required=False)
# zone_id = StringField(required=True)
# vendor info - nikaza, nearby, google
vendor = StringField(default='nearby', required=False)
# Mac Addr
mac_address = StringField(required=True)
# Every sensor can be associated with
# an optional [lat,lng] fields.
location = GeoPointField(required=False)
# reference to the venue
venue = ReferenceField('Venue', required=False)
# vendor metadata
# for e.g. Nikaza {zoneId: "", zoneName: ""}
sensor_meta = DictField(default={}, required=False)
meta = {'allow_inheritance': True}
class Meta:
abstract = True
def get_absolute_url(self):
return "/mediasource/sensor/%s/" % (self.id)