class NPM_VULNERS(peewee.Model):
class Meta:
database = database
table_name = "npm_vulners"
id = peewee.PrimaryKeyField(null=False)
item = peewee.TextField(unique=True, default="", verbose_name="NPM ID")
title = peewee.TextField(default="", verbose_name="NPM Title")
created_at = peewee.TextField(default="", verbose_name="NPM Created At")
updated_at = peewee.TextField(default="", verbose_name="NPM Updated At")
publish_date = peewee.TextField(default="",
verbose_name="NPM Publish Date")
author = peewee.TextField(default="", verbose_name="NPM Vulner Author")
module_name = peewee.TextField(default="", verbose_name="NPM Module Name")
cves = ArrayField(peewee.TextField, default=[])
vulnerable_versions = peewee.TextField(
default="", verbose_name="NPM Vulnerable Versions")
patched_versions = peewee.TextField(default="",
verbose_name="NPM Patched Versions")
slug = peewee.TextField(default="", verbose_name="NPM Slug")
overview = peewee.TextField(default="", verbose_name="NPM Overview")
recomendation = peewee.TextField(default="",
verbose_name="NPM Recomendation")
references = peewee.TextField(default="", verbose_name="NPM References")
legacy_slug = peewee.TextField(default="", verbose_name="NPM Legacy Slug")
allowed_scopes = ArrayField(
peewee.TextField,
default=[],
)
cvss_vector = peewee.TextField(default="", verbose_name="NPM CVSS Vector")
cvss_score = peewee.TextField(default="", verbose_name="NPM CVSS Score")
cwe = peewee.TextField(default="", verbose_name="NPM CWE Reference")
def __unicode__(self):
return "NPM"
def __str__(self):
return self.item
@property
def data(self):
npm_data = {}
npm_data["id"] = self.id
npm_data["item"] = self.item
npm_data["title"] = self.title
npm_data["created_at"] = self.created_at
npm_data["updated_at"] = self.updated_at
npm_data["publish_date"] = self.publish_date
npm_data["author"] = self.author
npm_data["module_name"] = self.module_name
npm_data["cves"] = self.cves
npm_data["vulnerable_versions"] = self.vulnerable_versions
npm_data["patched_versions"] = self.patched_versions
npm_data["slug"] = self.slug
npm_data["overview"] = self.overview
npm_data["recomendation"] = self.recomendation
npm_data["references"] = self.references
npm_data["legacy_slug"] = self.legacy_slug
npm_data["allowed_scopes"] = self.allowed_scopes
npm_data["cvss_vector"] = self.cvss_vector
npm_data["cvss_score"] = self.cvss_score
npm_data["cwe"] = self.cwe
return npm_data
def save(self, *args, **kwargs):
self.item = ''.join(filter(lambda x: x.isdigit(), self.item))
self.item = "NPM-" + self.item
super(NPM_VULNERS, self).save(*args, **kwargs)
class StoreItem(BaseModel):
item_id = pw.PrimaryKeyField(verbose_name='条目ID')
store_id = pw.IntegerField(verbose_name='商店ID')
product_id = pw.IntegerField(verbose_name='物品ID')
product_num = pw.IntegerField(verbose_name='物品数量')
title = pw.CharField(max_length=512, verbose_name='商品描述')
credit_type = pw.IntegerField(choices=CREDIT_TYPE, verbose_name='价格类型')
credit_value = pw.IntegerField(default=0, verbose_name='价格数值')
total_num = pw.IntegerField(verbose_name='总库存(份)')
use_num = pw.IntegerField(verbose_name='消耗数量(份)')
left_num = pw.IntegerField(verbose_name='剩余库存(份)')
order = pw.IntegerField(verbose_name='显示顺序')
identity = pw.CharField(default='',
max_length=512,
verbose_name='营销平台奖项标识')
class Meta:
db_table = 'store_item'
@classmethod
def get_store_items(cls, store_id):
key = STORE_ITEM_KEY % ({'store_id': store_id})
items = Redis.get(key)
if items:
items = cjson.loads(items)
items = [dict_to_model(StoreItem, i) for i in items]
else:
items = list(cls.select().where(cls.store_id == int(store_id)))
_items = [model_to_dict(i) for i in items]
Redis.setex(key, 86400, cjson.dumps(_items, 2))
return sorted(items, key=lambda x: x.order)
@classmethod
def get_store_item(cls, store_id, item_id):
items = filter(lambda x: x.item_id == int(item_id),
cls.get_store_items(store_id))
return items[0] if items else None
@classmethod
def get_item_by_identity(cls, store_id, identity):
if not identity:
return None
items = filter(
lambda x: x.store_id == int(store_id) and x.identity == identity,
cls.get_store_items(store_id))
return items[0] if items else None
def format(self):
product = Product.get_product(self.product_id)
data = {
'item_id': self.item_id,
'title': self.title,
'credit_type': self.credit_type,
'credit_value': self.credit_value,
'product': product.format(),
'product_num': self.product_num,
'left_num': self.left_num,
'use_num': self.use_num
}
return data
def save(self, *args, **kwargs):
ret = super(StoreItem, self).save(*args, **kwargs)
key = STORE_ITEM_KEY % ({'store_id': self.store_id})
Redis.delete(key)
return ret
class UserGiftCodeOrder(BaseModel):
order_id = pw.PrimaryKeyField(verbose_name='订单ID')
user_id = pw.CharField(max_length=64, verbose_name='用户ID')
vgc_id = pw.CharField(max_length=64, verbose_name='礼包ID')
vgc_name = pw.CharField(max_length=64, verbose_name='礼包名称')
campaign_id = pw.CharField(default='',
max_length=64,
verbose_name='营销平台活动ID')
gift_code = pw.CharField(max_length=64, verbose_name='礼包兑换码')
result = pw.CharField(default='', max_length=1024, verbose_name='营销平台结果')
recid = pw.CharField(null=True, max_length=128, verbose_name='营销平台订单号')
expire_at = pw.DateTimeField(
constraints=[pw.SQL('DEFAULT CURRENT_TIMESTAMP')],
formats='%Y-%m-%d %H:%M:%S',
verbose_name='失效时间')
status = pw.IntegerField(choices=STATUS, verbose_name='状态')
user_ip = pw.CharField(default=None,
max_length=16,
verbose_name='用户抽奖ip地址')
class Meta:
db_table = 'user_gift_code_order'
def format(self):
data = {
'order_id': self.order_id,
'create_at': util.datetime2timestamp(self.create_at),
'name': self.vgc_name,
'exchange_code': self.gift_code,
'expire_at': util.datetime2timestamp(self.expire_at)
}
return data
@classmethod
@util.cached_list(lambda cls, uid: USER_GIFT_CODE_KEY % {'uid': uid},
snowslide=True)
def _load_user_gift_code_ids(cls, uid):
orders = list(
cls.select(UserGiftCodeOrder.vgc_id).where(cls.user_id == uid))
return [odr.vgc_id for odr in orders]
@classmethod
def get_user_gift_code_ids(cls, uid):
key = USER_GIFT_CODE_KEY % {'uid': uid}
if not Redis.exists(key):
cls._load_user_gift_code_ids(uid)
try:
ids = Redis.lrange(key, 0, -1)
except:
ids = []
return ids
@classmethod
def get_user_orders(cls, user_id, page, pagesize):
orders = list(cls.select().where(cls.user_id == user_id).order_by(
cls.create_at.desc()).paginate(page, pagesize))
return orders
@classmethod
def clear_redis(cls, uid):
key = USER_GIFT_CODE_KEY % {'uid': uid}
Redis.delete(key)
class PayForGift(BaseModel):
order_id = pw.PrimaryKeyField(verbose_name='订单号')
user_id = pw.CharField(max_length=64, verbose_name='用户ID')
phone = pw.CharField(max_length=64, verbose_name='手机号')
exchange_time = pw.DateTimeField(formats='%Y-%m-%d %H:%M:%S',
verbose_name='申请兑换时间')
exchange_mon = pw.CharField(max_length=64, verbose_name='申请兑换月份')
exchange_status = pw.IntegerField(choices=EXCHANGE_RESULT,
verbose_name='兑换状态')
gift_value = pw.FloatField(default=0.0, verbose_name='礼物总价')
exchange_value = pw.IntegerField(default=0, verbose_name='兑换总价')
action = pw.IntegerField(choices=TRADE_ACTION, verbose_name='交易描述')
class Meta:
db_table = 'pay_for_gift'
@classmethod
def get_all_value(cls):
mon = time.strftime('%Y-%m', time.localtime(time.time()))
key = 'PayForGift:all_exchange_value:{0}'.format(mon)
value = Redis.get(key)
if not value:
value = 0
pay_for_gift_logs = list(cls.select())
for log in pay_for_gift_logs:
value += log.exchange_value
Redis.set(key, value)
return value
@classmethod
def is_exchange(cls, user):
mon = time.strftime('%Y-%m', time.localtime(time.time()))
try:
log = cls.select().where(cls.user_id == user._id,
cls.exchange_mon == mon).get()
is_exchange = True if log else False
except:
is_exchange = False
return is_exchange
@classmethod
def get_user_logs(cls, user_id, page, pagesize):
logs = list(cls.select().where(cls.user_id == user_id,
cls.exchange_status == 1).order_by(
cls.create_at.desc()).paginate(
page, pagesize))
return logs
@classmethod
def create_log(cls, user, exchange_status, gift_value, exchange_value,
action):
if float(gift_value) > 0:
exchange_time = datetime.now()
order_id = uuid.uuid1()
mon = time.strftime('%Y-%m', time.localtime(time.time()))
return cls.create(order_id=order_id,
user_id=user._id,
phone=user.phone,
exchange_time=exchange_time,
exchange_mon=mon,
exchange_status=exchange_status,
gift_value=gift_value,
exchange_value=exchange_value,
action=action)
else:
return
def format(self):
data = {
'exchange_time': util.datetime2timestamp(self.exchange_time),
'gift_value': self.gift_value,
'exchange_value': self.exchange_value,
'action': self.action,
'desc': util.get_choices_desc(TRADE_ACTION, self.action),
'create_at': util.datetime2timestamp(self.create_at)
}
return data
class Query(ModelTimestampsMixin, BaseModel):
id = peewee.PrimaryKeyField()
data_source = peewee.ForeignKeyField(DataSource)
latest_query_data = peewee.ForeignKeyField(QueryResult, null=True)
name = peewee.CharField(max_length=255)
description = peewee.CharField(max_length=4096, null=True)
query = peewee.TextField()
query_hash = peewee.CharField(max_length=32)
api_key = peewee.CharField(max_length=40)
user_email = peewee.CharField(max_length=360, null=True)
user = peewee.ForeignKeyField(User)
last_modified_by = peewee.ForeignKeyField(User,
null=True,
related_name="modified_queries")
is_archived = peewee.BooleanField(default=False, index=True)
schedule = peewee.CharField(max_length=10, null=True)
class Meta:
db_table = 'queries'
def to_dict(self,
with_stats=False,
with_visualizations=False,
with_user=True):
d = {
'id': self.id,
'latest_query_data_id': self._data.get('latest_query_data', None),
'name': self.name,
'description': self.description,
'query': self.query,
'query_hash': self.query_hash,
'schedule': self.schedule,
'api_key': self.api_key,
'is_archived': self.is_archived,
'updated_at': self.updated_at,
'created_at': self.created_at,
'data_source_id': self._data.get('data_source', None)
}
if with_user:
d['user'] = self.user.to_dict()
d['last_modified_by'] = self.last_modified_by.to_dict()
else:
d['user_id'] = self._data['user']
if with_stats:
d['retrieved_at'] = self.retrieved_at
d['runtime'] = self.runtime
if with_visualizations:
d['visualizations'] = [
vis.to_dict(with_query=False) for vis in self.visualizations
]
return d
def archive(self):
self.is_archived = True
self.schedule = None
for vis in self.visualizations:
for w in vis.widgets:
w.delete_instance()
self.save()
@classmethod
def all_queries(cls):
q = Query.select(Query, User, QueryResult.retrieved_at, QueryResult.runtime)\
.join(QueryResult, join_type=peewee.JOIN_LEFT_OUTER)\
.switch(Query).join(User)\
.where(Query.is_archived==False)\
.group_by(Query.id, User.id, QueryResult.id, QueryResult.retrieved_at, QueryResult.runtime)\
.order_by(cls.created_at.desc())
return q
@classmethod
def outdated_queries(cls):
queries = cls.select(cls, QueryResult.retrieved_at, DataSource)\
.join(QueryResult)\
.switch(Query).join(DataSource)\
.where(cls.schedule != None)
now = datetime.datetime.utcnow().replace(
tzinfo=psycopg2.tz.FixedOffsetTimezone(offset=0, name=None))
outdated_queries = {}
for query in queries:
if should_schedule_next(query.latest_query_data.retrieved_at, now,
query.schedule):
key = "{}:{}".format(query.query_hash, query.data_source.id)
outdated_queries[key] = query
return outdated_queries.values()
@classmethod
def search(cls, term):
# This is very naive implementation of search, to be replaced with PostgreSQL full-text-search solution.
where = (cls.name**u"%{}%".format(term)) | (cls.description**
u"%{}%".format(term))
if term.isdigit():
where |= cls.id == term
where &= cls.is_archived == False
return cls.select().where(where).order_by(cls.created_at.desc())
@classmethod
def recent(cls, user_id):
# TODO: instead of t2 here, we should define table_alias for Query table
return cls.select().where(Event.created_at > peewee.SQL("current_date - 7")).\
join(Event, on=(Query.id == peewee.SQL("t2.object_id::integer"))).\
where(Event.action << ('edit', 'execute', 'edit_name', 'edit_description', 'view_source')).\
where(Event.user == user_id).\
where(~(Event.object_id >> None)).\
where(Event.object_type == 'query'). \
where(cls.is_archived == False).\
group_by(Event.object_id, Query.id).\
order_by(peewee.SQL("count(0) desc"))
@classmethod
def update_instance(cls, query_id, **kwargs):
if 'query' in kwargs:
kwargs['query_hash'] = utils.gen_query_hash(kwargs['query'])
update = cls.update(**kwargs).where(cls.id == query_id)
return update.execute()
def pre_save(self, created):
super(Query, self).pre_save(created)
self.query_hash = utils.gen_query_hash(self.query)
self._set_api_key()
if self.last_modified_by is None:
self.last_modified_by = self.user
def post_save(self, created):
if created:
self._create_default_visualizations()
def _create_default_visualizations(self):
table_visualization = Visualization(query=self,
name="Table",
description='',
type="TABLE",
options="{}")
table_visualization.save()
def _set_api_key(self):
if not self.api_key:
self.api_key = hashlib.sha1(u''.join(
(str(time.time()), self.query, str(self._data['user']),
self.name)).encode('utf-8')).hexdigest()
@property
def runtime(self):
return self.latest_query_data.runtime
@property
def retrieved_at(self):
return self.latest_query_data.retrieved_at
def __unicode__(self):
return unicode(self.id)
class Copy(BaseModel):
"""Data model for Copy
"""
CopyID = pw.PrimaryKeyField()
#DocReference = pw.ForeignKeyField(Document, related_name = 'copies')
docClass = pw.CharField()
docId = pw.IntegerField()
active = pw.BooleanField(default=True)
#checked_out = pw.BooleanField(default=False)
# 0 - not cheked out, 1 - reserved, 2 - checked out
checked_out = pw.SmallIntegerField(default=0)
storage = pw.CharField(default='')
def get_doc(self):
"""Get the document to which this copy referred
"""
doc_class = name_to_class()[self.docClass]
return doc_class.get_by_id(self.docId)
@classmethod
def get_by_id(cls, copy_id):
return cls.get(CopyID=copy_id)
@classmethod
def add(cls, doc, storage=''):
"""Add copy of specific document
"""
res = cls.create(docClass=class_to_name()[type(doc)],
docId=doc.DocumentID)
# Activate document if it is deactivated
if (doc.active == False):
doc.active = True
doc.save()
event_manager.send_event('free_copy', res)
return res
@classmethod
def edit_storage(cls, copy_id, new_storage):
"""Edit storage place for specific copy
"""
temp = cls.get(CopyID=copy_id)
temp.storage = new_storage
temp.save()
@classmethod
def remove(cls, copy_id):
"""Removes (deactivates) copy"""
entry = cls.get(CopyID=copy_id)
entry.active = False
entry.save()
# Check number of active copies of document. If zero - deactivate document
doc = entry.get_doc()
doc_class = class_to_name()[type(doc)]
query = Copy.select().where(Copy.docClass == doc_class,
Copy.docId == doc.DocumentID,
Copy.active == True).count()
if (query == 0):
doc.active = False
doc.save()
@classmethod
def restore(cls, copy_id):
"""Restores deleted copy"""
entry = cls.get_by_id(copy_id)
entry.active = True
entry.save()
doc = entry.get_doc()
if (doc.active == False):
doc.active = True
doc.save()
class User(BaseModel):
id = peewee.PrimaryKeyField()
username = peewee.CharField(max_length=64)
password = peewee.CharField(max_length=64)
level = peewee.IntegerField() # 0: Salseman, 1 Gunsmith
class DataSource(BelongsToOrgMixin, BaseModel):
id = peewee.PrimaryKeyField()
org = peewee.ForeignKeyField(Organization, related_name="data_sources")
name = peewee.CharField()
type = peewee.CharField()
options = ConfigurationField()
queue_name = peewee.CharField(default="queries")
scheduled_queue_name = peewee.CharField(default="scheduled_queries")
created_at = DateTimeTZField(default=datetime.datetime.now)
class Meta:
db_table = 'data_sources'
indexes = (
(('org', 'name'), True),
)
def to_dict(self, all=False, with_permissions=False):
d = {
'id': self.id,
'name': self.name,
'type': self.type,
'syntax': self.query_runner.syntax,
'paused': self.paused,
'pause_reason': self.pause_reason
}
if all:
schema = get_configuration_schema_for_query_runner_type(self.type)
self.options.set_schema(schema)
d['options'] = self.options.to_dict(mask_secrets=True)
d['queue_name'] = self.queue_name
d['scheduled_queue_name'] = self.scheduled_queue_name
d['groups'] = self.groups
if with_permissions:
d['view_only'] = self.data_source_groups.view_only
return d
def __unicode__(self):
return self.name
@classmethod
def create_with_group(cls, *args, **kwargs):
data_source = cls.create(*args, **kwargs)
DataSourceGroup.create(data_source=data_source, group=data_source.org.default_group)
return data_source
def get_schema(self, refresh=False):
key = "data_source:schema:{}".format(self.id)
cache = None
if not refresh:
cache = redis_connection.get(key)
if cache is None:
query_runner = self.query_runner
schema = sorted(query_runner.get_schema(get_stats=refresh), key=lambda t: t['name'])
redis_connection.set(key, json.dumps(schema))
else:
schema = json.loads(cache)
return schema
def _pause_key(self):
return 'ds:{}:pause'.format(self.id)
@property
def paused(self):
return redis_connection.exists(self._pause_key())
@property
def pause_reason(self):
return redis_connection.get(self._pause_key())
def pause(self, reason=None):
redis_connection.set(self._pause_key(), reason)
def resume(self):
redis_connection.delete(self._pause_key())
def add_group(self, group, view_only=False):
dsg = DataSourceGroup.create(group=group, data_source=self, view_only=view_only)
setattr(self, 'data_source_groups', dsg)
def remove_group(self, group):
DataSourceGroup.delete().where(DataSourceGroup.group==group, DataSourceGroup.data_source==self).execute()
def update_group_permission(self, group, view_only):
dsg = DataSourceGroup.get(DataSourceGroup.group==group, DataSourceGroup.data_source==self)
dsg.view_only = view_only
dsg.save()
setattr(self, 'data_source_groups', dsg)
@property
def query_runner(self):
return get_query_runner(self.type, self.options)
@classmethod
def all(cls, org, groups=None):
data_sources = cls.select().where(cls.org==org).order_by(cls.id.asc())
if groups:
data_sources = data_sources.join(DataSourceGroup).where(DataSourceGroup.group << groups)
return data_sources
@property
def groups(self):
groups = DataSourceGroup.select().where(DataSourceGroup.data_source==self)
return dict(map(lambda g: (g.group_id, g.view_only), groups))
class QueryResult(BaseModel, BelongsToOrgMixin):
id = peewee.PrimaryKeyField()
org = peewee.ForeignKeyField(Organization)
data_source = peewee.ForeignKeyField(DataSource)
query_hash = peewee.CharField(max_length=32, index=True)
query = peewee.TextField()
data = peewee.TextField()
runtime = peewee.FloatField()
retrieved_at = DateTimeTZField()
class Meta:
db_table = 'query_results'
def to_dict(self):
return {
'id': self.id,
'query_hash': self.query_hash,
'query': self.query,
'data': json.loads(self.data),
'data_source_id': self.data_source_id,
'runtime': self.runtime,
'retrieved_at': self.retrieved_at
}
@classmethod
def unused(cls, days=7):
age_threshold = datetime.datetime.now() - datetime.timedelta(days=days)
unused_results = cls.select().where(Query.id == None, cls.retrieved_at < age_threshold)\
.join(Query, join_type=peewee.JOIN_LEFT_OUTER)
return unused_results
@classmethod
def get_latest(cls, data_source, query, max_age=0):
query_hash = utils.gen_query_hash(query)
if max_age == -1:
query = cls.select().where(cls.query_hash == query_hash,
cls.data_source == data_source).order_by(cls.retrieved_at.desc())
else:
query = cls.select().where(cls.query_hash == query_hash, cls.data_source == data_source,
peewee.SQL("retrieved_at at time zone 'utc' + interval '%s second' >= now() at time zone 'utc'",
max_age)).order_by(cls.retrieved_at.desc())
return query.first()
@classmethod
def store_result(cls, org_id, data_source_id, query_hash, query, data, run_time, retrieved_at):
query_result = cls.create(org=org_id,
query_hash=query_hash,
query=query,
runtime=run_time,
data_source=data_source_id,
retrieved_at=retrieved_at,
data=data)
logging.info("Inserted query (%s) data; id=%s", query_hash, query_result.id)
sql = "UPDATE queries SET latest_query_data_id = %s WHERE query_hash = %s AND data_source_id = %s RETURNING id"
query_ids = [row[0] for row in db.database.execute_sql(sql, params=(query_result.id, query_hash, data_source_id))]
# TODO: when peewee with update & returning support is released, we can get back to using this code:
# updated_count = Query.update(latest_query_data=query_result).\
# where(Query.query_hash==query_hash, Query.data_source==data_source_id).\
# execute()
logging.info("Updated %s queries with result (%s).", len(query_ids), query_hash)
return query_result, query_ids
def __unicode__(self):
return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at)
@property
def groups(self):
return self.data_source.groups
class Papers_NR_NSW_STE(BaseModel):
id = pw.PrimaryKeyField()
pdf_name = pw.TextField()
paper_text = pw.TextField()
paper_title = pw.TextField()
class User(ModelTimestampsMixin, BaseModel, BelongsToOrgMixin, UserMixin, PermissionsCheckMixin):
id = peewee.PrimaryKeyField()
org = peewee.ForeignKeyField(Organization, related_name="users")
name = peewee.CharField(max_length=320)
email = peewee.CharField(max_length=320)
password_hash = peewee.CharField(max_length=128, null=True)
groups = ArrayField(peewee.IntegerField, null=True)
api_key = peewee.CharField(max_length=40, unique=True)
class Meta:
db_table = 'users'
indexes = (
(('org', 'email'), True),
)
def __init__(self, *args, **kwargs):
super(User, self).__init__(*args, **kwargs)
def to_dict(self, with_api_key=False):
d = {
'id': self.id,
'name': self.name,
'email': self.email,
'gravatar_url': self.gravatar_url,
'groups': self.groups,
'updated_at': self.updated_at,
'created_at': self.created_at
}
if self.password_hash is None:
d['auth_type'] = 'external'
else:
d['auth_type'] = 'password'
if with_api_key:
d['api_key'] = self.api_key
return d
def pre_save(self, created):
super(User, self).pre_save(created)
if not self.api_key:
self.api_key = generate_token(40)
@property
def gravatar_url(self):
email_md5 = hashlib.md5(self.email.lower()).hexdigest()
return "https://www.gravatar.com/avatar/%s?s=40" % email_md5
@property
def permissions(self):
# TODO: this should be cached.
return list(itertools.chain(*[g.permissions for g in
Group.select().where(Group.id << self.groups)]))
@classmethod
def get_by_email_and_org(cls, email, org):
return cls.get(cls.email == email, cls.org == org)
@classmethod
def get_by_api_key_and_org(cls, api_key, org):
return cls.get(cls.api_key == api_key, cls.org == org)
@classmethod
def all(cls, org):
return cls.select().where(cls.org == org)
@classmethod
def find_by_email(cls, email):
return cls.select().where(cls.email == email)
def __unicode__(self):
return u'%s (%s)' % (self.name, self.email)
def hash_password(self, password):
self.password_hash = pwd_context.encrypt(password)
def verify_password(self, password):
return self.password_hash and pwd_context.verify(password, self.password_hash)
def update_group_assignments(self, group_names):
groups = Group.find_by_name(self.org, group_names)
groups.append(self.org.default_group)
self.groups = map(lambda g: g.id, groups)
self.save()
class Citations(BaseModel):
id = pw.PrimaryKeyField()
source_paper = pw.ForeignKeyField(Papers, related_name='sorce')
cited_paper = pw.ForeignKeyField(Papers, related_name='cited')
class Paper_authors(BaseModel):
id = pw.PrimaryKeyField()
paper_id = pw.ForeignKeyField(Papers)
author_id = pw.ForeignKeyField(Authors)
class Authors(BaseModel):
id = pw.PrimaryKeyField()
name = pw.TextField()
class League(BaseModel):
id = peewee.PrimaryKeyField()
name = peewee.TextField()
class Query(ModelTimestampsMixin, BaseModel, BelongsToOrgMixin):
id = peewee.PrimaryKeyField()
org = peewee.ForeignKeyField(Organization, related_name="queries")
data_source = peewee.ForeignKeyField(DataSource, null=True)
latest_query_data = peewee.ForeignKeyField(QueryResult, null=True)
name = peewee.CharField(max_length=255)
description = peewee.CharField(max_length=4096, null=True)
query = peewee.TextField()
query_hash = peewee.CharField(max_length=32)
api_key = peewee.CharField(max_length=40)
user = peewee.ForeignKeyField(User)
last_modified_by = peewee.ForeignKeyField(User, null=True, related_name="modified_queries")
is_archived = peewee.BooleanField(default=False, index=True)
schedule = peewee.CharField(max_length=10, null=True)
options = JSONField(default={})
class Meta:
db_table = 'queries'
def to_dict(self, with_stats=False, with_visualizations=False, with_user=True, with_last_modified_by=True):
d = {
'id': self.id,
'latest_query_data_id': self._data.get('latest_query_data', None),
'name': self.name,
'description': self.description,
'query': self.query,
'query_hash': self.query_hash,
'schedule': self.schedule,
'api_key': self.api_key,
'is_archived': self.is_archived,
'updated_at': self.updated_at,
'created_at': self.created_at,
'data_source_id': self.data_source_id,
'options': self.options
}
if with_user:
d['user'] = self.user.to_dict()
else:
d['user_id'] = self.user_id
if with_last_modified_by:
d['last_modified_by'] = self.last_modified_by.to_dict() if self.last_modified_by is not None else None
else:
d['last_modified_by_id'] = self.last_modified_by_id
if with_stats:
d['retrieved_at'] = self.retrieved_at
d['runtime'] = self.runtime
if with_visualizations:
d['visualizations'] = [vis.to_dict(with_query=False)
for vis in self.visualizations]
return d
def archive(self):
self.is_archived = True
self.schedule = None
for vis in self.visualizations:
for w in vis.widgets:
w.delete_instance()
for alert in self.alerts:
alert.delete_instance(recursive=True)
self.save()
@classmethod
def all_queries(cls, groups, drafts=False):
q = Query.select(Query, User, QueryResult.retrieved_at, QueryResult.runtime)\
.join(QueryResult, join_type=peewee.JOIN_LEFT_OUTER)\
.switch(Query).join(User)\
.join(DataSourceGroup, on=(Query.data_source==DataSourceGroup.data_source))\
.where(Query.is_archived==False)\
.where(DataSourceGroup.group << groups)\
.group_by(Query.id, User.id, QueryResult.id, QueryResult.retrieved_at, QueryResult.runtime)\
.order_by(cls.created_at.desc())
if drafts:
q = q.where(Query.name == 'New Query')
else:
q = q.where(Query.name != 'New Query')
return q
@classmethod
def by_user(cls, user, drafts):
return cls.all_queries(user.groups, drafts).where(Query.user==user)
@classmethod
def outdated_queries(cls):
queries = cls.select(cls, QueryResult.retrieved_at, DataSource)\
.join(QueryResult)\
.switch(Query).join(DataSource)\
.where(cls.schedule != None)
now = utils.utcnow()
outdated_queries = {}
for query in queries:
if should_schedule_next(query.latest_query_data.retrieved_at, now, query.schedule):
key = "{}:{}".format(query.query_hash, query.data_source.id)
outdated_queries[key] = query
return outdated_queries.values()
@classmethod
def search(cls, term, groups):
# TODO: This is very naive implementation of search, to be replaced with PostgreSQL full-text-search solution.
where = (cls.name**u"%{}%".format(term)) | (cls.description**u"%{}%".format(term))
if term.isdigit():
where |= cls.id == term
where &= cls.is_archived == False
query_ids = cls.select(peewee.fn.Distinct(cls.id))\
.join(DataSourceGroup, on=(Query.data_source==DataSourceGroup.data_source)) \
.where(where) \
.where(DataSourceGroup.group << groups)
return cls.select(Query, User).join(User).where(cls.id << query_ids)
@classmethod
def recent(cls, groups, user_id=None, limit=20):
query = cls.select(Query, User).where(Event.created_at > peewee.SQL("current_date - 7")).\
join(Event, on=(Query.id == Event.object_id.cast('integer'))). \
join(DataSourceGroup, on=(Query.data_source==DataSourceGroup.data_source)). \
switch(Query).join(User).\
where(Event.action << ('edit', 'execute', 'edit_name', 'edit_description', 'view_source')).\
where(~(Event.object_id >> None)).\
where(Event.object_type == 'query'). \
where(DataSourceGroup.group << groups).\
where(cls.is_archived == False).\
group_by(Event.object_id, Query.id, User.id).\
order_by(peewee.SQL("count(0) desc"))
if user_id:
query = query.where(Event.user == user_id)
query = query.limit(limit)
return query
def pre_save(self, created):
super(Query, self).pre_save(created)
self.query_hash = utils.gen_query_hash(self.query)
self._set_api_key()
if self.last_modified_by is None:
self.last_modified_by = self.user
def post_save(self, created):
if created:
self._create_default_visualizations()
def _create_default_visualizations(self):
table_visualization = Visualization(query=self, name="Table",
description='',
type="TABLE", options="{}")
table_visualization.save()
def _set_api_key(self):
if not self.api_key:
self.api_key = hashlib.sha1(
u''.join((str(time.time()), self.query, str(self.user_id), self.name)).encode('utf-8')).hexdigest()
@property
def runtime(self):
return self.latest_query_data.runtime
@property
def retrieved_at(self):
return self.latest_query_data.retrieved_at
@property
def groups(self):
if self.data_source is None:
return {}
return self.data_source.groups
def __unicode__(self):
return unicode(self.id)
class Season(BaseModel):
id = peewee.PrimaryKeyField()
name = peewee.TextField()
class Alert(ModelTimestampsMixin, BaseModel):
UNKNOWN_STATE = 'unknown'
OK_STATE = 'ok'
TRIGGERED_STATE = 'triggered'
id = peewee.PrimaryKeyField()
name = peewee.CharField()
query = peewee.ForeignKeyField(Query, related_name='alerts')
user = peewee.ForeignKeyField(User, related_name='alerts')
options = JSONField()
state = peewee.CharField(default=UNKNOWN_STATE)
last_triggered_at = DateTimeTZField(null=True)
rearm = peewee.IntegerField(null=True)
class Meta:
db_table = 'alerts'
@classmethod
def all(cls, groups):
return cls.select(Alert, User, Query)\
.join(Query)\
.join(DataSourceGroup, on=(Query.data_source==DataSourceGroup.data_source))\
.where(DataSourceGroup.group << groups)\
.switch(Alert)\
.join(User)\
.group_by(Alert, User, Query)
@classmethod
def get_by_id_and_org(cls, id, org):
return cls.select(Alert, User, Query).join(Query).switch(Alert).join(User).where(cls.id==id, Query.org==org).get()
def to_dict(self, full=True):
d = {
'id': self.id,
'name': self.name,
'options': self.options,
'state': self.state,
'last_triggered_at': self.last_triggered_at,
'updated_at': self.updated_at,
'created_at': self.created_at,
'rearm': self.rearm
}
if full:
d['query'] = self.query.to_dict()
d['user'] = self.user.to_dict()
else:
d['query_id'] = self.query_id
d['user_id'] = self.user_id
return d
def evaluate(self):
data = json.loads(self.query.latest_query_data.data)
# todo: safe guard for empty
value = data['rows'][0][self.options['column']]
op = self.options['op']
if op == 'greater than' and value > self.options['value']:
new_state = self.TRIGGERED_STATE
elif op == 'less than' and value < self.options['value']:
new_state = self.TRIGGERED_STATE
elif op == 'equals' and value == self.options['value']:
new_state = self.TRIGGERED_STATE
else:
new_state = self.OK_STATE
return new_state
def subscribers(self):
return User.select().join(AlertSubscription).where(AlertSubscription.alert==self)
@property
def groups(self):
return self.query.groups
class Document(BaseModel):
"""Base data model for all document classes
"""
DocumentID = pw.PrimaryKeyField()
title = pw.CharField()
author = pw.CharField()
cost = pw.IntegerField()
keywords = pw.CharField()
active = pw.BooleanField(default=True)
requested = pw.BooleanField(default=False)
def get_document_copies(self):
"""Get list of copies of speciific document
"""
doc_class = class_to_name()[type(self)]
query = Copy.select().where(Copy.docClass == doc_class,
Copy.docId == self.DocumentID)
res = []
for entry in query:
res.append(entry)
return res
def enable_request(self):
self.requested = True
self.save()
def cancel_request(self):
self.requested = False
self.save()
@classmethod
def get_by_id(cls, doc_id):
"""Get document by id
"""
return cls.get(DocumentID=doc_id)
@classmethod
def add(cls, args):
"""Creates a new entity of selected type. Takes on input dictionary of values
"""
return cls.create(**args)
@classmethod
def remove(cls, doc_id):
"""Removes an entity with specific DocumentID
"""
entry = cls.get(DocumentID=doc_id)
entry.active = False
entry.save()
# Removing all copies
doc_class = class_to_name()[cls]
update_query = Copy.update(active=False).where(
Copy.docClass == doc_class, Copy.docId == doc_id,
Copy.active == True)
update_query.execute()
@classmethod
def edit(cls, doc_id, new_values):
"""Edit certain values in an entity with specific DocumentID
"""
temp = cls.get(DocumentID=doc_id)
for k in new_values.keys():
temp.__dict__['_data'][k] = new_values[k]
temp.save()
@classmethod
def get_list(cls,
rows_number,
page,
active=0,
search={}): # TODO : rework arguments
"""Returns a content from certain page of document list
Active - active=1, Not active - active=-1, All - active=0
Search query structure :
{'Field name': ('Your query', 'strict')}, where strict, means
strict search for strings for True and otherwise for False.
For other types it could be either True or False
"""
select_query = None
if (active == 0):
select_query = cls.select()
select_query = cls.search(select_query, search)
query = select_query.order_by(
cls.title.asc()).offset(0 + (page - 1) *
rows_number).limit(rows_number)
elif (active == 1):
select_query = cls.select().where(cls.active == True)
select_query = cls.search(select_query, search)
query = select_query.order_by(
cls.title.asc()).offset(0 + (page - 1) *
rows_number).limit(rows_number)
elif (active == -1):
select_query = cls.select().where(cls.active == False)
select_query = cls.search(select_query, search)
query = select_query.order_by(
cls.title.asc()).offset(0 + (page - 1) *
rows_number).limit(rows_number)
else:
return ([], 0)
res = []
for entry in query:
res.append(entry)
# Counting number of pages
page_number = int(select_query.count()) // rows_number
if (select_query.count() % rows_number > 0):
page_number += 1
return res, page_number
@classmethod
def search(cls, query, search):
"""Accorfing to fields and values in search dictionary,
use additional conditions on the query
"""
fields = cls._get_fields_dict_raw()
for key in search.keys():
search_query = search[key][0]
strict = search[key][1]
if key in fields.keys():
field = fields[key]
if (isinstance(field, pw.IntegerField)
or isinstance(field, pw.BigIntegerField)):
query = query.where(field == int(search_query))
elif (isinstance(field, pw.CharField)
or isinstance(field, pw.TextField)):
if key is 'keywords': #TODO : Find better solution if possible
for keyword in search_query:
search_string = '%' + str(keyword).strip() + '%'
query = query.where(field**search_string)
else:
if strict:
query = query.where(
field == str(search_query).strip())
else:
search_string = '%' + str(
search_query).strip() + '%'
query = query.where(
fields[key]**str(search_string).strip())
elif (isinstance(field, pw.BooleanField)):
query = query.where(field == bool(search_query))
return query
@classmethod
def get_fields(cls):
"""Returns list of fields of specific document type
"""
temp = {**Document.__dict__, **cls.__dict__}
temp.pop('__doc__')
temp.pop('__module__')
res = []
for key in temp.keys():
if (isinstance(temp[key], pw.FieldDescriptor)):
res.append(key)
return res
@classmethod
def _get_fields_dict_raw(cls):
"""Returns dictionary with field name as a key and peewee type of the field as a value
"""
temp = {**Document.__dict__, **cls.__dict__}
temp.pop('__doc__')
temp.pop('__module__')
res = {}
for key in temp.keys():
if (isinstance(temp[key], pw.FieldDescriptor)):
res[key] = temp[key].field
return res
@classmethod
def get_fields_dict(cls):
"""Returns dictionary with field name as a key and type of the field as a value
"""
map_to_types = {
pw.IntegerField: int,
pw.CharField: str,
pw.TextField: str
}
temp = {**Document.__dict__, **cls.__dict__}
temp.pop('__doc__')
temp.pop('__module__')
res = {}
for key in temp.keys():
if (isinstance(temp[key], pw.FieldDescriptor)):
if (isinstance(temp[key].field, pw.IntegerField)
or isinstance(temp[key].field, pw.BigIntegerField)):
res[key] = int
elif (isinstance(temp[key].field, pw.CharField)
or isinstance(temp[key].field, pw.TextField)):
res[key] = str
elif (isinstance(temp[key].field, pw.BooleanField)):
res[key] = bool
elif (isinstance(temp[key].field, pw.ForeignKeyField)):
res[key] = 'foreign'
return res
class Dashboard(ModelTimestampsMixin, BaseModel, BelongsToOrgMixin):
id = peewee.PrimaryKeyField()
org = peewee.ForeignKeyField(Organization, related_name="dashboards")
slug = peewee.CharField(max_length=140, index=True)
name = peewee.CharField(max_length=100)
user = peewee.ForeignKeyField(User)
layout = peewee.TextField()
dashboard_filters_enabled = peewee.BooleanField(default=False)
is_archived = peewee.BooleanField(default=False, index=True)
class Meta:
db_table = 'dashboards'
def to_dict(self, with_widgets=False, user=None):
layout = json.loads(self.layout)
if with_widgets:
widget_list = Widget.select(Widget, Visualization, Query, User)\
.where(Widget.dashboard == self.id)\
.join(Visualization, join_type=peewee.JOIN_LEFT_OUTER)\
.join(Query, join_type=peewee.JOIN_LEFT_OUTER)\
.join(User, join_type=peewee.JOIN_LEFT_OUTER)
widgets = {}
for w in widget_list:
if w.visualization_id is None:
widgets[w.id] = w.to_dict()
elif user and has_access(w.visualization.query.groups, user, view_only):
widgets[w.id] = w.to_dict()
else:
widgets[w.id] = project(w.to_dict(),
('id', 'width', 'dashboard_id', 'options', 'created_at', 'updated_at'))
widgets[w.id]['restricted'] = True
# The following is a workaround for cases when the widget object gets deleted without the dashboard layout
# updated. This happens for users with old databases that didn't have a foreign key relationship between
# visualizations and widgets.
# It's temporary until better solution is implemented (we probably should move the position information
# to the widget).
widgets_layout = []
for row in layout:
new_row = []
for widget_id in row:
widget = widgets.get(widget_id, None)
if widget:
new_row.append(widget)
widgets_layout.append(new_row)
else:
widgets_layout = None
return {
'id': self.id,
'slug': self.slug,
'name': self.name,
'user_id': self.user_id,
'layout': layout,
'dashboard_filters_enabled': self.dashboard_filters_enabled,
'widgets': widgets_layout,
'is_archived': self.is_archived,
'updated_at': self.updated_at,
'created_at': self.created_at
}
@classmethod
def all(cls, org, groups, user_id):
query = cls.select().\
join(Widget, peewee.JOIN_LEFT_OUTER, on=(Dashboard.id == Widget.dashboard)). \
join(Visualization, peewee.JOIN_LEFT_OUTER, on=(Widget.visualization == Visualization.id)). \
join(Query, peewee.JOIN_LEFT_OUTER, on=(Visualization.query == Query.id)). \
join(DataSourceGroup, peewee.JOIN_LEFT_OUTER, on=(Query.data_source == DataSourceGroup.data_source)). \
where(Dashboard.is_archived == False). \
where((DataSourceGroup.group << groups) |
(Dashboard.user == user_id) |
(~(Widget.dashboard >> None) & (Widget.visualization >> None))). \
where(Dashboard.org == org). \
group_by(Dashboard.id)
return query
@classmethod
def recent(cls, org, groups, user_id, for_user=False, limit=20):
query = cls.select().where(Event.created_at > peewee.SQL("current_date - 7")). \
join(Event, peewee.JOIN_LEFT_OUTER, on=(Dashboard.id == Event.object_id.cast('integer'))). \
join(Widget, peewee.JOIN_LEFT_OUTER, on=(Dashboard.id == Widget.dashboard)). \
join(Visualization, peewee.JOIN_LEFT_OUTER, on=(Widget.visualization == Visualization.id)). \
join(Query, peewee.JOIN_LEFT_OUTER, on=(Visualization.query == Query.id)). \
join(DataSourceGroup, peewee.JOIN_LEFT_OUTER, on=(Query.data_source == DataSourceGroup.data_source)). \
where(Event.action << ('edit', 'view')). \
where(~(Event.object_id >> None)). \
where(Event.object_type == 'dashboard'). \
where(Dashboard.is_archived == False). \
where(Dashboard.org == org). \
where((DataSourceGroup.group << groups) |
(Dashboard.user == user_id) |
(~(Widget.dashboard >> None) & (Widget.visualization >> None))). \
group_by(Event.object_id, Dashboard.id). \
order_by(peewee.SQL("count(0) desc"))
if for_user:
query = query.where(Event.user == user_id)
query = query.limit(limit)
return query
@classmethod
def get_by_slug_and_org(cls, slug, org):
return cls.get(cls.slug == slug, cls.org==org)
def save(self, *args, **kwargs):
if not self.slug:
self.slug = utils.slugify(self.name)
tries = 1
while self.select().where(Dashboard.slug == self.slug).first() is not None:
self.slug = utils.slugify(self.name) + "_{0}".format(tries)
tries += 1
super(Dashboard, self).save(*args, **kwargs)
def __unicode__(self):
return u"%s=%s" % (self.id, self.name)
class Gift(BaseModel):
gift_id = pw.PrimaryKeyField(verbose_name='礼物ID')
product_id = pw.IntegerField(verbose_name='物品ID')
credit_type = pw.IntegerField(choices=CREDIT_TYPE, verbose_name='价格类型')
credit_value = pw.IntegerField(verbose_name='价格数值')
per_piece_limit = pw.CharField(verbose_name='单用户各档位:大厅id(1:id1,10:id2)')
order_num = pw.IntegerField(verbose_name='显示顺序')
os = pw.CharField(max_length=64, verbose_name='平台要求')
version_code_max = pw.CharField(max_length=64,
verbose_name='版本要求(版本号小于等于)')
version_code_mix = pw.CharField(max_length=64,
verbose_name='版本要求(版本号大于等于)')
on_sale = pw.BooleanField(default=True, verbose_name='是否在售')
class Meta:
db_table = 'gift'
constraints = [pw.Check('credit_value > 0')]
@classmethod
@util.cached_object(ALL_GIFT_KEY)
def _load_all_gifts(cls):
gifts = list(cls.select())
gifts = [model_to_dict(gf) for gf in gifts]
return gifts
@classmethod
def get_all_gifts(cls):
gifts = Redis.get(ALL_GIFT_KEY)
if not gifts:
gifts = cls._load_all_gifts()
else:
gifts = cjson.loads(gifts)
gifts = [dict_to_model(Gift, gf) for gf in gifts]
return gifts
@classmethod
@util.cached_object(ONSALE_GIFT_KEY)
def _load_onsale_gifts(cls):
gifts = list(cls.select().where(Gift.on_sale == True))
gifts = [model_to_dict(gf) for gf in gifts]
return gifts
@classmethod
def get_onsale_gifts(cls):
gifts = Redis.get(ONSALE_GIFT_KEY)
if not gifts:
gifts = cls._load_onsale_gifts()
else:
gifts = cjson.loads(gifts)
gifts = [dict_to_model(Gift, gf) for gf in gifts]
return gifts
@classmethod
def get_free_gifts(cls):
gifts = filter(lambda x: x.credit_type == const.DAILY_FREE,
cls.get_onsale_gifts())
return gifts
@classmethod
def get_gift(cls, gift_id):
gifts = filter(lambda x: x.gift_id == gift_id, cls.get_onsale_gifts())
return gifts[0] if gifts else None
@classmethod
def get_gift_by_product_id(cls, product_id):
gifts = filter(lambda x: x.product_id == product_id,
cls.get_all_gifts())
return gifts[0] if gifts else None
def format(self):
data = model_to_dict(self)
if self.credit_type == const.MONEY and data.get("per_piece_limit"):
data['per_piece_id'] = {
int(k_v.split(":")[0]): k_v.split(":")[1]
for k_v in self.per_piece_limit.split(",") if k_v
}
product = Product.get_product(self.product_id)
if product:
data.update(product.format())
return data
@property
def gold_price(self):
if self.credit_type == const.DAILY_FREE:
return 100
elif self.credit_type == const.SALE_GOLD:
return self.credit_value
elif self.credit_type == const.SALE_GEM:
return self.credit_value * 100
elif self.credit_type == const.MONEY:
return self.credit_value
return 0
def send_to_user(self, from_user, to_user, num, gift_from, from_id,
**kwargs):
if num < 1:
return error.InvalidArguments
uc = from_user_uc = UserCredit.get_or_create_user_credit(from_user)
product = Product.get_product(self.product_id)
if self.credit_type == const.SALE_GOLD:
total_gold = self.credit_value * num
if uc.gold < total_gold:
return error.GiftError('你的游米不足，做任务可获取游米')
with MYDB.atomic():
uc.reduce_gold(total_gold, const.GIFT)
product.add_product2user(to_user, num, const.GIFT)
UserGiftLog.create(user_id=to_user,
from_user=from_user,
product_id=self.product_id,
credit_type=self.credit_type,
credit_value=self.credit_value,
num=num,
gold_price=self.gold_price * num,
gift_from=gift_from,
from_id=from_id,
send_success=1)
elif self.credit_type == const.SALE_GEM:
total_gem = self.credit_value * num
if uc.gem < total_gem:
return error.GiftError('你的游票不足')
with MYDB.atomic():
uc.reduce_gem(total_gem, const.GIFT)
product.add_product2user(to_user, num, const.GIFT)
UserGiftLog.create(user_id=to_user,
from_user=from_user,
product_id=self.product_id,
credit_type=self.credit_type,
credit_value=self.credit_value,
num=num,
gold_price=self.gold_price * num,
gift_from=gift_from,
from_id=from_id,
send_success=1)
elif self.credit_type == const.DAILY_FREE:
from_up = UserProduct.get_or_create_user_product(
from_user, self.product_id)
if from_up.gift_free < num:
return error.GiftError('对不起，您今天的免费礼物已用完')
with MYDB.atomic():
from_up.gift_free -= num
from_up.save()
product.add_product2user(to_user, num, const.GIFT)
UserGiftLog.create(user_id=to_user,
from_user=from_user,
product_id=self.product_id,
credit_type=self.credit_type,
credit_value=self.credit_value,
num=num,
gold_price=self.gold_price * num,
gift_from=gift_from,
from_id=from_id,
send_success=1)
elif self.credit_type == const.MONEY:
# 请求支付
with MYDB.atomic():
# uc.reduce_gem(total_gem, const.GIFT)
#
send_success = kwargs.get("send_success", 0)
if not send_success:
UserGiftLog.create(
user_id=to_user,
from_user=from_user,
product_id=self.product_id,
credit_type=self.credit_type,
credit_value=self.credit_value,
num=num,
gold_price=self.gold_price * num,
gift_from=gift_from,
from_id=from_id,
gift_id=self.gift_id,
send_success=0,
transaction_id=kwargs.get("transactionId"))
else:
total_money = self.gold_price * num
from_user_uc.add_cost_money(total_money)
to_user_uc = UserCredit.get_or_create_user_credit(to_user)
to_user_uc.add_get_money(total_money)
# 状态为1
log = UserGiftLog.get_by_transaction_id(
kwargs.get("transactionId"))
if not log:
return error.GiftError('未发现支付记录')
log.send_success = 1
log.save()
# redis 更新
UserGiftLog.update_redis_gift(log)
product.add_product2user(to_user,
num,
const.GIFT,
extra={"is_money": True})
return True
class User(ModelTimestampsMixin, BaseModel, UserMixin, PermissionsCheckMixin):
DEFAULT_GROUPS = ['default']
id = peewee.PrimaryKeyField()
name = peewee.CharField(max_length=320)
email = peewee.CharField(max_length=320, index=True, unique=True)
password_hash = peewee.CharField(max_length=128, null=True)
groups = ArrayField(peewee.CharField, default=DEFAULT_GROUPS)
api_key = peewee.CharField(max_length=40, unique=True)
class Meta:
db_table = 'users'
def to_dict(self):
return {
'id': self.id,
'name': self.name,
'email': self.email,
'gravatar_url': self.gravatar_url,
'updated_at': self.updated_at,
'created_at': self.created_at
}
def __init__(self, *args, **kwargs):
super(User, self).__init__(*args, **kwargs)
self._allowed_tables = None
def pre_save(self, created):
super(User, self).pre_save(created)
if not self.api_key:
self.api_key = generate_token(40)
@property
def gravatar_url(self):
email_md5 = hashlib.md5(self.email.lower()).hexdigest()
return "https://www.gravatar.com/avatar/%s?s=40" % email_md5
@property
def permissions(self):
# TODO: this should be cached.
return list(
itertools.chain(*[
g.permissions
for g in Group.select().where(Group.name << self.groups)
]))
@property
def allowed_tables(self):
# TODO: cache this as weel
if self._allowed_tables is None:
self._allowed_tables = set([
t.lower() for t in itertools.chain(*[
g.tables
for g in Group.select().where(Group.name << self.groups)
])
])
return self._allowed_tables
@classmethod
def get_by_email(cls, email):
return cls.get(cls.email == email)
@classmethod
def get_by_api_key(cls, api_key):
return cls.get(cls.api_key == api_key)
def __unicode__(self):
return u'%s (%s)' % (self.name, self.email)
def hash_password(self, password):
self.password_hash = pwd_context.encrypt(password)
def verify_password(self, password):
return self.password_hash and pwd_context.verify(
password, self.password_hash)
class QueryResult(BaseModel):
id = peewee.PrimaryKeyField()
data_source = peewee.ForeignKeyField(DataSource)
query_hash = peewee.CharField(max_length=32, index=True)
query = peewee.TextField()
data = peewee.TextField()
runtime = peewee.FloatField()
retrieved_at = DateTimeTZField()
class Meta:
db_table = 'query_results'
def to_dict(self):
return {
'id': self.id,
'query_hash': self.query_hash,
'query': self.query,
'data': json.loads(self.data),
'data_source_id': self._data.get('data_source', None),
'runtime': self.runtime,
'retrieved_at': self.retrieved_at
}
@classmethod
def unused(cls):
week_ago = datetime.datetime.now() - datetime.timedelta(days=7)
unused_results = cls.select().where(Query.id == None, cls.retrieved_at < week_ago)\
.join(Query, join_type=peewee.JOIN_LEFT_OUTER)
return unused_results
@classmethod
def get_latest(cls, data_source, query, max_age=0):
query_hash = utils.gen_query_hash(query)
if max_age == -1:
query = cls.select().where(
cls.query_hash == query_hash,
cls.data_source == data_source).order_by(
cls.retrieved_at.desc())
else:
query = cls.select().where(
cls.query_hash == query_hash, cls.data_source == data_source,
peewee.SQL(
"retrieved_at + interval '%s second' >= now() at time zone 'utc'",
max_age)).order_by(cls.retrieved_at.desc())
return query.first()
@classmethod
def store_result(cls, data_source_id, query_hash, query, data, run_time,
retrieved_at):
query_result = cls.create(query_hash=query_hash,
query=query,
runtime=run_time,
data_source=data_source_id,
retrieved_at=retrieved_at,
data=data)
logging.info("Inserted query (%s) data; id=%s", query_hash,
query_result.id)
updated_count = Query.update(latest_query_data=query_result).\
where(Query.query_hash==query_hash, Query.data_source==data_source_id).\
execute()
logging.info("Updated %s queries with result (%s).", updated_count,
query_hash)
return query_result
def __unicode__(self):
return u"%d | %s | %s" % (self.id, self.query_hash, self.retrieved_at)
class User_has_buy(MySQLModel):
id = peewee.PrimaryKeyField()
User = peewee.ForeignKeyField(rel_model=User, to_field=User.id)
Buy = peewee.ForeignKeyField(rel_model=Buy, to_field=Buy.id)
class Dashboard(ModelTimestampsMixin, BaseModel):
id = peewee.PrimaryKeyField()
slug = peewee.CharField(max_length=140, index=True)
name = peewee.CharField(max_length=100)
user_email = peewee.CharField(max_length=360, null=True)
user = peewee.ForeignKeyField(User)
layout = peewee.TextField()
dashboard_filters_enabled = peewee.BooleanField(default=False)
is_archived = peewee.BooleanField(default=False, index=True)
class Meta:
db_table = 'dashboards'
def to_dict(self, with_widgets=False):
layout = json.loads(self.layout)
if with_widgets:
widgets = Widget.select(Widget, Visualization, Query, User)\
.where(Widget.dashboard == self.id)\
.join(Visualization, join_type=peewee.JOIN_LEFT_OUTER)\
.join(Query, join_type=peewee.JOIN_LEFT_OUTER)\
.join(User, join_type=peewee.JOIN_LEFT_OUTER)
widgets = {w.id: w.to_dict() for w in widgets}
# The following is a workaround for cases when the widget object gets deleted without the dashboard layout
# updated. This happens for users with old databases that didn't have a foreign key relationship between
# visualizations and widgets.
# It's temporary until better solution is implemented (we probably should move the position information
# to the widget).
widgets_layout = []
for row in layout:
new_row = []
for widget_id in row:
widget = widgets.get(widget_id, None)
if widget:
new_row.append(widget)
widgets_layout.append(new_row)
# widgets_layout = map(lambda row: map(lambda widget_id: widgets.get(widget_id, None), row), layout)
else:
widgets_layout = None
return {
'id': self.id,
'slug': self.slug,
'name': self.name,
'user_id': self._data['user'],
'layout': layout,
'dashboard_filters_enabled': self.dashboard_filters_enabled,
'widgets': widgets_layout,
'updated_at': self.updated_at,
'created_at': self.created_at
}
@classmethod
def get_by_slug(cls, slug):
return cls.get(cls.slug == slug)
@classmethod
def recent(cls, user_id):
return cls.select().where(Event.created_at > peewee.SQL("current_date - 7")). \
join(Event, on=(Dashboard.id == peewee.SQL("t2.object_id::integer"))). \
where(Event.action << ('edit', 'view')).\
where(Event.user == user_id). \
where(~(Event.object_id >> None)). \
where(Event.object_type == 'dashboard'). \
group_by(Event.object_id, Dashboard.id). \
order_by(peewee.SQL("count(0) desc"))
def save(self, *args, **kwargs):
if not self.slug:
self.slug = utils.slugify(self.name)
tries = 1
while self.select().where(
Dashboard.slug == self.slug).first() is not None:
self.slug = utils.slugify(self.name) + "_{0}".format(tries)
tries += 1
super(Dashboard, self).save(*args, **kwargs)
def __unicode__(self):
return u"%s=%s" % (self.id, self.name)
class Admin(MySQLModel):
id = peewee.PrimaryKeyField()
name = peewee.CharField()
user = peewee.CharField()
password = peewee.CharField()
class UserOrder(BaseModel):
order_id = pw.PrimaryKeyField(verbose_name='订单ID')
user_id = pw.CharField(max_length=64, verbose_name='用户ID')
item_id = pw.IntegerField(verbose_name='条目ID')
store_id = pw.IntegerField(verbose_name='商店ID')
store_type = pw.IntegerField(choices=STORE_TYPE, verbose_name='商店类型')
campaign_id = pw.CharField(default='',
max_length=512,
verbose_name='营销平台活动ID')
title = pw.CharField(default=512, verbose_name='商品描述')
product_id = pw.IntegerField(verbose_name='物品ID')
product_num = pw.IntegerField(verbose_name='物品数量')
status = pw.IntegerField(choices=STATUS, verbose_name='状态')
result = pw.CharField(default='', max_length=1024, verbose_name='营销平台结果')
recid = pw.CharField(null=True, max_length=128, verbose_name='营销平台订单号')
user_ip = pw.CharField(default=None,
max_length=16,
verbose_name='用户抽奖ip地址')
class Meta:
db_table = 'user_order'
@classmethod
def get_order(cls, order_id):
try:
order = cls.get(cls.order_id == int(order_id))
except cls.DoesNotExist:
order = None
return order
@classmethod
def get_user_store_orders(cls, user_id, store_id, page, pagesize):
orders = list(cls.select().where(
cls.user_id == user_id, cls.store_id == int(store_id)).order_by(
cls.create_at.desc()).paginate(page, pagesize))
return orders
@classmethod
def get_orders_in_hand(cls):
orders = list(cls.select().where(cls.campaign_id != '',
cls.status == const.ORDER_IN_HAND,
~(cls.recid >> None)))
return orders
def format(self):
product = Product.get_product(self.product_id)
result = json.loads(self.result) if self.result else None
rid = result['id'] if result and 'id' in result else None
code = result['giftCode'] if result and 'giftCode' in result else None
name = result['name'] if result and 'name' in result else None
data = {
'order_id': self.order_id,
'title': self.title,
'product': product and product.format(),
'status': self.status,
'create_at': util.datetime2timestamp(self.create_at),
'result': {
'code': code,
'name': name,
'rid': rid
}
}
return data
class Note(MySQLModel):
id = peewee.PrimaryKeyField()
date = peewee.DateField()
title = peewee.CharField()
text = peewee.CharField()
User = peewee.ForeignKeyField(rel_model=User, to_field=User.id)
class Store(BaseModel):
store_id = pw.PrimaryKeyField(verbose_name='商店ID')
store_type = pw.IntegerField(choices=STORE_TYPE, verbose_name='商店类型')
title = pw.CharField(max_length=512, verbose_name='商店描述')
credit_type = pw.IntegerField(choices=CREDIT_TYPE, verbose_name='参与价格类型')
credit_value = pw.IntegerField(default=0, verbose_name='参与价格数值')
status = pw.IntegerField(choices=STORE_STATUS, verbose_name='状态')
begin_at = pw.DateTimeField(formats='%Y-%m-%d %H:%M:%S',
verbose_name='开始时间')
expire_at = pw.DateTimeField(formats='%Y-%m-%d %H:%M:%S',
verbose_name='结束时间')
action = pw.CharField(default='', max_length=512, verbose_name='跳转链接')
order = pw.IntegerField(verbose_name='显示顺序')
campaign_id = pw.CharField(default='',
max_length=512,
verbose_name='营销平台活动ID')
resource_campaign_id = pw.CharField(default='',
max_length=512,
verbose_name='营销平台流量话费发放活动ID')
share_image = pw.CharField(default='', max_length=512, verbose_name='分享图片')
share_title = pw.CharField(default='', max_length=512, verbose_name='分享标题')
yxmember = pw.BooleanField(default=False, verbose_name='是否为会员活动')
lotterynum = pw.IntegerField(default=1, verbose_name='每人抽奖次数')
class Meta:
db_table = 'store'
@classmethod
def all_stores_for_admin(cls):
stores = list(cls.select())
store_list = []
for s in stores:
type_desc = util.get_choices_desc(STORE_TYPE, s.store_type)
value = u'【%s(%s)】%s' % (type_desc, s.store_id, s.title)
store_list.append((s.store_id, value))
return store_list
@classmethod
def get_all_stores(cls):
stores = Redis.get(ALL_STORE_KEY)
if stores:
stores = cjson.loads(stores)
stores = [dict_to_model(Store, s) for s in stores]
else:
stores = list(cls.select())
_stores = [model_to_dict(s) for s in stores]
Redis.setex(ALL_STORE_KEY, 86400, cjson.dumps(_stores, 2))
return stores
@classmethod
def get_store(cls, store_id):
stores = filter(lambda x: x.store_id == int(store_id),
cls.get_all_stores())
return stores[0] if stores else None
@classmethod
def get_lotteries(cls):
lotteries = filter(lambda x: x.store_type == LOTTERY and x.online(),
cls.get_all_stores())
return sorted(lotteries, key=lambda x: x.order)
@classmethod
def get_exchanges(cls):
exchanges = filter(lambda x: x.store_type == EXCHANGE and x.online(),
cls.get_all_stores())
return sorted(exchanges, key=lambda x: x.order)
def format(self):
_share_uri = '/page/html/sharelottery.html?store_id=%s' % (
self.store_id)
share_url = urljoin(app.config.get('SHARE_URL'), _share_uri)
share_image = ''
if self.share_image:
share_image = urljoin(app.config.get("MEDIA_URL"),
self.share_image)
data = {
'store_id': self.store_id,
'title': self.title,
'credit_type': self.credit_type,
'credit_value': self.credit_value,
'status': self.status,
'action': self.action,
'share_url': share_url,
'share_image': share_image,
'share_title': self.share_title,
'expire_at': util.datetime2timestamp(self.expire_at)
}
return data
def online(self):
if self.status == 3:
return False
# 还未到上线时间或已到下线时间
if self.begin_at > datetime.now() or self.expire_at < datetime.now():
return False
return True
def pause(self):
if self.online() and self.status == 2:
return True
return False
class MS(peewee.Model):
class Meta:
database = database
ordering = ("ms_id", )
table_name = "vilnerabilities_ms"
id = peewee.PrimaryKeyField(null=False)
published_date = peewee.DateTimeField(default=datetime.now,
verbose_name="Published date")
cve_number = peewee.TextField(default="")
cve_url = peewee.TextField(default="")
name = peewee.TextField(default="")
platform = peewee.TextField(default="")
family = peewee.TextField(default="")
impact_id = peewee.TextField(default="")
impact = peewee.TextField(default="")
severity_id = peewee.TextField(default="")
severity = peewee.TextField(default="")
knowledge_base_id = peewee.TextField(default="")
knowledge_base_url = peewee.TextField(default="")
monthly_knowledge_base_id = peewee.TextField(default="")
monthly_knowledge_base_url = peewee.TextField(default="")
download_url1 = peewee.TextField(default="")
download_title1 = peewee.TextField(default="")
download_url2 = peewee.TextField(default="")
download_title2 = peewee.TextField(default="")
download_url3 = peewee.TextField(default="")
download_title3 = peewee.TextField(default="")
download_url4 = peewee.TextField(default="")
download_title4 = peewee.TextField(default="")
article_title1 = peewee.TextField(default="")
article_url1 = peewee.TextField(default="")
article_title2 = peewee.TextField(default="")
article_url2 = peewee.TextField(default="")
article_title3 = peewee.TextField(default="")
article_url3 = peewee.TextField(default="")
article_title4 = peewee.TextField(default="")
article_url4 = peewee.TextField(default="")
def __unicode__(self):
return "ms"
def __str__(self):
return str(self.cve_number)
@property
def to_json(self):
return dict(
id=self.id,
published_date=self.published_date,
cve_number=self.cve_number,
cve_url=self.cve_url,
name=self.name,
platform=self.platform,
family=self.family,
impact_id=self.impact_id,
impact=self.impact,
severity_id=self.severity_id,
severity=self.severity,
knowledge_base_id=self.knowledge_base_id,
knowledge_base_url=self.knowledge_base_url,
monthly_knowledge_base_id=self.monthly_knowledge_base_id,
monthly_knowledge_base_url=self.monthly_knowledge_base_url,
download_url1=self.download_url1,
download_title1=self.download_title1,
download_url2=self.download_url2,
download_title2=self.download_title2,
download_url3=self.download_url3,
download_title3=self.download_title3,
download_url4=self.download_url4,
download_title4=self.download_title4,
article_title1=self.article_title1,
article_url1=self.article_url1,
article_title2=self.article_title2,
article_url2=self.article_url2,
article_title3=self.article_title3,
article_url3=self.article_url3,
article_title4=self.article_title4,
article_url4=self.article_url4,
)