class Position(DocType):
# 证券id
securityId = Keyword()
# 持有数量
amount = Long()
# 可交易数量
availableAmount = Long()
# 盈亏
profit = Float()
# 市值
value = Float()
# 成本价
cost = Float()
# 交易类型(0代表T+0,1代表T+1)
tradingT = Short()
def __init__(self, meta=None, security_id=None, trading_t=1, **kwargs):
super().__init__(meta, **kwargs)
self.securityId = security_id
self.availableAmount = 0
self.amount = 0
self.profit = 0
self.value = 0
self.cost = 0
self.tradingT = trading_t
class PriceSubscription(BaseDocType):
"""
{
"id": 123,
"userId": 111,
"type": "price",
"securityType": "cryptocurrency",
"exchange": "binance",
"code": "BTC-USDT",
"upPct": 1,
"downPct": 2,
"up": 7000,
"down": 6000,
"actions": ["weixin", "email", "shortMsg"],
"repeat": False
}
"""
id = Keyword()
userId = Keyword()
securityType = Keyword()
exchange = Keyword()
code = Keyword()
upPct = Float()
downPct = Float()
up = Float()
down = Float()
repeat = Boolean()
actions = Keyword()
class Meta:
doc_type = 'doc'
all = MetaField(enabled=False)
class EventPlace(DocType):
place_id = Text(fields={'raw': Keyword()})
provider_id = Text(fields={'raw': Keyword()})
provider = Text()
date_added = Date()
slug = String(fields={'raw': Keyword()})
title = Text(fields={'raw': Keyword()}, analyzer='russian')
address = Text(fields={'raw': Keyword()}, analyzer='russian')
city = Text(fields={'raw': Keyword()}, analyzer='russian')
description = Text()
lat = Float()
lng = Float()
geometry = GeoPoint()
email = Text()
website = Text()
phone = Text()
class Meta:
doc_type = 'places'
index = 'place-index'
def save(self, **kwargs):
if not self.date_added:
self.date_added = datetime.now()
return super(EventPlace, self).save(**kwargs)
class Account(DocType):
traderId = Keyword()
cash = Float()
positions = Nested()
allValue = Float()
timestamp = Date()
tradingClose = Boolean()
def copy_for_save(self, trading_close):
account = Account()
account.cash = self.cash
account.traderId = self.traderId
account.allValue = self.allValue
account.positions = self.positions
account.timestamp = account.timestamp
account.tradingClose = trading_close
return account
def save(self, using=None, index=None, validate=True, **kwargs):
self.meta.id = "{}_{}".format(
self.traderId, self.timestamp.strftime('%Y-%m-%d %H:%M:%S'))
return super().save(using, index, validate, **kwargs)
class Meta:
doc_type = 'doc'
all = MetaField(enabled=False)
class PositionType(DocType):
"""
学习视频类型
"""
# 搜索建议的mapping设置
suggest = Completion(analyzer=ik_analyzer)
url_object_id = Keyword()
url = Keyword()
position_name = Text(analyzer='ik_max_word')
salary_min = Float()
salary_max = Float()
welfare = Keyword()
working_place = Keyword()
working_exp = Keyword()
education = Keyword()
abstract = Text(analyzer='ik_max_word')
data_source = Keyword()
company_name = Text(analyzer='ik_max_word')
# 用来初始化index以及type的名称
class Meta:
# 类比到mysql就是数据库名
index = 'position_message'
# 类比到mysql就是表名
doc_type = 'position'
class Indicator(DocType):
indicator = Keyword()
indicator_ipv4 = Ip()
indicator_ipv4_mask = Integer()
indicator_ipv6 = Keyword()
indicator_ipv6_mask = Integer()
group = Keyword()
itype = Keyword()
tlp = Keyword()
provider = Keyword()
portlist = Text()
asn = Float()
asn_desc = Text()
cc = Text(fields={'raw': Keyword()})
protocol = Text(fields={'raw': Keyword()})
reporttime = Date()
lasttime = Date()
firsttime = Date()
confidence = Float()
timezone = Text()
city = Text(fields={'raw': Keyword()})
description = Keyword()
tags = Keyword(multi=True, fields={'raw': Keyword()})
rdata = Keyword()
count = Integer()
message = Text(multi=True)
location = GeoPoint()
class VideoType(DocType):
"""
学习视频类型
"""
# 搜索建议的mapping设置
suggest = Completion(analyzer=ik_analyzer)
url_object_id = Keyword()
url = Keyword()
class_name = Text(analyzer='ik_max_word')
price = Float()
learner_nums = Integer()
class_nums = Integer()
category = Keyword()
abstract = Text(analyzer='ik_max_word')
data_source = Keyword()
institution = Keyword()
second_classify = Text(analyzer='ik_max_word')
evaluation_score = Float()
evaluation_person = Integer()
fit_people = Text(analyzer='ik_max_word')
first_classify = Text(analyzer='ik_max_word')
evaluation_content = Keyword()
# 用来初始化index以及type的名称
class Meta:
# 类比就是mysql的数据库名称
index = 'learning_video'
# 类比就是mysql的表名称
doc_type = 'video'
class Page(Document):
pdf_name = Text(fields={'raw': Keyword()})
img_pth = Text(fields={'raw': Keyword()})
page_num = Integer()
dataset_id = Text(fields={'raw': Keyword()})
postprocess_cls = Text(fields={'raw': Keyword()}, multi=True)
postprocess_score = Float(multi=True)
detect_cls = Text(fields={'raw': Keyword()}, multi=True)
detect_score = Float(multi=True)
pdf_dims = Float(multi=True)
# arrays of arrays. not sure how to handle initializing properly, so I'll just let ES do it on ingest
# bbox = Integer(multi=True)
# classes = Text(multi=True)
# scores = Float(multi=True)
def get_id(self):
'''
Elasticsearch ingest process would be greatly improved by having a unique ID per object.
TODO: is this actually unique and deterministic?
'''
return hashlib.sha1(
f"{self.pdf_name}_{self.page_num}".encode('utf-8')).hexdigest()
class Index:
name = 'page'
settings = {'number_of_shards': 1, 'number_of_replicas': 0}
class Products(DocType):
about = Text(fields={"keyword": Keyword()}, required=True)
brand = Text(fields={"keyword": Keyword()}, required=True)
care = Text(fields={"keyword": Keyword()}, required=True)
code = Text(fields={"keyword": Keyword()}, required=True)
details = Text(fields={"keyword": Keyword()}, required=True)
gender = Text(fields={"keyword": Keyword()}, required=True)
images = Text(fields={"keyword": Keyword()}, required=True)
kind = Text(fields={"keyword": Keyword()}, required=True)
link = Text(fields={"keyword": Keyword()}, required=True)
name = Text(fields={"keyword": Keyword()}, required=True)
price = Nested(
properties={
"currency": Text(fields={'keyword': Keyword()}, required=True),
"outlet": Float(required=True),
"retail": Float(required=True)
})
sessionid = Text(fields={"keyword": Keyword()}, required=True)
sessionname = Text(fields={"keyword": Keyword()}, required=True)
storename = Text(fields={"keyword": Keyword()}, required=True)
class Meta:
index = "store"
type = "products"
doc_type = "products"
class JobType(DocType):
# 搜索建议字段
# Completion 用来做搜索建议的类型
# 不能直接指定分词器名称，需要指定一个自定义对象
suggest = Completion(analyzer=ik_analyzer)
job_name = Text(analyzer='ik_max_word')
job_money = Text()
max_money = Float()
min_money = Float()
job_date = Text()
company_name = Text(analyzer='ik_max_word')
job_place = Text(analyzer='ik_max_word')
job_city = Text()
job_area = Text(analyzer='ik_max_word')
job_education = Text()
job_fuli = Text(analyzer='ik_max_word')
job_from = Text()
job_type = Text(analyzer='ik_max_word')
job_detail_href = Text()
class Meta:
# index 索引名（数据库）
index = 'jobs'
# doc_type 类型（表名称）
doc_type = 'job'
class Indicator(Document):
class Index:
name = 'indicators-*'
indicator = Keyword()
indicator_ipv4 = Ip()
indicator_ipv4_mask = Integer()
indicator_ipv6 = Ip()
indicator_ipv6_mask = Integer()
group = Keyword()
itype = Keyword()
tlp = Keyword()
provider = Keyword()
portlist = Text()
asn = Float()
asn_desc = Text()
cc = Text(fields={'raw': Keyword()})
protocol = Text(fields={'raw': Keyword()})
confidence = Integer()
timezone = Text()
city = Text(fields={'raw': Keyword()})
description = Keyword()
tags = Keyword(multi=True, fields={'raw': Keyword()})
rdata = Keyword()
count = Integer()
location = GeoPoint()
region = Keyword()
latitude = Float()
longitude = Float()
ns = Keyword()
mx = Keyword()
reported_at = Date()
last_at = Date()
first_at = Date()
created_at = Date()
class SDGeo(Document):
"""Document structure for the sd_geo index"""
location = GeoPoint()
number = Integer
street = Text()
latitude = Float()
longitude = Float()
def mapping():
return {
'year': Integer(),
'percentile_trr': Float(),
'percentile_allegation': Float(),
'percentile_allegation_internal': Float(),
'percentile_allegation_civilian': Float(),
}
class Entity(Document):
id = Integer()
name = Keyword()
cityId = Keyword()
wardId = Integer()
level = Integer()
menuId = Keyword()
rb_pin = GeoPoint()
lat = Float()
lng = Float()
total = Integer()
type = Keyword()
data = Text()
wardName = Keyword()
cityName = Keyword()
address = Keyword()
jurisdiction = Text()
icon = Keyword()
impact = Integer()
category = Keyword()
subcategory = Keyword()
closed_at = Date()
closed_by = Keyword()
phone = Keyword()
created_by = Keyword()
creator_org = Keyword()
persons = Nested(Person)
class Index:
name = 'rb_locations'
settings = {"number_of_shards": 2}
def mprint(self):
print("Org: ", self.meta.id, self.name, self.cityId, self.wardId,
self.level, self.menuId, self.lat, self.lng, self.total,
self.type, self.data, self.wardName, self.cityName, self.address,
self.jurisdiction, self.impact, self.category, self.subcategory,
self.closed_at, self.closed_by, self.phone, self.created_by,
self.creator_org)
for person in self.persons:
person.mprint()
def __eq__(self, other):
if self.id != other.id or self.name != other.name or self.cityId != other.cityId or self.wardId != other.wardId or self.level != other.level or self.menuId != other.menuId or self.total != other.total or self.type != other.type or self.data != other.data or self.wardName != other.wardName or self.cityName != other.cityName or self.address != other.address or self.jurisdiction != other.jurisdiction or self.icon != other.icon or self.impact != other.impact or self.category != other.category or self.subcategory != other.subcategory or self.closed_by != other.closed_by or self.closed_at != other.closed_at or self.phone != other.phone or self.created_by != other.created_by or self.creator_org != other.creator_org:
return False
if abs(self.lat - float(other.lat)) > math.pow(
10, -8) or abs(self.lng - float(other.lng)) > math.pow(10, -8):
return False
if len(self.persons) != len(other.persons):
return False
for i, person in enumerate(self.persons):
if not person == other.persons[i]:
return False
return True
class PerFinTransaction(Document):
created_at = get_date()
category = Keyword()
aggregates = Keyword()
account_name = Keyword()
account_type = Keyword()
amount = Float()
description = Text(fielddata=True)
original = Keyword()
check_num = Long()
date = get_date()
posted_date = get_date()
trans_date = get_date()
trans_type = Keyword()
credit = Float()
debit = Float()
key = Keyword()
memo = Text(fielddata=True)
class Index:
name = "transactions"
@staticmethod
def create(**kwargs):
doc = kwargs["doc"]
doc_key = kwargs["doc_key"]
doc_type = kwargs["doc_type"]
aggregates = make_key(doc.get("description") or "")
transaction = PerFinTransaction(
account_name=doc_key,
account_type=doc_type,
aggregates=aggregates,
description=doc.get("description"),
trans_date=doc.get("transaction_date"),
posted_date=doc.get("transaction_posted_date"),
card_num=doc.get("card_num"),
category=doc.get("category"),
original=doc.get("original"),
debit=doc.get("debit"),
credit=doc.get("credit"),
trans_type=doc.get("transaction_type"),
amount=doc.get("amount"),
memo=doc.get("memo"),
check_num=doc.get("check_num"),
)
if not transaction.date:
transaction.date = transaction.posted_date
transaction.save()
def save(self, **kwargs):
self.created_at = datetime.utcnow()
self.meta["id"] = generate_specific_key(self.description,
dfmt(self.date),
str(self.amount))
logger.info(f"inserting {self.__dict__}")
return super().save(**kwargs)
class StockTickItem(BaseDocType):
securityId = Keyword()
code = Keyword()
timestamp = Date()
price = Float()
change = Float()
direction = Keyword()
volume = Float()
turnover = Float()
class EosAccount(BaseDocType):
id = Keyword()
timestamp = Date()
updateTimestamp = Date()
userId = Keyword()
liquidEos = Float()
stackedEos = Float()
totalEos = Float()
unstackingEos = Float()
class StoryIndex(DocType):
id = Integer()
title = Text(analyzer=my_analyzer, search_analyzer='standard')
author = Text()
score = Float()
link = Text()
timestamp = Text()
created_at = Date()
sentiment_score = Float()
class Reply(Document):
timestamp = Date(required=True)
post_id = Keyword(required=True)
post_timestamp = Date(required=True)
message = Text(analyzer="german", required=True)
tags = Keyword(multi=True)
post_message = Text(analyzer="german", required=True)
post_tags = Keyword(multi=True)
color = Keyword(required=True)
post_color = Keyword(required=True)
distance = Integer(required=True)
got_thanks = Boolean(required=True)
location_name = Keyword(required=True)
from_home = Boolean()
vote_count = Integer(required=True)
replier = Integer(required=True)
polarity = Float(required=True)
post_pin_count = Integer(required=True)
post_share_count = Integer(required=True)
post_vote_count = Integer(required=True)
post_polarity = Float(required=True)
class Index:
name = "jodel-reply-*-*"
settings = {"number_of_shards": 1}
def from_dict(reply, post):
blob = TextBlobDE(reply["message"])
return Reply(_id=reply["post_id"],
timestamp=dateutil.parser.parse(reply["created_at"]),
post_id=reply["parent_id"],
post_timestamp=post.timestamp,
message=reply["message"],
tags=extract_tags(reply["message"]),
post_message=post.message,
post_tags=post.tags,
color=reply["color"],
post_color=post.color,
distance=reply["distance"],
got_thanks=reply["got_thanks"],
location_name=reply["location"]["name"],
from_home=reply.get("from_home"),
vote_count=reply["vote_count"],
replier=reply["replier"],
polarity=blob.polarity,
post_pin_count=post.pin_count,
post_share_count=post.share_count,
post_vote_count=post.vote_count,
post_polarity=post.polarity)
def save(self, **kwargs):
time_format = "jodel-reply-" + index_name(self.location_name)\
+ "-%Y%m%d"
kwargs["index"] = self.timestamp.strftime(time_format)
return super().save(**kwargs)
class StoryIndex(DocType):
id = Integer()
abstract = Text(
fields={
'pattern': field.Text(analyzer=pattern_analyzer),
'english': field.Text(analyzer='english')
})
category__gn_description = Text()
distribution_description = Text()
distribution_url = Keyword()
owner__username = Keyword(fields={'text': field.Text()})
popular_count = Integer()
share_count = Integer()
rating = Integer()
thumbnail_url = Keyword()
detail_url = Keyword()
uuid = Keyword()
title = Text(
fields={
'pattern': field.Text(analyzer=pattern_analyzer),
'english': field.Text(analyzer='english')
})
date = Date()
type = Keyword(fields={
'text': field.Text(),
'english': field.Text(analyzer='english')
})
title_sortable = Keyword()
category = Keyword(fields={
'text': field.Text(),
'english': field.Text(analyzer='english')
})
bbox_left = Float()
bbox_right = Float()
bbox_bottom = Float()
bbox_top = Float()
temporal_extent_start = Date()
temporal_extent_end = Date()
keywords = Keyword(fields={
'text': field.Text(),
'english': field.Text(analyzer='english')
})
regions = Keyword(fields={
'text': field.Text(),
'english': field.Text(analyzer='english')
})
num_ratings = Integer()
num_comments = Integer()
num_chapters = Integer()
owner__first_name = Text()
owner__last_name = Text()
is_published = Boolean()
featured = Boolean()
class Meta:
index = 'story-index'
class DishIndex(DocType):
restaurant_name = Text()
restaurant_id = Text()
name = Text()
price = Float()
description = Text()
rating = Float()
class Meta:
index = 'dish-index'
class Comment(InnerDoc):
author = Text()
content = Text()
created_at = Date()
price_min = Float()
price_max = Float()
def age(self):
return datetime.now() - self.created_at
class DataToBeIndex(DocType):
field_1_int = Integer()
field_2_char = Text()
field_3_float = Float()
field_4_float = Float()
field_5_char = Text()
field_6_int = Integer()
class Index:
name = 'datatobe-index'
class Extension(DocType):
name = String()
url = String()
description = String()
user_count = Integer()
review_count = Float()
review_score = Float()
class Meta:
index = 'exts'
class ProductsIndex(DocType):
id = Integer()
sku = Keyword()
name = Text()
attribute_set_id = Integer()
price = Integer()
status = Integer()
visibility = Integer()
type_id = Text()
created_at = Date()
updated_at = Date()
extension_attributes = Long()
product_links = Long()
tier_prices = Long()
custom_attributes = Long()
category = Object(properties={
'category_id': Long(),
'name': Text(),
})
description = Text()
image = Text()
small_image = Text()
thumbnail = Text()
options_container = Text()
required_options = Integer()
has_options = Integer()
url_key = Text()
tax_class_id = Integer()
children_data = Nested()
configurable_options = Object()
configurable_children = Object()
category_ids = Long()
stock = Object(properties={'is_in_stock': Boolean()})
special_price = Float()
new = Integer()
sale = Integer()
special_from_date = Date()
special_to_date = Date()
priceInclTax = Float()
originalPriceInclTax = Float()
originalPrice = Float()
specialPriceInclTax = Float()
sgn = Text()
class Index:
name = 'vue_storefront_catalog'
doc_type = 'product'
class Meta:
doc_type = 'product'
class Order(Document):
customer = customer_field
created_at = Date()
data = Object(properties={
'total_amount': Float(),
'cost': Float(),
'margin': Float(),
})
class Index:
name = 'py-orders'
class Message(DocType):
text = String()
community = String(fields={'raw': String(index='not_analyzed')})
date = Date()
score = Integer()
positive = Float()
negative = Float()
neutral = Float()
def save(self, **kwargs):
return super(Message, self).save(**kwargs)
class Order(DocType):
customer = customer_field
created_at = Date()
data = Object(properties={
'total_amount': Float(),
'cost': Float(),
'margin': Float(),
})
class Meta:
index = 'py-orders'
class MapIndex(DocType):
id = Integer()
abstract = Text(
fields={
'pattern': field.Text(analyzer=pattern_analyzer),
'english': field.Text(analyzer='english')
})
category__gn_description = Text()
csw_type = Keyword()
csw_wkt_geometry = Keyword()
detail_url = Keyword()
owner__username = Keyword(fields={'text': field.Text()})
popular_count = Integer()
share_count = Integer()
rating = Integer()
srid = Keyword()
supplemental_information = Text()
thumbnail_url = Keyword()
uuid = Keyword()
title = Text(
fields={
'pattern': field.Text(analyzer=pattern_analyzer),
'english': field.Text(analyzer='english')
})
date = Date()
type = Keyword(fields={
'text': field.Text(),
'english': field.Text(analyzer='english')
})
title_sortable = Keyword()
category = Keyword(fields={
'text': field.Text(),
'english': field.Text(analyzer='english')
})
bbox_left = Float()
bbox_right = Float()
bbox_bottom = Float()
bbox_top = Float()
temporal_extent_start = Date()
temporal_extent_end = Date()
keywords = Keyword(fields={
'text': field.Text(),
'english': field.Text(analyzer='english')
})
regions = Keyword(fields={
'text': field.Text(),
'english': field.Text(analyzer='english')
})
num_ratings = Integer()
num_comments = Integer()
class Meta:
index = 'map-index'
class TokenTopHistoryType(DocType):
symbol = Keyword()
name = Keyword()
address = Keyword()
quantity = Float()
rank = Integer()
percentage = Float()
timestamp = Date()
transaction = Date()
last_transaction = Date()
class Meta:
index = "token_top_history_2"
doc_type = "token_top"
