class Position(DocType): # 证券id securityId = Keyword() # 持有数量 amount = Long() # 可交易数量 availableAmount = Long() # 盈亏 profit = Float() # 市值 value = Float() # 成本价 cost = Float() # 交易类型(0代表T+0,1代表T+1) tradingT = Short() def __init__(self, meta=None, security_id=None, trading_t=1, **kwargs): super().__init__(meta, **kwargs) self.securityId = security_id self.availableAmount = 0 self.amount = 0 self.profit = 0 self.value = 0 self.cost = 0 self.tradingT = trading_t
class PriceSubscription(BaseDocType): """ { "id": 123, "userId": 111, "type": "price", "securityType": "cryptocurrency", "exchange": "binance", "code": "BTC-USDT", "upPct": 1, "downPct": 2, "up": 7000, "down": 6000, "actions": ["weixin", "email", "shortMsg"], "repeat": False } """ id = Keyword() userId = Keyword() securityType = Keyword() exchange = Keyword() code = Keyword() upPct = Float() downPct = Float() up = Float() down = Float() repeat = Boolean() actions = Keyword() class Meta: doc_type = 'doc' all = MetaField(enabled=False)
class EventPlace(DocType): place_id = Text(fields={'raw': Keyword()}) provider_id = Text(fields={'raw': Keyword()}) provider = Text() date_added = Date() slug = String(fields={'raw': Keyword()}) title = Text(fields={'raw': Keyword()}, analyzer='russian') address = Text(fields={'raw': Keyword()}, analyzer='russian') city = Text(fields={'raw': Keyword()}, analyzer='russian') description = Text() lat = Float() lng = Float() geometry = GeoPoint() email = Text() website = Text() phone = Text() class Meta: doc_type = 'places' index = 'place-index' def save(self, **kwargs): if not self.date_added: self.date_added = datetime.now() return super(EventPlace, self).save(**kwargs)
class Account(DocType): traderId = Keyword() cash = Float() positions = Nested() allValue = Float() timestamp = Date() tradingClose = Boolean() def copy_for_save(self, trading_close): account = Account() account.cash = self.cash account.traderId = self.traderId account.allValue = self.allValue account.positions = self.positions account.timestamp = account.timestamp account.tradingClose = trading_close return account def save(self, using=None, index=None, validate=True, **kwargs): self.meta.id = "{}_{}".format( self.traderId, self.timestamp.strftime('%Y-%m-%d %H:%M:%S')) return super().save(using, index, validate, **kwargs) class Meta: doc_type = 'doc' all = MetaField(enabled=False)
class PositionType(DocType): """ 学习视频类型 """ # 搜索建议的mapping设置 suggest = Completion(analyzer=ik_analyzer) url_object_id = Keyword() url = Keyword() position_name = Text(analyzer='ik_max_word') salary_min = Float() salary_max = Float() welfare = Keyword() working_place = Keyword() working_exp = Keyword() education = Keyword() abstract = Text(analyzer='ik_max_word') data_source = Keyword() company_name = Text(analyzer='ik_max_word') # 用来初始化index以及type的名称 class Meta: # 类比到mysql就是数据库名 index = 'position_message' # 类比到mysql就是表名 doc_type = 'position'
class Indicator(DocType): indicator = Keyword() indicator_ipv4 = Ip() indicator_ipv4_mask = Integer() indicator_ipv6 = Keyword() indicator_ipv6_mask = Integer() group = Keyword() itype = Keyword() tlp = Keyword() provider = Keyword() portlist = Text() asn = Float() asn_desc = Text() cc = Text(fields={'raw': Keyword()}) protocol = Text(fields={'raw': Keyword()}) reporttime = Date() lasttime = Date() firsttime = Date() confidence = Float() timezone = Text() city = Text(fields={'raw': Keyword()}) description = Keyword() tags = Keyword(multi=True, fields={'raw': Keyword()}) rdata = Keyword() count = Integer() message = Text(multi=True) location = GeoPoint()
class VideoType(DocType): """ 学习视频类型 """ # 搜索建议的mapping设置 suggest = Completion(analyzer=ik_analyzer) url_object_id = Keyword() url = Keyword() class_name = Text(analyzer='ik_max_word') price = Float() learner_nums = Integer() class_nums = Integer() category = Keyword() abstract = Text(analyzer='ik_max_word') data_source = Keyword() institution = Keyword() second_classify = Text(analyzer='ik_max_word') evaluation_score = Float() evaluation_person = Integer() fit_people = Text(analyzer='ik_max_word') first_classify = Text(analyzer='ik_max_word') evaluation_content = Keyword() # 用来初始化index以及type的名称 class Meta: # 类比就是mysql的数据库名称 index = 'learning_video' # 类比就是mysql的表名称 doc_type = 'video'
class Page(Document): pdf_name = Text(fields={'raw': Keyword()}) img_pth = Text(fields={'raw': Keyword()}) page_num = Integer() dataset_id = Text(fields={'raw': Keyword()}) postprocess_cls = Text(fields={'raw': Keyword()}, multi=True) postprocess_score = Float(multi=True) detect_cls = Text(fields={'raw': Keyword()}, multi=True) detect_score = Float(multi=True) pdf_dims = Float(multi=True) # arrays of arrays. not sure how to handle initializing properly, so I'll just let ES do it on ingest # bbox = Integer(multi=True) # classes = Text(multi=True) # scores = Float(multi=True) def get_id(self): ''' Elasticsearch ingest process would be greatly improved by having a unique ID per object. TODO: is this actually unique and deterministic? ''' return hashlib.sha1( f"{self.pdf_name}_{self.page_num}".encode('utf-8')).hexdigest() class Index: name = 'page' settings = {'number_of_shards': 1, 'number_of_replicas': 0}
class Products(DocType): about = Text(fields={"keyword": Keyword()}, required=True) brand = Text(fields={"keyword": Keyword()}, required=True) care = Text(fields={"keyword": Keyword()}, required=True) code = Text(fields={"keyword": Keyword()}, required=True) details = Text(fields={"keyword": Keyword()}, required=True) gender = Text(fields={"keyword": Keyword()}, required=True) images = Text(fields={"keyword": Keyword()}, required=True) kind = Text(fields={"keyword": Keyword()}, required=True) link = Text(fields={"keyword": Keyword()}, required=True) name = Text(fields={"keyword": Keyword()}, required=True) price = Nested( properties={ "currency": Text(fields={'keyword': Keyword()}, required=True), "outlet": Float(required=True), "retail": Float(required=True) }) sessionid = Text(fields={"keyword": Keyword()}, required=True) sessionname = Text(fields={"keyword": Keyword()}, required=True) storename = Text(fields={"keyword": Keyword()}, required=True) class Meta: index = "store" type = "products" doc_type = "products"
class JobType(DocType): # 搜索建议字段 # Completion 用来做搜索建议的类型 # 不能直接指定分词器名称,需要指定一个自定义对象 suggest = Completion(analyzer=ik_analyzer) job_name = Text(analyzer='ik_max_word') job_money = Text() max_money = Float() min_money = Float() job_date = Text() company_name = Text(analyzer='ik_max_word') job_place = Text(analyzer='ik_max_word') job_city = Text() job_area = Text(analyzer='ik_max_word') job_education = Text() job_fuli = Text(analyzer='ik_max_word') job_from = Text() job_type = Text(analyzer='ik_max_word') job_detail_href = Text() class Meta: # index 索引名(数据库) index = 'jobs' # doc_type 类型(表名称) doc_type = 'job'
class Indicator(Document): class Index: name = 'indicators-*' indicator = Keyword() indicator_ipv4 = Ip() indicator_ipv4_mask = Integer() indicator_ipv6 = Ip() indicator_ipv6_mask = Integer() group = Keyword() itype = Keyword() tlp = Keyword() provider = Keyword() portlist = Text() asn = Float() asn_desc = Text() cc = Text(fields={'raw': Keyword()}) protocol = Text(fields={'raw': Keyword()}) confidence = Integer() timezone = Text() city = Text(fields={'raw': Keyword()}) description = Keyword() tags = Keyword(multi=True, fields={'raw': Keyword()}) rdata = Keyword() count = Integer() location = GeoPoint() region = Keyword() latitude = Float() longitude = Float() ns = Keyword() mx = Keyword() reported_at = Date() last_at = Date() first_at = Date() created_at = Date()
class SDGeo(Document): """Document structure for the sd_geo index""" location = GeoPoint() number = Integer street = Text() latitude = Float() longitude = Float()
def mapping(): return { 'year': Integer(), 'percentile_trr': Float(), 'percentile_allegation': Float(), 'percentile_allegation_internal': Float(), 'percentile_allegation_civilian': Float(), }
class Entity(Document): id = Integer() name = Keyword() cityId = Keyword() wardId = Integer() level = Integer() menuId = Keyword() rb_pin = GeoPoint() lat = Float() lng = Float() total = Integer() type = Keyword() data = Text() wardName = Keyword() cityName = Keyword() address = Keyword() jurisdiction = Text() icon = Keyword() impact = Integer() category = Keyword() subcategory = Keyword() closed_at = Date() closed_by = Keyword() phone = Keyword() created_by = Keyword() creator_org = Keyword() persons = Nested(Person) class Index: name = 'rb_locations' settings = {"number_of_shards": 2} def mprint(self): print("Org: ", self.meta.id, self.name, self.cityId, self.wardId, self.level, self.menuId, self.lat, self.lng, self.total, self.type, self.data, self.wardName, self.cityName, self.address, self.jurisdiction, self.impact, self.category, self.subcategory, self.closed_at, self.closed_by, self.phone, self.created_by, self.creator_org) for person in self.persons: person.mprint() def __eq__(self, other): if self.id != other.id or self.name != other.name or self.cityId != other.cityId or self.wardId != other.wardId or self.level != other.level or self.menuId != other.menuId or self.total != other.total or self.type != other.type or self.data != other.data or self.wardName != other.wardName or self.cityName != other.cityName or self.address != other.address or self.jurisdiction != other.jurisdiction or self.icon != other.icon or self.impact != other.impact or self.category != other.category or self.subcategory != other.subcategory or self.closed_by != other.closed_by or self.closed_at != other.closed_at or self.phone != other.phone or self.created_by != other.created_by or self.creator_org != other.creator_org: return False if abs(self.lat - float(other.lat)) > math.pow( 10, -8) or abs(self.lng - float(other.lng)) > math.pow(10, -8): return False if len(self.persons) != len(other.persons): return False for i, person in enumerate(self.persons): if not person == other.persons[i]: return False return True
class PerFinTransaction(Document): created_at = get_date() category = Keyword() aggregates = Keyword() account_name = Keyword() account_type = Keyword() amount = Float() description = Text(fielddata=True) original = Keyword() check_num = Long() date = get_date() posted_date = get_date() trans_date = get_date() trans_type = Keyword() credit = Float() debit = Float() key = Keyword() memo = Text(fielddata=True) class Index: name = "transactions" @staticmethod def create(**kwargs): doc = kwargs["doc"] doc_key = kwargs["doc_key"] doc_type = kwargs["doc_type"] aggregates = make_key(doc.get("description") or "") transaction = PerFinTransaction( account_name=doc_key, account_type=doc_type, aggregates=aggregates, description=doc.get("description"), trans_date=doc.get("transaction_date"), posted_date=doc.get("transaction_posted_date"), card_num=doc.get("card_num"), category=doc.get("category"), original=doc.get("original"), debit=doc.get("debit"), credit=doc.get("credit"), trans_type=doc.get("transaction_type"), amount=doc.get("amount"), memo=doc.get("memo"), check_num=doc.get("check_num"), ) if not transaction.date: transaction.date = transaction.posted_date transaction.save() def save(self, **kwargs): self.created_at = datetime.utcnow() self.meta["id"] = generate_specific_key(self.description, dfmt(self.date), str(self.amount)) logger.info(f"inserting {self.__dict__}") return super().save(**kwargs)
class StockTickItem(BaseDocType): securityId = Keyword() code = Keyword() timestamp = Date() price = Float() change = Float() direction = Keyword() volume = Float() turnover = Float()
class EosAccount(BaseDocType): id = Keyword() timestamp = Date() updateTimestamp = Date() userId = Keyword() liquidEos = Float() stackedEos = Float() totalEos = Float() unstackingEos = Float()
class StoryIndex(DocType): id = Integer() title = Text(analyzer=my_analyzer, search_analyzer='standard') author = Text() score = Float() link = Text() timestamp = Text() created_at = Date() sentiment_score = Float()
class Reply(Document): timestamp = Date(required=True) post_id = Keyword(required=True) post_timestamp = Date(required=True) message = Text(analyzer="german", required=True) tags = Keyword(multi=True) post_message = Text(analyzer="german", required=True) post_tags = Keyword(multi=True) color = Keyword(required=True) post_color = Keyword(required=True) distance = Integer(required=True) got_thanks = Boolean(required=True) location_name = Keyword(required=True) from_home = Boolean() vote_count = Integer(required=True) replier = Integer(required=True) polarity = Float(required=True) post_pin_count = Integer(required=True) post_share_count = Integer(required=True) post_vote_count = Integer(required=True) post_polarity = Float(required=True) class Index: name = "jodel-reply-*-*" settings = {"number_of_shards": 1} def from_dict(reply, post): blob = TextBlobDE(reply["message"]) return Reply(_id=reply["post_id"], timestamp=dateutil.parser.parse(reply["created_at"]), post_id=reply["parent_id"], post_timestamp=post.timestamp, message=reply["message"], tags=extract_tags(reply["message"]), post_message=post.message, post_tags=post.tags, color=reply["color"], post_color=post.color, distance=reply["distance"], got_thanks=reply["got_thanks"], location_name=reply["location"]["name"], from_home=reply.get("from_home"), vote_count=reply["vote_count"], replier=reply["replier"], polarity=blob.polarity, post_pin_count=post.pin_count, post_share_count=post.share_count, post_vote_count=post.vote_count, post_polarity=post.polarity) def save(self, **kwargs): time_format = "jodel-reply-" + index_name(self.location_name)\ + "-%Y%m%d" kwargs["index"] = self.timestamp.strftime(time_format) return super().save(**kwargs)
class StoryIndex(DocType): id = Integer() abstract = Text( fields={ 'pattern': field.Text(analyzer=pattern_analyzer), 'english': field.Text(analyzer='english') }) category__gn_description = Text() distribution_description = Text() distribution_url = Keyword() owner__username = Keyword(fields={'text': field.Text()}) popular_count = Integer() share_count = Integer() rating = Integer() thumbnail_url = Keyword() detail_url = Keyword() uuid = Keyword() title = Text( fields={ 'pattern': field.Text(analyzer=pattern_analyzer), 'english': field.Text(analyzer='english') }) date = Date() type = Keyword(fields={ 'text': field.Text(), 'english': field.Text(analyzer='english') }) title_sortable = Keyword() category = Keyword(fields={ 'text': field.Text(), 'english': field.Text(analyzer='english') }) bbox_left = Float() bbox_right = Float() bbox_bottom = Float() bbox_top = Float() temporal_extent_start = Date() temporal_extent_end = Date() keywords = Keyword(fields={ 'text': field.Text(), 'english': field.Text(analyzer='english') }) regions = Keyword(fields={ 'text': field.Text(), 'english': field.Text(analyzer='english') }) num_ratings = Integer() num_comments = Integer() num_chapters = Integer() owner__first_name = Text() owner__last_name = Text() is_published = Boolean() featured = Boolean() class Meta: index = 'story-index'
class DishIndex(DocType): restaurant_name = Text() restaurant_id = Text() name = Text() price = Float() description = Text() rating = Float() class Meta: index = 'dish-index'
class Comment(InnerDoc): author = Text() content = Text() created_at = Date() price_min = Float() price_max = Float() def age(self): return datetime.now() - self.created_at
class DataToBeIndex(DocType): field_1_int = Integer() field_2_char = Text() field_3_float = Float() field_4_float = Float() field_5_char = Text() field_6_int = Integer() class Index: name = 'datatobe-index'
class Extension(DocType): name = String() url = String() description = String() user_count = Integer() review_count = Float() review_score = Float() class Meta: index = 'exts'
class ProductsIndex(DocType): id = Integer() sku = Keyword() name = Text() attribute_set_id = Integer() price = Integer() status = Integer() visibility = Integer() type_id = Text() created_at = Date() updated_at = Date() extension_attributes = Long() product_links = Long() tier_prices = Long() custom_attributes = Long() category = Object(properties={ 'category_id': Long(), 'name': Text(), }) description = Text() image = Text() small_image = Text() thumbnail = Text() options_container = Text() required_options = Integer() has_options = Integer() url_key = Text() tax_class_id = Integer() children_data = Nested() configurable_options = Object() configurable_children = Object() category_ids = Long() stock = Object(properties={'is_in_stock': Boolean()}) special_price = Float() new = Integer() sale = Integer() special_from_date = Date() special_to_date = Date() priceInclTax = Float() originalPriceInclTax = Float() originalPrice = Float() specialPriceInclTax = Float() sgn = Text() class Index: name = 'vue_storefront_catalog' doc_type = 'product' class Meta: doc_type = 'product'
class Order(Document): customer = customer_field created_at = Date() data = Object(properties={ 'total_amount': Float(), 'cost': Float(), 'margin': Float(), }) class Index: name = 'py-orders'
class Message(DocType): text = String() community = String(fields={'raw': String(index='not_analyzed')}) date = Date() score = Integer() positive = Float() negative = Float() neutral = Float() def save(self, **kwargs): return super(Message, self).save(**kwargs)
class Order(DocType): customer = customer_field created_at = Date() data = Object(properties={ 'total_amount': Float(), 'cost': Float(), 'margin': Float(), }) class Meta: index = 'py-orders'
class MapIndex(DocType): id = Integer() abstract = Text( fields={ 'pattern': field.Text(analyzer=pattern_analyzer), 'english': field.Text(analyzer='english') }) category__gn_description = Text() csw_type = Keyword() csw_wkt_geometry = Keyword() detail_url = Keyword() owner__username = Keyword(fields={'text': field.Text()}) popular_count = Integer() share_count = Integer() rating = Integer() srid = Keyword() supplemental_information = Text() thumbnail_url = Keyword() uuid = Keyword() title = Text( fields={ 'pattern': field.Text(analyzer=pattern_analyzer), 'english': field.Text(analyzer='english') }) date = Date() type = Keyword(fields={ 'text': field.Text(), 'english': field.Text(analyzer='english') }) title_sortable = Keyword() category = Keyword(fields={ 'text': field.Text(), 'english': field.Text(analyzer='english') }) bbox_left = Float() bbox_right = Float() bbox_bottom = Float() bbox_top = Float() temporal_extent_start = Date() temporal_extent_end = Date() keywords = Keyword(fields={ 'text': field.Text(), 'english': field.Text(analyzer='english') }) regions = Keyword(fields={ 'text': field.Text(), 'english': field.Text(analyzer='english') }) num_ratings = Integer() num_comments = Integer() class Meta: index = 'map-index'
class TokenTopHistoryType(DocType): symbol = Keyword() name = Keyword() address = Keyword() quantity = Float() rank = Integer() percentage = Float() timestamp = Date() transaction = Date() last_transaction = Date() class Meta: index = "token_top_history_2" doc_type = "token_top"