def usa_stock_finance_to_es(force=False):
es_index_mapping('finance_summary', FinanceSummary)
for _, security_item in get_security_list(exchanges=['nasdaq'], codes=US_STOCK_CODES).iterrows():
try:
start_date = None
if not force:
query = {
"term": {"securityId": ""}
}
query["term"]["securityId"] = security_item["id"]
latest_record = es_get_latest_record(index='finance_summary', time_field='reportDate', query=query)
logger.info("latest_record:{}".format(latest_record))
if latest_record:
start_date = latest_record['reportDate']
actions = []
for _, json_object in get_finance_summary_items(security_item, start_date=start_date).iterrows():
if start_date and is_same_date(start_date, json_object['reportDate']):
continue
finance_summary = FinanceSummary(meta={'id': json_object['id']})
fill_doc_type(finance_summary, json_object.to_dict())
actions.append(finance_summary.to_dict(include_meta=True))
if actions:
resp = elasticsearch.helpers.bulk(es, actions)
logger.info(resp)
except Exception as e:
logger.warn("wrong FinanceSummary:{},error:{}", security_item, e)
def usa_stock_finance_to_es(force=False):
es_index_mapping('finance_summary', FinanceSummary)
for _, security_item in get_security_list(exchanges=['nasdaq'],
codes=US_STOCK_CODES).iterrows():
try:
start_date = None
if not force:
query = {"term": {"securityId": ""}}
query["term"]["securityId"] = security_item["id"]
latest_record = es_get_latest_record(index='finance_summary',
time_field='reportDate',
query=query)
logger.info("latest_record:{}".format(latest_record))
if latest_record:
start_date = latest_record['reportDate']
actions = []
for _, json_object in get_finance_summary_items(
security_item, start_date=start_date).iterrows():
if start_date and is_same_date(start_date,
json_object['reportDate']):
continue
finance_summary = FinanceSummary(
meta={'id': json_object['id']})
fill_doc_type(finance_summary, json_object.to_dict())
actions.append(finance_summary.to_dict(include_meta=True))
if actions:
resp = elasticsearch.helpers.bulk(es, actions)
logger.info(resp)
except Exception as e:
logger.warn("wrong FinanceSummary:{},error:{}", security_item, e)