def run(self):
for category, url in self.category_map_url.items():
resp = requests.get(url)
results = json_callback_param(resp.text)
for result in results:
items = result.split(',')
code = items[1]
name = items[2]
id = 'index_cn_{}'.format(code)
if id in self.index_ids:
continue
self.session.add(
Index(id=id,
entity_id=id,
entity_type='index',
exchange='cn',
code=code,
name=name,
category=category.value))
self.session.commit()
indices = get_entities(session=self.session,
entity_type='index',
return_type='domain',
filters=[
Index.category.in_([
StockCategory.concept.value,
StockCategory.industry.value
])
],
provider=self.provider)
for index_item in indices:
resp = requests.get(
self.category_stocks_url.format(index_item.code, '1'))
try:
results = json_callback_param(resp.text)
the_list = []
for result in results:
items = result.split(',')
stock_code = items[1]
stock_id = china_stock_code_to_id(stock_code)
index_id = index_item.id
the_list.append({
'id': '{}_{}'.format(index_id, stock_id),
'index_id': index_id,
'stock_id': stock_id
})
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(data_schema=self.data_schema,
df=df,
provider=self.provider)
self.logger.info('finish recording index:{},{}'.format(
index_item.category, index_item.name))
except Exception as e:
self.logger.error("error:,resp.text:", e, resp.text)
self.sleep()
def run(self):
http_session = get_http_session()
for category, url in self.category_map_url.items():
resp = request_get(http_session, url)
results = json_callback_param(resp.text)
the_list = []
for result in results:
items = result.split(',')
code = items[1]
name = items[2]
entity_id = f'block_cn_{code}'
the_list.append({
'id': entity_id,
'entity_id': entity_id,
'entity_type': EntityType.Block.value,
'exchange': 'cn',
'code': code,
'name': name,
'category': category.value
})
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(df=df,
region=Region.CHN,
data_schema=self.data_schema,
provider=self.provider,
force_update=True)
self.logger.info(f"finish record sina blocks:{category.value}")
def record(self, entity, start, end, size, timestamps):
resp = requests.get(self.category_stocks_url.format(entity.code, '1'))
try:
results = json_callback_param(resp.text)
the_list = []
for result in results:
items = result.split(',')
stock_code = items[1]
stock_id = china_stock_code_to_id(stock_code)
block_id = entity.id
the_list.append({
'id': '{}_{}'.format(block_id, stock_id),
'entity_id': block_id,
'entity_type': 'block',
'exchange': entity.exchange,
'code': entity.code,
'name': entity.name,
'timestamp': now_pd_timestamp(),
'stock_id': stock_id,
'stock_code': stock_code,
'stock_name': items[2],
})
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(data_schema=self.data_schema, df=df, provider=self.provider, force_update=True)
self.logger.info('finish recording block:{},{}'.format(entity.category, entity.name))
except Exception as e:
self.logger.error("error:,resp.text:", e, resp.text)
self.sleep()
def record(self, entity, start, end, size, timestamps):
resp = requests.get(self.category_stocks_url.format(entity.code, "1"), headers=DEFAULT_HEADER)
try:
results = json_callback_param(resp.text)
the_list = []
for result in results:
items = result.split(",")
stock_code = items[1]
stock_id = china_stock_code_to_id(stock_code)
block_id = entity.id
the_list.append(
{
"id": "{}_{}".format(block_id, stock_id),
"entity_id": block_id,
"entity_type": "block",
"exchange": entity.exchange,
"code": entity.code,
"name": entity.name,
"timestamp": now_pd_timestamp(),
"stock_id": stock_id,
"stock_code": stock_code,
"stock_name": items[2],
}
)
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(data_schema=self.data_schema, df=df, provider=self.provider, force_update=True)
self.logger.info("finish recording block:{},{}".format(entity.category, entity.name))
except Exception as e:
self.logger.error("error:,resp.text:", e, resp.text)
self.sleep()
def run(self):
for category, url in self.category_map_url.items():
resp = requests.get(url, headers=DEFAULT_HEADER)
results = json_callback_param(resp.text)
the_list = []
for result in results:
items = result.split(",")
code = items[1]
name = items[2]
entity_id = f"block_cn_{code}"
the_list.append(
{
"id": entity_id,
"entity_id": entity_id,
"entity_type": "block",
"exchange": "cn",
"code": code,
"name": name,
"category": category.value,
}
)
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(data_schema=self.data_schema, df=df, provider=self.provider, force_update=self.force_update)
self.logger.info(f"finish record eastmoney blocks:{category.value}")
def record(self, entity, start, end, size, timestamps, http_session):
url = self.category_stocks_url.format(entity.code, '1')
text = sync_get(http_session, url, return_type='text')
if text is None:
return None
results = json_callback_param(text)
# @njit(nopython=True)
def numba_boost_up(results):
the_list = []
for result in results:
items = result.split(',')
stock_code = items[1]
stock_id = china_stock_code_to_id(stock_code)
the_list.append({
'stock_id': stock_id,
'stock_code': stock_code,
'stock_name': items[2],
})
return the_list
the_list = numba_boost_up(results)
if the_list:
df = pd.DataFrame.from_records(the_list)
return df
self.sleep()
return None
def run(self):
for category, url in self.category_map_url.items():
resp = requests.get(url, headers=DEFAULT_HEADER)
results = json_callback_param(resp.text)
the_list = []
for result in results:
items = result.split(',')
code = items[1]
name = items[2]
entity_id = f'block_cn_{code}'
the_list.append({
'id': entity_id,
'entity_id': entity_id,
'entity_type': 'block',
'exchange': 'cn',
'code': code,
'name': name,
'category': category.value
})
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(data_schema=self.data_schema,
df=df,
provider=self.provider,
force_update=self.force_update)
self.logger.info(f"finish record sina blocks:{category.value}")
def record(self, entity, start, end, size, time_array):
if entity.type == 'index':
id_flag = "{}1".format(entity.code)
elif entity.type == 'stock':
if entity.exchange == 'sh':
id_flag = "{}1".format(entity.code)
if entity.exchange == 'sz':
id_flag = "{}2".format(entity.code)
the_url = self.url.format("{}".format(id_flag),
eastmoney_map_zvt_trading_level(self.level),
now_time_str(fmt=TIME_FORMAT_MINUTE), size)
resp = requests.get(the_url)
results = json_callback_param(resp.text)
kdatas = []
for result in results:
the_timestamp = to_pd_timestamp(result['time'])
the_id = generate_kdata_id(entity_id=entity.id,
timestamp=the_timestamp,
level=self.level)
if not data_exist(self.session, self.kdata_schema, the_id):
kdatas.append(
self.kdata_schema(id=the_id,
timestamp=the_timestamp,
entity_id=entity.id,
code=entity.code,
name=entity.name,
level=self.level,
open=to_float(result['open']),
close=to_float(result['close']),
high=to_float(result['high']),
low=to_float(result['low']),
volume=to_float(result['volume']),
turnover=to_float(result['amount']),
turnover_rate=to_float(
result['turnoverrate'])))
return kdatas
def record(self, entity, start, end, size, timestamps):
the_url = self.url.format("{}".format(entity.code),
level_flag(self.level), size,
now_time_str(fmt=TIME_FORMAT_DAY1))
resp = requests.get(the_url)
results = json_callback_param(resp.text)
kdatas = []
if results:
klines = results['data']['klines']
# TODO: ignore the last unfinished kdata now,could control it better if need
for result in klines[:-1]:
# "2000-01-28,1005.26,1012.56,1173.12,982.13,3023326,3075552000.00"
# time,open,close,high,low,volume,turnover
fields = result.split(',')
the_timestamp = to_pd_timestamp(fields[0])
the_id = generate_kdata_id(entity_id=entity.id,
timestamp=the_timestamp,
level=self.level)
kdatas.append(
dict(id=the_id,
timestamp=the_timestamp,
entity_id=entity.id,
code=entity.code,
name=entity.name,
level=self.level.value,
open=to_float(fields[1]),
close=to_float(fields[2]),
high=to_float(fields[3]),
low=to_float(fields[4]),
volume=to_float(fields[5]),
turnover=to_float(fields[6])))
return kdatas
def process_loop(self, entity, http_session):
category, url = entity
text = sync_get(http_session, url, return_type='text')
if text is None:
return
results = json_callback_param(text)
@njit(nopython=True)
def numba_boost_up(results):
the_list = []
for result in results:
items = result.split(',')
code = items[1]
name = items[2]
entity_id = f'block_cn_{code}'
the_list.append({
'id': entity_id,
'entity_id': entity_id,
'entity_type': EntityType.Block.value,
'exchange': 'cn',
'code': code,
'name': name,
'category': category.value
})
return the_list
the_list = numba_boost_up(results)
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(df=df,
ref_df=None,
region=Region.CHN,
data_schema=self.data_schema,
provider=self.provider)
self.logger.info(f"finish record sina blocks:{category.value}")
def record(self, entity, start, end, size, timestamps, http_session):
url = self.url.format(
"{}".format(entity.code), level_flag(self.level), size,
now_time_str(region=Region.CHN, fmt=TIME_FORMAT_DAY1))
text = sync_get(http_session, url, return_type='text')
if text is None:
return None
results = json_callback_param(text)
if results:
klines = results['data']['klines']
@njit(nopython=True)
def numba_boost_up(klines):
kdatas = []
# TODO: ignore the last unfinished kdata now,could control it better if need
for result in klines[:-1]:
# "2000-01-28,1005.26,1012.56,1173.12,982.13,3023326,3075552000.00"
# time,open,close,high,low,volume,turnover
fields = result.split(',')
kdatas.append(
dict(timestamp=fields[0],
open=to_float(fields[1]),
close=to_float(fields[2]),
high=to_float(fields[3]),
low=to_float(fields[4]),
volume=to_float(fields[5]),
turnover=to_float(fields[6])))
return kdatas
kdatas = numba_boost_up(klines)
if len(kdatas) > 0:
df = pd.DataFrame.from_records(kdatas)
return df
return None
