def get_all_stock_major_data(stock_ids=get_stockID_twse('sii').tolist() +
get_stockID_twse('otc').tolist()):
all_data_df_list = []
error_stock_ids = []
#stock_ids = stock_ids[:10]
while len(stock_ids) > 0:
print "len(stock_ids) = ", len(stock_ids)
one_stock_id = stock_ids.pop()
try:
one_stock_data = get_stock_major_data(one_stock_id)
all_data_df_list.append(one_stock_data["data"])
print "%s -- SUCCESS" % one_stock_id
except Exception as e:
print "%s -- ERROR" % one_stock_id
print e
print traceback.format_exc()
#print e.__dict__
#print e.message
#print pickle.dumps(e.request)
error_stock_ids.append({
"stockId": one_stock_id,
"message": e.message,
"traceback": traceback.format_exc()
})
return {"data": pd.concat(all_data_df_list), "error": error_stock_ids}
def get_all_stock_major_data(stock_ids = get_stockID_twse('sii').tolist() + get_stockID_twse('otc').tolist()):
all_data_df_list = []
error_stock_ids = []
#stock_ids = stock_ids[:10]
while len(stock_ids) > 0:
print "len(stock_ids) = ",len(stock_ids)
one_stock_id = stock_ids.pop()
try:
one_stock_data = get_stock_major_data(one_stock_id)
all_data_df_list.append(one_stock_data["data"])
print "%s -- SUCCESS" % one_stock_id
except Exception as e:
print "%s -- ERROR" % one_stock_id
print e
print traceback.format_exc()
#print e.__dict__
#print e.message
#print pickle.dumps(e.request)
error_stock_ids.append({"stockId":one_stock_id,
"message":e.message,
"traceback":traceback.format_exc()})
return {"data":pd.concat(all_data_df_list), "error":error_stock_ids}
def update_all_stock_major_data(data_collection, error_collection):
twse_ids = get_stockID_twse('sii').tolist()
twotc_ids = get_stockID_twse('otc').tolist()
stock_ids = twse_ids + twotc_ids
#stock_ids = stock_ids[:10]
counter = 0
len_stock_ids = len(stock_ids)
for one_stock_id in stock_ids:
print "~~~~~~~~~~~~~~~~~~~~~~~~"
print "%s/%s = " % (counter, len_stock_ids)
counter = counter + 1
#one_stock_id = stock_ids.pop()
try:
one_stock_data = get_stock_major_data(one_stock_id)
one_stock_data_in_db = data_collection.find(
{
"stockId": one_stock_data["stockId"],
"dateString": one_stock_data["date"]
}).count() > 0
if not one_stock_data_in_db:
data_collection.insert(
one_stock_data["data"].to_dict(outtype="record"))
print "%s -- SUCCESS: insert into db " % one_stock_id
else:
print "%s -- SUCCESS: already in db " % one_stock_id
except Exception as e:
print "%s -- ERROR" % one_stock_id
print e
print traceback.format_exc()
#print e.__dict__
#print e.message
#print pickle.dumps(e.request)
try:
error_collection.insert({
"stockId": one_stock_id,
"message": e.message,
"traceback": traceback.format_exc(),
"time": datetime.datetime.now()
})
except:
error_collection.insert({
"stockId": one_stock_id,
"traceback": traceback.format_exc(),
"time": datetime.datetime.now()
})
def update_all_stock_major_data(data_collection, error_collection):
twse_ids = get_stockID_twse('sii').tolist()
twotc_ids = get_stockID_twse('otc').tolist()
stock_ids = twse_ids + twotc_ids
#stock_ids = stock_ids[:10]
counter = 0
len_stock_ids = len(stock_ids)
for one_stock_id in stock_ids:
print "~~~~~~~~~~~~~~~~~~~~~~~~"
print "%s/%s = " % (counter,len_stock_ids)
counter = counter + 1
#one_stock_id = stock_ids.pop()
try:
one_stock_data = get_stock_major_data(one_stock_id)
one_stock_data_in_db = data_collection.find({"stockId":one_stock_data["stockId"],"dateString":one_stock_data["date"]}).count()>0
if not one_stock_data_in_db:
data_collection.insert(one_stock_data["data"].to_dict(outtype="record"))
print "%s -- SUCCESS: insert into db " % one_stock_id
else:
print "%s -- SUCCESS: already in db " % one_stock_id
except Exception as e:
print "%s -- ERROR" % one_stock_id
print e
print traceback.format_exc()
#print e.__dict__
#print e.message
#print pickle.dumps(e.request)
try:
error_collection.insert({"stockId":one_stock_id,
"message":e.message,
"traceback":traceback.format_exc(),
"time":datetime.datetime.now()})
except:
error_collection.insert({"stockId":one_stock_id,
"traceback":traceback.format_exc(),
"time":datetime.datetime.now()})