def crawler_history():
date_name = 'ExchangeRate'
CER = CrawlerExchangeRate()
CER.main()
C2S = BasedClass.Crawler2SQL(date_name, 'Financial_DataSet')
try:
C2S.create_table(CER.data.columns, text_col=['country'])
except:
123
C2S.upload2sql(CER.data, no_float_col=['date', 'country'])
print('create process table')
BasedClass.create_datatable('ExchangeRate')
def crawler_history():
CCOP = CrawlerCrudeOilPrices()
CCOP.main()
#CII.data
C2S = BasedClass.Crawler2SQL('CrudeOilPrices', 'Financial_DataSet')
try:
C2S.create_table(CCOP.data.columns)
except:
123
C2S.upload2sql(CCOP.data)
print('create process table')
BasedClass.create_datatable('CrudeOilPrices')
def crawler_history():
date_name = 'GovernmentBonds'
self = CrawlerGovernmentBonds()
self.main()
C2S = BasedClass.Crawler2SQL(date_name, 'Financial_DataSet')
try:
C2S.create_table(self.data.columns,
text_col=['data_name', 'country', 'curr_id'])
except:
123
C2S.upload2sql(self.data,
no_float_col=['Date', 'data_name', 'country', 'curr_id'])
print('create process table')
BasedClass.create_datatable(date_name)
def crawler_history():
CII = CrawlerInstitutionalInvestors()
CII.main()
#CII.data
C2S = BasedClass.Crawler2SQL('InstitutionalInvestors', 'Financial_DataSet')
try:
C2S.create_table(CII.data.columns)
except:
123
C2S.upload2sql(CII.data)
print('create process table')
BasedClass.create_datatable('InstitutionalInvestors')
def crawler_history():
CIR = CrawlerInterestRate()
CIR.main()
C2S = BasedClass.Crawler2SQL('InterestRate', 'Financial_DataSet')
try:
C2S.create_table(CIR.data.columns,
text_col=['country', 'full_country_name'])
except:
123
C2S.upload2sql(CIR.data,
no_float_col=['country', 'full_country_name', 'date'])
print('create process table')
BasedClass.create_datatable('InterestRate')
def crawler_history():
CTD = CrawlerStockDividend()
CTD.main()
C2S = BasedClass.Crawler2SQL('StockDividend','Financial_DataSet')
try:
C2S.create_table()
except:
123
for i in range(len(CTD.url_set)):
print(str(i)+'/'+str(len(CTD.url_set)))#i=0
data = CTD.get_value(i)
C2S.upload2sql(data,
no_float_col = ['meeting_data',
'Ex_right_trading_day',
'Ex_dividend_transaction_day',
'stock_id'])
print('create process table')
BasedClass.create_datatable('StockDividend')
def crawler_history():
CFS = CrawlerFinancialStatements()
CFS.crawler()
CFS.fix()
CFS.stock_financial_statements['year'] = CFS.stock_financial_statements['year'] + 1911
C2S = BasedClass.Crawler2SQL('FinancialStatements','Financial_DataSet')
try:
C2S.create_table()
except:
123
C2S.upload2sql(CFS.stock_financial_statements,
no_float_col = ['stock_id','url'],
int_col = ['year','quar'])
print('create process table')
BasedClass.create_datatable('FinancialStatements')
def crawler_history():
date_name = 'GoldPrice'
# get hostory by download https://www.gold.org/data/gold-price file
file_path = '/home/' + path + '/github/FinancialMining/CrawlerCode/'
data = pd.read_csv(file_path + 'glod.csv',skiprows = 1)
date = [ datetime.datetime.strptime(d,'%Y/%m/%d').date() for d in data.date ]
data = data[ [ d < datetime.datetime.strptime('2018-1-1','%Y-%m-%d').date() for d in date ] ]
data['date'] = [ d.replace('/','-') + ' 00:00:00' for d in data.date ]
data.columns = ['datetime','Price']
C2S = BasedClass.Crawler2SQL(date_name,'Financial_DataSet')
try:
C2S.create_table(data.columns,dt_col = ['datetime'])
except:
123
C2S.upload2sql( data )
print('create process table')
BasedClass.create_datatable(date_name)
def crawler_history():
dataset_name = 'StockPrice'
self = CrawlerHistoryStockPrice()
#self.main()
os.chdir('/home/linsam/github')
self.data = pd.read_csv('test.csv')
print('crawler data and upload 2 sql')
C2S = BasedClass.Crawler2SQL(dataset_name, 'Financial_DataSet')
try:
C2S.create_table(self.data.columns,
text_col=['stock'],
BIGINT_col=['Volume'])
except:
123
C2S.upload2sql(self.data,
no_float_col=['date', 'stock'],
int_col=['Volume'])
print('create process table')
BasedClass.create_datatable('StockPrice')