os.chdir('C:/Users/Administrator/Desktop/daily/law')
while iter_flag == False:
if os.path.exists("complete_log" + str(law_url.split('/')[-2]) +
".txt"): # 如果是中途断点
with open("complete_log" + str(law_url.split('/')[-2]) + ".txt",
'r') as f:
params = f.read().split(',')
start_pag = int(params[0])
iter_flag, message = history_data_daily(law_url, start_pag)
else: # 如果是首次运行
iter_flag, message = history_data_daily(law_url, 1)
if message == '今日无增量':
with open('law' + today + '.txt', 'w', encoding='utf-8') as f:
f.write("今日中国人民银行-国家法律-无日增")
s3_function.upload_to_aws_s3('law' + today + '.txt', 's3qingdao',
'law' + today + '.txt')
os.remove('law' + today + '.txt')
else:
#压缩文件，上传到云
os.chdir('C:/Users/Administrator/Desktop/daily')
print(' has daily increment')
s3_function.zip_ya('law')
#upload
s3_function.upload_to_aws_s3('law.zip', 's3qingdao',
'law_daily' + today + '.zip')
except:
#创建错误日志
with open('FAIL_law' + today + '.txt', 'w', encoding='utf-8') as f:
f.write("今日中国人民银行-国家法律-日增任务启动失败")
s3_function.upload_to_aws_s3('FAIL_law' + today + '.txt', 's3qingdao',
'FAIL_law' + today + '.txt')
import rpa_daily_increment
import rpa_history_data_without_date
import rpa_history_data
import s3_function as upload_s3
import os
import datetime
try:
file_name = rpa_daily_increment.main()
flag = upload_s3.upload_to_aws_s3(file_name,'s3qingdao',file_name)
if flag == True:
os.remove(file_name)
except:
with open('FAIL_rpa_daily' + str((datetime.datetime.today()).date()) + '.txt', 'w', encoding='utf-8') as f:
f.write("今日理财日增数据拉取失败")
upload_s3.upload_to_aws_s3('FAIL_rpa_daily' + str((datetime.datetime.today()).date()) + '.txt', 's3qingdao',
'FAIL_rpa_daily' + str((datetime.datetime.today()).date()) + '.txt')
os.remove('FAIL_rpa_daily' + str((datetime.datetime.today()).date()) + '.txt')
#file_name = rpa_history_data_without_date.main()
#flag = upload_s3.upload_to_aws_s3(file_name,'storageforrpa',file_name)
#if flag == True:
# os.remove(file_name)
#
#input = '2020'
#file_name = rpa_history_data.main(input)
#upload_s3.upload_to_aws_s3(file_name,'storageforrpa',file_name)
#if flag == True:
# os.remove(file_name)
'returnType': returnType,
'predictMaxReturnRate': predictMaxReturnRate,
'realMaxReturnRate': realMaxReturnRate,
'comparisonWithDeposit': comparisonWithDeposit,
'startDateOfReturn': startDateOfReturn,
'endDateOfReturn': endDateOfReturn,
'returnDesc': returnDesc,
'purchaseCondition': purchaseCondition,
'iterminationCondition': iterminationCondition,
'redemptionCondition': redemptionCondition,
'investmentRisk': investmentRisk
})
df.to_csv(str_to_append + "product_detail.csv",
encoding='utf-8',
index=False)
t.close()
except:
str_to_append = str(datetime.datetime.today().date())
with open("FAIL_" + str_to_append + "product_detail.txt",
'w',
encoding='utf-8') as f:
f.write("今日理财详情页拉取失败")
s3_function.upload_to_aws_s3(
"FAIL_" + str_to_append + "product_detail.txt", 's3qingdao',
"FAIL_" + str_to_append + "product_detail.txt")
os.remove("FAIL_" + str_to_append + "product_detail.txt")
#上传数据
s3_function.upload_to_aws_s3(str_to_append + "product_detail.csv", 's3qingdao',
str_to_append + "product_detail.csv")
#删除数据
os.remove(str_to_append + "product_detail.csv")
if os.path.exists('baojianhui_log.txt'):
os.remove('baojianhui_log.txt') #删除之前的log
while not guizhang_flag:
try:
t.init()
guizhang_flag = main(url=guizhang_url, mode='txt')
t.close()
except Exception as e:
traceback.print_exc()
# print("i am wrong")
t.close()
if os.path.exists('baojianhui_log.txt'):
os.remove('baojianhui_log.txt') #运行成功之后删除log
os.chdir('C:/Users/Administrator/Desktop')
s3_function.zip_ya('baojian_guizhang')
s3_function.upload_to_aws_s3('baojian_guizhang.zip', 'storageforccbrpa',
'baojian_guizhang_history.zip')
# os.chdir('/Users/maoyuanq/Desktop')
# os.mkdir('baojian_falv')
# os.chdir('/Users/maoyuanq/Desktop/baojian_falv')
# falv_flag = False
# falv_url = 'http://www.cbirc.gov.cn/cn/view/pages/ItemList.html?itemPId=923&itemId=927&itemUrl=ItemListRightList.html&itemName=法律法规&itemsubPId=926#'
# if os.path.exists('baojianhui_log.txt'):
# os.remove('baojianhui_log.txt') #删除之前的log
# while not falv_flag:
# try:
# t.init()
# falv_flag = main(url=falv_url, mode = 'txt')
# t.close()
# except Exception as e:
# traceback.print_exc()
while not guizhang_flag:
try:
t.init()
guizhang_flag = main(url=guizhang_url, mode='txt')
t.close()
except Exception as e:
traceback.print_exc()
# print("i am wrong")
t.close()
if os.path.exists('baojianhui_log.txt'):
os.remove('baojianhui_log.txt') #运行成功之后删除log
os.chdir('C:/Users/Administrator/Desktop')
if search('C:/Users/Administrator/Desktop/baojian_guizhang'):
s3_function.zip_ya('baojian_guizhang')
s3_function.upload_to_aws_s3(
'baojian_guizhang.zip', 's3qingdao',
'baojian_guizhang_history' + str(
(datetime.datetime.today()).date()) + '.zip')
os.remove('baojian_guizhang.zip')
else:
print('文件夹空的')
with open('baojian_guizhang_history' + str(
(datetime.datetime.today()).date()) + '.txt',
'w',
encoding='utf-8') as f:
f.write("今天保监会规章无日增")
s3_function.upload_to_aws_s3(
'baojian_guizhang_history' + str(
(datetime.datetime.today()).date()) + '.txt', 's3qingdao',
'baojian_guizhang_history' + str(
(datetime.datetime.today()).date()) + '.txt')
os.remove('baojian_guizhang_history' +
os.chdir('C:/Users/Administrator/Desktop/wenjian/law')
while iter_flag == False:
if os.path.exists("complete_log" + str(law_url.split('/')[-2]) + ".txt"): # 如果是中途断点
with open("complete_log" + str(law_url.split('/')[-2]) + ".txt", 'r') as f:
params = f.read().split(',')
start_pag = int(params[0])
iter_flag = history_data(law_url, start_pag)
else: # 如果是首次运行
iter_flag = history_data(law_url, 1)
os.remove("complete_log" + str(law_url.split('/')[-2]) + ".txt")
#压缩文件，上传到云
os.chdir('C:/Users/Administrator/Desktop/wenjian')
s3_function.zip_ya('law')
#upload
s3_function.upload_to_aws_s3('law.zip','storageforccbrpa','law_history.zip')
#test case 2.
iter_flag = False
admin_law = 'http://www.pbc.gov.cn/tiaofasi/144941/144953/21888/index'
os.chdir('C:/Users/Administrator/Desktop/wenjian')
if os.path.exists('C:/Users/Administrator/Desktop/wenjian/admin'):
os.remove('C:/Users/Administrator/Desktop/wenjian/admin')
os.mkdir('admin')
else:
os.mkdir('admin')
os.chdir('C:/Users/Administrator/Desktop/wenjian/admin')
while iter_flag == False:
if os.path.exists("complete_log" + str(admin_law.split('/')[-2]) + ".txt"): # 如果是中途断点
with open("complete_log" + str(admin_law.split('/')[-2]) + ".txt", 'r') as f:
t.init()
t.wait(5)
#read page
t.url("http://bank.jrj.com.cn/bankpro/")
t.wait(5)
#locate items
item_list = t.read(element_identifier='//div[@class = "qutop mt"]').split()
#txt
today = datetime.datetime.today()
today = str(today.date())
file_name = today+'product_daily.txt'
with open(file_name,'w',encoding='utf-8') as f:
f.writelines([item+'\n' for item in item_list])
#excel
# daily_on_sale = int(item_list[1][:-11])
# monthly_on_sale =int(item_list[2][:-13])
# monthly_expire = int(item_list[3][:-13])
#
# pd.DataFrame({'今日在售银行理财产品':int(daily_on_sale),'本月累计发售银行理财产品':int(monthly_on_sale),'本月累计到期银行理财产品':int(monthly_expire)},index=[0]).to_csv(today+'product_daily.csv',index = False, encoding = 'utf-8')
s3_function.upload_to_aws_s3(file_name,'s3qingdao',file_name)
os.remove(file_name)
t.close()
except:
with open('FAIL_product_daily' + str((datetime.datetime.today()).date()) + '.txt', 'w', encoding='utf-8') as f:
f.write("今日理财产品日报任务启动失败")
s3_function.upload_to_aws_s3('FAIL_product_daily' + str((datetime.datetime.today()).date()) + '.txt', 's3qingdao',
'FAIL_product_daily' + str((datetime.datetime.today()).date()) + '.txt')
os.remove('FAIL_product_daily' + str((datetime.datetime.today()).date()) + '.txt')