def login(arg_url):
acc = web_account_dic('cnfeol_acc')
pwd = web_account_dic('cnfeol_pwd')
driver.get(arg_url)
# driver.find_element_by_xpath("//*[@id='LoginBox']/a[2]").click()
time.sleep(10)
username = driver.find_element_by_xpath('//*[@id="TextBoxScreenName"]')
username.send_keys(acc)
password = driver.find_element_by_xpath("//*[@id='TextBoxPassword']")
password.send_keys(pwd)
driver.find_element_by_xpath(
"//input[@type='submit'][@name='ButtonLogin']").click()
def info_amm():
global err_log
global err_flag
err_flag = False
print('開始 amm 報價資料收集程序...')
#LOG File
log_name = "scrp_amm_" + parser.parse(str(
datetime.datetime.now())).strftime("%Y%m%d") + ".txt"
err_log = open(log_name, 'a', encoding='UTF-8')
tStart = time.time() #計時開始
err_log.write("\n\n\n*** LOG datetime " + str(datetime.datetime.now()) +
" ***\n")
acc = web_account_dic('amm_acc')
pwd = web_account_dic('amm_pwd')
#print(acc)
#print(pwd)
chrome_options = Options()
#chrome_options.add_argument('--headless')
#chrome_options.add_argument('--disable-gpu')
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.implicitly_wait(5)
driver.get("https://www.amm.com/")
driver.find_element_by_xpath(
'//*[@id="mhHeader_loginInfo_lnkLogin"]').click()
#輸入帳號密碼登入網站
elem = driver.find_element_by_xpath('//*[@id="username"]')
elem.send_keys(acc)
elem = driver.find_element_by_xpath('//*[@id="password"]')
elem.send_keys(pwd)
driver.find_element_by_xpath(
'/html/body/div[1]/div/div/div[2]/div/div/div/div[2]/form/fieldset/div[2]/div[2]/div[1]/button'
).click()
#Target website arguments dict
tar_args_dic = {
'03001':
'https://www.amm.com/pricing/Ferro-Chrome/Ferro-chrome-Japan-import-8-9-C-basis-60-Cr-CIF-Japan-duty-unpaid--per-lb-contained-chrome-ymgnlh4zd32s',
'03002':
'https://www.amm.com/pricing/Ferro-Chrome/Ferro-chrome-China-import-50-Cr-US--per-Ib-contained-chrome-CIF-Shanghai-qs7l6lwpsshg'
}
#執行各項報表抓取
price_all = []
for tar, url in tar_args_dic.items():
print('開始進行 ' + tar + ' 資料抓取.')
err_log.write('\n\n開始進行 ' + tar + ' 資料抓取.\n')
try:
price = web_info_collect(driver, tar, url)
err_log.write(str(price) + '\n')
print(price)
if len(price) > 0:
price_all.append(price)
err_log.write('來源 ' + tar + ' 資料抓取完畢.\n')
except Exception as e:
err_flag = True
print("info_amm -> " + tar + " 資料抓取錯誤，例外訊息如下:")
print(e.args)
print("\n\n")
err_log.write("info_amm -> " + tar + " 資料抓取錯誤，例外訊息如下:\n")
err_log.write(str(e.args))
err_log.write("\n\n")
#關閉瀏覽器視窗
driver.quit()
#寫入資料庫
if len(price_all) > 0:
#print(price_all)
db_process(price_all)
tEnd = time.time() #計時結束
err_log.write("\n\n\n結轉耗時 %f sec\n" % (tEnd - tStart)) #會自動做進位
err_log.write("*** End LOG ***\n\n")
# Close File
err_log.close()
#如果執行過程無錯誤，最後刪除log file
if err_flag == False:
os.remove(log_name)
print('\n\n\n本次amm市場報價資訊抓取結束，等待下次執行...\n\n\n')
def info_argus():
global err_log
global err_flag
print('開始 Argus Metals 報價資料收集程序...')
err_flag = False
str_date = parser.parse(str(datetime.datetime.now())).strftime("%Y%m%d")
#LOG File
log_name = "scrp_argus_" + str_date + ".txt"
err_log = open(log_name, 'a', encoding='UTF-8')
tStart = time.time() #計時開始
err_log.write("\n\n\n*** LOG datetime " + str(datetime.datetime.now()) +
" ***\n")
acc = web_account_dic('argus_acc')
pwd = web_account_dic('argus_pwd')
#print(acc)
#print(pwd)
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
driver = webdriver.Chrome(chrome_options=chrome_options)
#driver.implicitly_wait(10)
driver.get("https://metals.argusmedia.com/n?ulog=True")
#輸入帳號密碼登入網站
elem = driver.find_element_by_name("Username")
elem.send_keys(acc)
elem = driver.find_element_by_name("Password")
elem.send_keys(pwd)
driver.find_element_by_xpath("//*[@id='buttonSubmitLogin']").click()
#若有出現Switch Device按鈕，就點下去
sw_device(driver)
#Target website hyperlink dict
tar_link_dic = {
'01001': {
'link':
'https://metals.argusmedia.com/price/assessment/lme-nickel-3-month-official',
'unit': 'MT'
},
'01002': {
'link':
'https://metals.argusmedia.com/price/assessment/lme-nickel-cash-official',
'unit': 'MT'
},
'01003': {
'link':
'https://metals.argusmedia.com/price/assessment/lme-nickel-trading-volume',
'unit': 'MT'
},
'01004': {
'link':
'https://metals.argusmedia.com/price/assessment/lme-nickel-warehouse-stocks',
'unit': 'MT'
},
'01005': {
'link':
'https://metals.argusmedia.com/price/assessment/PA00120250800',
'unit': 'dmt'
},
'01006': {
'link':
'https://metals.argusmedia.com/price/assessment/PA00162710000',
'unit': 'MT'
},
'01007': {
'link':
'https://metals.argusmedia.com/price/assessment/PA00187170000',
'unit': 'LB'
},
'01008': {
'link':
'https://metals.argusmedia.com/price/assessment/PA00187290000',
'unit': 'LB'
},
'01009': {
'link':
'https://metals.argusmedia.com/price/assessment/PA00187260000',
'unit': 'MT'
},
'01010': {
'link':
'https://metals.argusmedia.com/price/assessment/PA00162700000',
'unit': 'MT'
},
'01011': {
'link':
'https://metals.argusmedia.com/price/assessment/PA00162820000',
'unit': 'MT'
},
'01012': {
'link':
'https://metals.argusmedia.com/price/assessment/shfe-nickel-warehouse-stocks',
'unit': 'MT'
},
'01013': {
'link':
'https://metals.argusmedia.com/price/assessment/lme-copper-cash-official',
'unit': 'MT'
},
'01014': {
'link':
'https://metals.argusmedia.com/price/assessment/PA00162680000',
'unit': 'MT'
},
'01015': {
'link':
'https://metals.argusmedia.com/price/assessment/PA00142990000',
'unit': 'LB'
}
}
#執行各項報表抓取
price_all = []
for tar, args in tar_link_dic.items():
print('\n\n開始進行 ' + tar + ' 資料抓取.')
err_log.write('\n\n開始進行 ' + tar + ' 資料抓取.\n')
try:
price = web_info_collect(driver, tar, args['link'], args['unit'])
err_log.write(str(price) + '\n')
print(price)
if len(price) > 0:
price_all.append(price)
err_log.write('來源 ' + tar + ' 資料抓取完畢.\n')
except Exception as e:
err_flag = True
print("info_argus -> " + tar + " 資料抓取錯誤，例外訊息如下:")
print(e.args)
print("\n\n")
err_log.write("info_argus -> " + tar + " 資料抓取錯誤，例外訊息如下:\n")
err_log.write(str(e.args))
err_log.write("\n\n")
#關閉瀏覽器視窗
driver.quit()
#寫入資料庫
if len(price_all) > 0:
#print(price_all)
db_process(price_all)
tEnd = time.time() #計時結束
err_log.write("\n\n\n結轉耗時 %f sec\n" % (tEnd - tStart)) #會自動做進位
err_log.write("*** End LOG ***\n\n")
# Close File
err_log.close()
#如果執行過程無錯誤，最後刪除log file
if err_flag == False:
os.remove(log_name)
print('\n\n\n本次argus市場報價資訊抓取結束，等待下次執行...\n\n\n')
def info_fubao():
global err_log
global err_flag
err_flag = False
print('開始 fubao 報價資料收集程序...')
#LOG File
log_name = "scrp_fubao_" + parser.parse(str(datetime.datetime.now())).strftime("%Y%m%d") + ".txt"
err_log = open(log_name, 'a', encoding = 'UTF-8')
tStart = time.time()#計時開始
err_log.write("\n\n\n*** LOG datetime " + str(datetime.datetime.now()) + " ***\n")
acc = web_account_dic('fubao_acc')
pwd = web_account_dic('fubao_pwd')
#print(acc)
#print(pwd)
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
driver = webdriver.Chrome(chrome_options=chrome_options)
#driver.implicitly_wait(5)
#輸入帳號密碼登入網站
driver.get('http://www.f139.com/')
elem=driver.find_element_by_id("userName")
elem.send_keys(acc)
elem=driver.find_element_by_id("passWord")
elem.send_keys(pwd)
elem=driver.find_element_by_xpath('//*[@id="loginForm"]/input[5]')
elem.click()
#Target website arguments dict
tar_args_dic = {
'fubao01': {'link': 'http://data.f139.com/list.do?pid=&vid=75&qw=5:231'},
'fubao02': {'link': 'http://data.f139.com/list.do?pid=&vid=75&qw=5:215'}
}
#執行各項報表抓取
price_all = []
for tar, url in tar_args_dic.items():
print('開始進行 ' + tar + ' 資料抓取.' )
err_log.write('\n\n開始進行 ' + tar + ' 資料抓取.\n' )
try:
price = web_info_collect(driver, tar, url)
err_log.write(str(price) + '\n')
print(price)
if len(price) > 0:
price_all += price
err_log.write('來源 ' + tar + ' 資料抓取完畢.\n' )
except Exception as e:
err_flag = True
print("info_fubao -> " + tar + " 資料抓取錯誤，例外訊息如下:")
print(e.args)
print("\n\n")
err_log.write("info_fubao -> " + tar + " 資料抓取錯誤，例外訊息如下:\n")
err_log.write(str(e.args))
err_log.write("\n\n")
#關閉瀏覽器視窗
driver.quit()
#寫入資料庫
if len(price_all) > 0:
#print(price_all)
db_process(price_all)
tEnd = time.time()#計時結束
err_log.write ("\n\n\n結轉耗時 %f sec\n" % (tEnd - tStart)) #會自動做進位
err_log.write("*** End LOG ***\n\n")
# Close File
err_log.close()
#如果執行過程無錯誤，最後刪除log file
if err_flag == False:
os.remove(log_name)
print('\n\n\n本次fubao市場報價資訊抓取結束，等待下次執行...\n\n\n')
def info_fubao():
global today, today_02, driver
global err_log
global err_flag
err_flag = False
log_name = "scrp_fubao_" + datetime.datetime.now().strftime(
"%Y%m%d") + ".txt"
err_log = open(log_name, 'a', encoding='UTF-8')
acc = web_account_dic('fubao_acc')
pwd = web_account_dic('fubao_pwd')
#print(acc)
#print(pwd)
chrome_options = Options()
driver = webdriver.Chrome(chrome_options=chrome_options)
#driver.implicitly_wait(5)
today = str(datetime.datetime.now())
today_02 = today[0:4] + str(int(today[5:7])) + str(int(today[8:10]))
print(today_02)
driver.get('http://www.f139.com/')
elem = driver.find_element_by_id("userName")
elem.send_keys(acc)
elem = driver.find_element_by_id("passWord")
elem.send_keys(pwd)
elem = driver.find_element_by_xpath('//*[@id="loginForm"]/input[6]')
elem.click()
try:
elem = driver.find_element_by_xpath('//*[@id="loginerror"]')
if len(elem.text) > 0:
err_flag = True
print("網站登入異常，error msg:")
print(elem.text)
err_log.write("網站登入異常，error msg:")
err_log.write(elem.text)
except:
print("網站正常登入.")
if err_flag == False:
#Target website arguments dict
tar_args_dic = {
'fubao01': {
'link': 'http://data.f139.com/list.do?pid=&vid=84&qw=5:231'
},
}
#執行各項報表抓取
price_all = []
for tar, url in tar_args_dic.items():
print('開始進行 ' + tar + ' 資料抓取.')
err_log.write('\n\n開始進行 ' + tar + ' 資料抓取.\n')
try:
price = web_info_collect(driver, tar, url)
err_log.write(str(price) + '\n')
print('price' + str(price))
if len(price) > 0:
price_all += price
err_log.write('來源 ' + tar + ' 資料抓取完畢.\n')
print('price_all' + str(price_all))
if len(price_all) > 0:
#print(price_all)
result = db_process(price_all)
err_flag = result[0]
err_log.write(result[1])
print(result)
err_log.write(str(price) + '\n')
err_log.write('來源 ' + tar + ' 資料抓取完畢.\n')
except Exception as e:
err_flag = True
print("info_fubao -> " + tar + " 資料抓取錯誤，例外訊息如下:")
print(e.args)
print("\n\n")
err_log.write("info_fubao -> " + tar + " 資料抓取錯誤，例外訊息如下:\n")
err_log.write(str(e.args))
err_log.write("\n\n")
#關閉瀏覽器視窗
driver.quit()
# Close File
err_log.close()