def read_content(page_num, url_prefix, i, today):
t.url(url_prefix + str(page_num) + '.html')
# 启动很慢
t.wait(2)
if t.read(element_identifier='//td[@colspan = "2"]//table[' + str(i) +
']//span[@class = "hui12"]') < today:
t.close()
return '', '', '', ''
if '' == t.read(element_identifier='//td[@colspan = "2"]//table[' +
str(i) + ']'):
print("no here")
raise Exception("an exception")
file_name = t.read(element_identifier='//td[@colspan = "2"]//table[' +
str(i) + ']')
file_name = file_name[:-10] + str("_") + file_name[-10:] + str('.txt')
time = file_name[-14:-4]
prefix = 'http://www.pbc.gov.cn'
content_url = prefix + t.read(
element_identifier='//td[@colspan = "2"]//table[' + str(i) +
']//a/@href')
if '' == t.read(element_identifier='//td[@colspan = "2"]//table[' +
str(i) + ']//a/@href'):
print("no here")
raise Exception("an exception")
flag = t.read(element_identifier='//td[@colspan = "2"]//table[' + str(i) +
']//a/@href') # 判断是否需要下载
return flag, time, content_url, file_name
def main(url, mode='txt'):
if os.path.exists('baojianhui_log.txt'):
with open('baojianhui_log.txt', 'r', encoding='utf-8') as f:
params = f.read().split(',')
curr_page = params[0]
else: # 如果是第一次执行，全取初始值；
curr_page = 1
url_link = str(url) + str(curr_page)
web_init(url_link)
total_page = int(
t.read(element_identifier='//div[@class = "ng-binding"][last()]').
split('/')[-1])
while int(curr_page) < int(total_page): #从1开始，做完之后翻页；
main_operation(url, mode) #如果有页可翻，就翻页
print('click once')
t.click(element_identifier='//a[@ng-click = "pager.next()"]') #翻页
t.wait(5)
curr_page = int(
t.read(element_identifier='//div[@class = "ng-binding"][last()]').
split('/')[0])
with open('baojianhui_log.txt', 'w', encoding='utf-8') as f:
f.write(str(curr_page) + ',' + str(1) + ',' +
str(1)) #翻页之后，index重置；i更新；
if curr_page == total_page:
main_operation(url, mode) #如果是最后一页了，只需要做一次main
t.close()
return True
def login(self):
t.url(
'https://login.live.com/login.srf?wa=wsignin1.0&rpsnv=13&ct=1586073207&rver=7.0.6737.0&wp=MBI_SSL&wreply=https%3a%2f%2foutlook.live.com%2fowa%2f%3fnlp%3d1%26RpsCsrfState%3d6590c65e-2e3f-b1ed-bda9-2c5e901a9000&id=292841&aadredir=1&whr=outlook.sg&CBCXT=out&lw=1&fl=dob%2cflname%2cwld&cobrandid=90015'
)
t.wait(1)
type_into('//*[@type="email"]', self.account + '[enter]')
t.wait(1)
type_into('//*[@name="passwd"]', self.password + '[enter]')
def direct_download(content_url, time, i):
# 当直接跳到需要下载的文件的时候
if 'cnhttp' in content_url:
content_url = content_url[21:] # 不知道为什么会出错这个
# 取到数据
print("文件{} 是直接下载文件。".format(i))
if '' == t.read(element_identifier='//td[@colspan = "2"]//table[' +
str(i) + ']//a/@href'):
print("no here")
raise Exception("an exception")
file_name = t.read(element_identifier='//td[@colspan = "2"]//table[' +
str(i) + ']//a/@href')
suffix = file_name.split('.')[-1]
file_name = file_name.split('/')[-1]
t.url(content_url)
# 启动很慢
wait_seconds = 1
total_seconds = 0
while os.path.exists(file_name) == False:
t.wait(wait_seconds)
total_seconds += wait_seconds
if total_seconds > MAX_WAIT:
print('download fails')
break
os.rename(
file_name, file_name[:-(len(suffix) + 1)] + "_" + time + '.' +
file_name[-(len(suffix) + 1):])
else:
# 取到数据
print("文件{} 是直接下载文件。".format(i))
if '' == t.read(element_identifier='//td[@colspan = "2"]//table[' +
str(i) + ']//a/@href'):
print("no here")
raise Exception("an exception")
file_name = t.read(element_identifier='//td[@colspan = "2"]//table[' +
str(i) + ']//a/@href')
suffix = file_name.split('.')[-1]
file_name = file_name.split('/')[-1]
t.url(content_url)
# 启动很慢
wait_seconds = 1
total_seconds = 0
while os.path.exists(file_name) == False:
t.wait(wait_seconds)
total_seconds += wait_seconds
if total_seconds > MAX_WAIT:
print('download fails')
break
os.rename(
file_name, file_name[:-(len(suffix) + 1)] + "_" + time + '.' +
file_name[-(len(suffix) + 1):])
def wait_for_pageload(selector):
wait_status = 0
for loop_wait in range(1, 20):
print(f"{loop_wait}. waiting for page to appear. wait for 1s...")
if t.present(selector):
wait_status = 1
break
else:
t.wait(1)
print("bp801.2 wait_status = {}".format(wait_status))
def get_count_values(page_num, url_prefix):
t.url(url_prefix + str(page_num) + '.html')
print("现在所在页面 {}".format(page_num))
t.wait(5)
# 拿到value
count_values = t.count(element_identifier='//td[@colspan = "2"]//table')
print("页面有{}个文件".format(count_values))
with open('count_items_' + str(page_num) + '_'+str(url_prefix.split('/')[-2]) + '.txt', 'w', encoding='utf-8') as f:
f.write('page:' + str(page_num) + ':' + str(count_values)) # 以：为分隔符；记录当前页面和页面总共item数量
return 'count_items_' + str(page_num) + '_'+str(url_prefix.split('/')[-2]) + '.txt'
def mark_as_unread(self):
more_action_xpath = '//button[@aria-label="More mail actions"]'
mark_as_unread_xpath = '//button[@name="Mark as unread"]'
mark_as_read_xpath = '//button[@name="Mark as read"]'
click(more_action_xpath)
if present(mark_as_read_xpath, timeout=3):
click(mark_as_read_xpath)
t.wait(1)
click(more_action_xpath)
if present(mark_as_unread_xpath, timeout=3):
click(mark_as_unread_xpath)
t.wait(1)
def get_shoe(shoe, g, email):
gender = g
# print('[nike]',gender)
t.init(visual_automation=True)
t.url('https://www.nike.com/sg/')
t.type('//input[@id = "TypeaheadSearchInput"]', shoe + " shoes")
t.click('//button[@class = "btn-search z2 bg-transparent"]')
t.wait(3)
if gender == " men":
t.click('(//span[contains(@class,"filter-item")])[1]')
elif gender == " women":
t.click('(//span[contains(@class,"filter-item")])[2]')
t.wait(1)
count = t.count('//a[@class ="product-card__link-overlay"]')
# print('[nike]',count)
details = []
if count != 0:
for i in range(0, min(count, 3)):
k = i + 1
name = t.read(f'(//a[@class = "product-card__link-overlay"])[{k}]')
price = t.read(f'(//div[@data-test="product-price"])[{k}]')
img = t.read(
f'(//div[contains(@class, "product-card__hero")]/picture/img)[{k}]/@src'
)
link = t.read(f'(//a[contains(@class,"product-card")])[{k}]/@href')
# print('[nike]',name , price, img)
details.append({
"email": email,
"name": name,
"price": price,
"img": img,
"Company": "Nike",
"link": link
})
else:
details.append({
"email": email,
"name": "NA",
"price": "NA",
"img": "NA",
"Company": "Nike",
"link": "NA"
})
# print(details)
return details
def wait_for_mainpageload(selector):
wait_status = 0
result = 1
for loop_wait in range(1, 15):
print(f"{loop_wait}. waiting for page to appear. wait for 1s...")
if t.present(selector):
wait_status = 1
break
else:
t.wait(1)
if loop_wait == 14:
result = 0
return result
print("bp801.2 wait_status = {}".format(wait_status))
def run():
conn = util.create_connection("./db/news.db")
site = util.getSiteByName(conn, "New York Times")
site_url = site[0][2]
site_id = site[0][0]
t.init(visual_automation = True, chrome_browser = True)
t.url(site_url)
t.wait(10)
df = catchContent()
df = util.fixImgLink(df,"https://cf-templates-fghyux9ggb7t-ap-southeast-1.s3-ap-southeast-1.amazonaws.com/NewYorkTimes.png")
df = util.fixSummary(df)
t.wait(20)
t.close()
util.updateNews(conn, site_id, df)
def number_of_travellers(adult_pax, children_pax, children_age):
print(f"Adults: {adult_pax} and Children: {children_pax}")
t.click(f'//select[@id="adult-count"]')
t.select('//select[@id="adult-count"]', f'{adult_pax}')
# set the number of child travellers
t.click(f'//select[@id="child-count"]')
t.select('//select[@id="child-count"]', f'{children_pax}')
# Set the age for each child traveller
if children_pax > 0:
for m in range(0, children_pax):
print(f'Child {m+1} age {str(children_age[m])}')
t.click(f'//select[@id="child-age-{m+1}"]')
t.wait(1)
t.select(f'//select[@id="child-age-{m+1}"]', str(children_age[m]))
t.wait(1)
def getFlightExcel(info,ind):
flight_main, time_lst, code_lst, dur_lst, ind = getFlightInfo(info['dates'], ind)
#print(flight_main['Details'])
print(code_lst)
print(dur_lst)
print(time_lst)
k = len(info['dates'])
q = len(info['city'])
flight_lst = []
for i in range(k):
if i == (k-1) and i > 0 and q == k:
flight_lst.append(info['dates'][i])
flight = info['city'][i] + '-' + info['city'][0]
flight_lst.append(flight)
else:
flight_lst.append(info['dates'][i])
flight = info['city'][i] + '-' + info['city'][i + 1]
flight_lst.append(flight)
print(flight_lst)
###Compare Price with Expedia (Hyperlink/Multi to be added)
for j in range(2):
t.close()
t.init()
t.wait(0.5)
flight_search(info)
t.wait(5)
flight_main['Flight Info'][j] = flight_lst
price_exp, url_exp = getExpFlightPrice(code_lst[k*j:k*(j+1)], time_lst[k*j:k*(j+1)], dur_lst[k*j:k*(j+1)])
print(price_exp)
print(url_exp)
print(flight_main['Price'])
if price_exp < flight_main['Price'][j]:
if price_exp != 0:
flight_main['Price'][j] = price_exp
flight_main['Hyperlink'][j] = url_exp
print(flight_main['Price'])
print(flight_main['Hyperlink'])
return flight_main
def flight_search(flight_request):
search_dt = dt.today()
request_id = flight_request['Request_ID']
info = flight_request['Request_Details']
t.init()
t.url('https://www.skyscanner.com.sg/')
tu.wait_for_pageload('//input[@id="fsc-trip-type-selector-return"]')
fill_search(info)
ind = 0
flight_main = getFlightExcel(info, ind)
t.wait(10.0)
t.close()
flight_main.update({
'Request_ID': request_id,
'Search_Datetime': search_dt
})
dbf.newFlightDeals(flight_main)
outFile = dbf.export_FlightDeals(request_id, search_dt)
return outFile
def get_count_values(page_num, url_prefix, today):
t.url(url_prefix + str(page_num) + '.html')
print("现在所在页面 {}".format(page_num))
t.wait(5)
# 拿到value
count_values = t.count(element_identifier='//td[@colspan = "2"]//table')
# today = '2018-04-24'
if t.read(
element_identifier=
'//td[@colspan = "2"]//table[1]//span[@class = "hui12"]') < today:
return '今日无增量'
print("页面有{}个文件".format(count_values))
with open('count_items_' + str(page_num) + '_' +
str(url_prefix.split('/')[-2]) + '.txt',
'w',
encoding='utf-8') as f:
f.write('page:' + str(page_num) + ':' +
str(count_values)) # 以：为分隔符；记录当前页面和页面总共item数量
return 'count_items_' + str(page_num) + '_' + str(
url_prefix.split('/')[-2]) + '.txt'
def catchContent():
number_to = t.count(
'(//div[@class="col"]/div[contains(@class, "today")]/ul/li[contains(@class, "col-md-12")])'
)
df_to = pd.DataFrame(index=range(0, number_to),
columns=['Sno', 'Title', 'URL', 'Summary', 'Img_URL'])
t.hover('//div[@class="container footer-main"]')
t.wait(2)
for n in range(1, number_to):
title = t.read(
'//div[@class="col"]/div[contains(@class, "today")]/ul/li[contains(@class, "col-md-12")][{}]//div[contains(@class, "article-listing_content")]//h2'
.format(n))
URL_o = t.read(
'//div[@class="col"]/div[contains(@class, "today")]/ul/li[contains(@class, "col-md-12")][{}]//@href'
.format(n))
URL = "https://www.todayonline.com" + str(URL_o)
Img_link = t.read(
'//div[@class="col"]/div[contains(@class, "today")]/ul/li[contains(@class, "col-md-12")][{}]//img/@src'
.format(n))
df_to.iloc[n - 1, 0] = n
df_to.iloc[n - 1, 1] = title.decode('utf-8')
df_to.iloc[n - 1, 2] = URL
df_to.iloc[n - 1, 4] = Img_link
for i in range(0, df_to.shape[0]):
if df_to['Img_URL'][i] == "":
df_to['Img_URL'][i] = np.nan
df_to.dropna(subset=['Img_URL'], inplace=True, how='any')
df_to = df_to.reset_index(drop=True)
df_to['Sno'] = df_to.index
df_to = util.fixImgLink(
df_to,
"https://cf-templates-fghyux9ggb7t-ap-southeast-1.s3-ap-southeast-1.amazonaws.com/todayOnline.png"
)
for n in range(0, df_to.shape[0]):
t.url(df_to.URL[n])
t.wait(4)
t.hover('//div[@class="article-detail_subscription"]')
t.wait(2)
number_p = t.count('//div/p[not(@class)]')
Content = ""
for i in range(1, number_p - 2):
cont = t.read('//div/p[not(@class)][{}]'.format(i))
Content = Content + "" + cont
summaries = Summarize(df_to.Title[n], unicode(str(Content), "utf-8"))
df_to.iloc[n - 1, 3] = summaries[0]
return df_to
def one_way_trip(enquiry):
start_date = dt.strptime(enquiry["dates"][0], '%d/%m/%Y')
start_month = start_date.strftime('%Y-%m')
adult_pax = int(enquiry['adult'])
child_pax = len(enquiry['child_age'])
child_age = enquiry['child_age']
t.click('//input[@id="fsc-trip-type-selector-one-way"]')
t.wait(0.5)
t.type('//input[@id="fsc-origin-search"]', enquiry["city"][0])
t.wait(0.5)
t.type('//input[@id="fsc-destination-search"]', enquiry["city"][1])
t.wait(0.5)
t.click(
'//button[@id="depart-fsc-datepicker-button"]//span[starts-with(@class,"DateInput")]'
)
t.click(f'//select[@id="depart-calendar__bpk_calendar_nav_select"]')
t.select('//select[@id="depart-calendar__bpk_calendar_nav_select"]',
f'{start_month}')
t.click(
f'//button[starts-with(@class,"BpkCalendarDate") and contains(@aria-label,"{start_date.strftime("%d %B %Y").lstrip("0")}")]'
)
t.click('//button[starts-with(@id,"CabinClassTravellersSelector")]')
t.click('//select[@id="search-controls-cabin-class-dropdown"]')
t.select('//select[@id="search-controls-cabin-class-dropdown"]',
lookup_cabin_class(enquiry["cabin_class"]))
# t.select('//select[@id="search-controls-cabin-class-dropdown"]',(enquiry["cabin_class"].capitalize()).replace(' ',''))
number_of_travellers(adult_pax, child_pax, child_age)
t.click('//button[@type="submit"][@aria-label="Search flights"]')
def multi_city_trip(enquiry):
t.click('//input[@id="fsc-trip-type-selector-multi-destination"]')
travel_dates = enquiry["dates"]
numDep = len(travel_dates)
cities = enquiry["city"]
numCity = len(cities)
form_flightleg = t.count(
'//*[@id="flights-search-controls-root"]/div/div/form/div[2]/ol/li')
if numDep < form_flightleg:
for cnt in range(form_flightleg - numDep):
t.click(
f'//*[@id="flights-search-controls-root"]/div/div/form/div[2]/ol/li[{form_flightleg-cnt}]/div[4]/button'
)
elif numDep > form_flightleg:
for cnt in range(numDep - form_flightleg):
t.click(
'//div[starts-with(@class,"MulticityControls_MulticityControls__add-leg-wrapper__2arYh")]/button'
)
t.wait(0.5)
for num in range(0, numDep):
start_date = dt.strptime(travel_dates[num], '%d/%m/%Y')
start_month = start_date.strftime('%Y-%m')
orig_city = cities[num]
if numCity == numDep:
if num < numDep - 1:
dest_city = cities[num + 1]
else:
dest_city = cities[0]
else:
dest_city = cities[num + 1]
t.type(f'//input[@id="fsc-origin-search-{num}"]', orig_city)
t.wait(0.5)
t.type(f'//input[@id="fsc-destination-search-{num}"]', dest_city)
t.wait(0.5)
t.click(
f'//button[@id="fsc-leg-date-{num}-fsc-datepicker-button"]//span[starts-with(@class,"DateInput")]'
)
t.click(
f'//select[@id="fsc-leg-date-{num}-calendar__bpk_calendar_nav_select"]'
)
t.select(
f'//select[@id="fsc-leg-date-{num}-calendar__bpk_calendar_nav_select"]',
f'{start_month}')
t.click(
f'//button[starts-with(@class,"BpkCalendarDate") and contains(@aria-label,"{start_date.strftime("%d %B %Y").lstrip("0")}")]'
)
t.click('//button[starts-with(@id,"CabinClassTravellersSelector")]')
t.click('//select[@id="search-controls-cabin-class-dropdown"]')
t.select('//select[@id="search-controls-cabin-class-dropdown"]',
lookup_cabin_class(enquiry["cabin_class"]))
adult_pax = int(enquiry['adult'])
child_pax = len(enquiry['child_age'])
child_age = enquiry['child_age']
number_of_travellers(adult_pax, child_pax, child_age)
t.click('//button[@type="submit"][@aria-label="Search flights"]')
def multi_city_trip(enquiry):
t.click('//input[@id="flight-type-multi-dest-hp-flight"]')
travel_dates = enquiry["dates"]
numDep = len(travel_dates)
cities = enquiry["city"]
numCity = len(cities)
form_flightleg = (t.count(
'//div[@class="cols-nested gcw-multidest-flights-container"]/div/fieldset'
))
print(form_flightleg)
if numDep < form_flightleg:
for cnt in range(form_flightleg - numDep):
t.click(
f'//*[@id="flightlegs-list-fieldset-{form_flightleg-cnt}-hp-flight"]/div/a'
)
elif numDep > form_flightleg:
for cnt in range(numDep - form_flightleg):
t.click('//a[@id="add-flight-leg-hp-flight"]')
t.wait(0.5)
t.type('//input[@id="flight-origin-hp-flight"]', cities[0])
t.type('//input[@id="flight-destination-hp-flight"]', cities[1])
t.type('//input[@id="flight-departing-single-hp-flight"]', '[clear]')
t.type('//input[@id="flight-departing-single-hp-flight"]',
(dt.strptime(travel_dates[0], '%d/%m/%Y')).strftime("%d/%m/%Y"))
for num in range(1, numDep):
print(f"num:{num} and form_flightleg:{form_flightleg}")
start_date = dt.strptime(travel_dates[num], '%d/%m/%Y')
orig_city = cities[num]
if numCity == numDep:
if num < numDep - 1:
dest_city = cities[num + 1]
else:
dest_city = cities[0]
else:
dest_city = cities[num + 1]
t.type(f'//input[@id="flight-{num+1}-origin-hp-flight"]', orig_city)
t.wait(0.5)
t.type(f'//input[@id="flight-{num+1}-destination-hp-flight"]',
dest_city)
t.wait(0.5)
t.type(f'//input[@id="flight-{num+1}-departing-hp-flight"]', '[clear]')
t.type(f'//input[@id="flight-{num+1}-departing-hp-flight"]',
start_date.strftime("%d/%m/%Y"))
t.click('//a[@id="flight-advanced-options-hp-flight"]')
t.select('//select[@id="flight-advanced-preferred-class-hp-flight"]',
lookup_cabin_class(enquiry["cabin_class"]))
t.click('//*[@id="gcw-flights-form-hp-flight"]/div[8]/label/button')
def check_availability(reservation_date,reservation_time,party_size,restaurant_name):
try:
#Convert User Defined Values to System Usable Values
reservation_day=reservation_date.split('/')[0]
reservation_month =reservation_date.split('/')[1]
reservation_month=int(reservation_month)-1
reservation_year =reservation_date.split('/')[2]
reservation_time_int=int(reservation_time)
start_time_hr= reservation_time[:2]
if reservation_time_int>1159:
if start_time_hr!="12":
start_time_hr=int(start_time_hr)-12
start_time_option = str(start_time_hr)+":"+reservation_time[2:4]+" pm"
else:
start_time_option = str(start_time_hr)+":"+reservation_time[2:4]+" am"
#Booking Parameters
chope_url ='https://www.chope.co/singapore-restaurants/category/restaurant/'
t.init()
t.url(chope_url)
t.wait(10)
#Date Field
t.click(f"(//span[contains(@class,'input-group-addon icon-calendar')])[1]")
t.wait(7)
boolean_flag=1
while boolean_flag:
if t.present(f"//td[@data-handler='selectDay'and @data-year='{reservation_year}' and @data-month='{reservation_month}']/a[text()='{reservation_day}']"):
t.click(f"//td[@data-handler='selectDay'and @data-year='{reservation_year}' and @data-month='{reservation_month}']/a[text()='{reservation_day}']")
boolean_flag=0
else:
t.click('//a[@title="Next"]')
t.click(f"//td[@data-handler='selectDay'and @data-month='{reservation_month}']/a[text()='{reservation_day}']")
#Time Field
t.select(f"//select[contains(@id,'time-field')]",start_time_option)
#Number of Diners Field
t.click(f"(//span[contains(@class,'input-group-addon icon-person')])[1]")
t.select(f"//select[contains(@id,'adults')]",party_size)
#Restaurant Field
t.type(f"//select[contains(@id,'sb-sel-restaurant')]",restaurant_name)
t.click('//button[@id="btn-search"]')
t.wait(5)
if t.present(f"//div[@class='alert alert-danger']"):
print('Not Available')
return 0
else:
print ('Available')
return 1
except:
print('Error')
return 'Reservation Unsuccessful. Unforunately, the restaurant was not able to accomodate your reservation.'
def run():
conn = util.create_connection("./db/news.db")
site = util.getSiteByName(conn, "Today Online")
site_url = site[0][2]
site_id = site[0][0]
t.init(visual_automation=True, chrome_browser=True)
t.url(site_url)
t.wait(2)
t.hover('//div[@class="container footer-main"]')
t.wait(6)
df = catchContent()
t.wait(20)
t.close()
util.updateNews(conn, site_id, df)
# '[enter]' = enter key, '[clear]' = clear field
t.type('search-box', 'github')
# use read() to fetch and return text from UI element
search_text = t.read('search-box')
# use echo() to print to output, same as Python print()
t.echo(search_text)
# use click() to click on an UI element or x, y location
# rclick() = right-click, dclick() = double-click
t.click('search-button')
# use wait() to wait for a number of seconds
# default wait() is 5 seconds
t.wait(6.6)
# use snap() to save screenshot of page or UI element
# page = web page, page.png = computer screen
t.snap('page', 'results.png')
t.snap('logo', 'logo.png')
# another example of interacting with a web page
# include http:// or https:// in URL parameter
t.url('https://duckduckgo.com')
t.type('search_form_input_homepage',
'The search engine that doesn\'t track you.')
t.snap('page', 'duckduckgo.png')
t.wait(4.4)
# use close() to close TagUI process and web browser
def get_shoe(shoe_name, g, email):
"""
Get shoe details from jdsports.com.sg
:param shoe_name: name of the shoe to search for
:param gender: gender of the subscriber
:param email: email id of the subscriber
:return: details, list of shoe details.
"""
details = []
t.init(visual_automation=True)
t.url('https://www.jdsports.com.sg/')
t.wait(5)
final_command = shoe_name + " shoes" + '[enter]'
t.keyboard('[esc]')
t.type('//input[@id = "srchInput"]', final_command)
#t.click('//input[@id ="srchButton"]')
t.wait(3)
if g == ' men':
if t.read(
'(//a[@data-e2e="plp-filterMenu-catItem"]/span)[contains(.,"Men")]'
):
t.click('(//a[@data-e2e="plp-filterMenu-catItem"]/span)[1]')
count = t.count(
'//ul[@id="productListMain"]//li[@class="productListItem "]')
t.wait(3)
if count != 0:
for i in range(1, min(count, 4)):
price = t.read(f'(//span[@class="pri"])[{i}]')
name = t.read(f'(//span[@class="itemTitle"])[{i}]')
img = t.read(
f'(//a[@class="itemImage"]/picture/img/@srcset)[{i}]')
link = "https://www.jdsports.com.sg" + t.read(
f'(//span[@class = "itemTitle"])[{i}]/a/@href')
details.append({
"email": email,
"name": name,
"price": price,
"img": img,
"Company": "JD",
"link": link
})
else:
details.append({
"email": email,
"name": "NA",
"price": "NA",
"img": "NA",
"Company": "JD",
"link": "NA"
})
elif g == ' women':
if t.read(
'(//a[@data-e2e="plp-filterMenu-catItem"]/span)[contains(.,"Women")]'
):
t.click(
'(//a[@data-e2e="plp-filterMenu-catItem"]/span)[.="Women"]')
count = t.count(
'//ul[@id="productListMain"]//li[@class="productListItem "]')
t.wait(3)
if count != 0:
for i in range(1, min(count, 4)):
price = t.read(f'(//span[@class="pri"])[{i}]')
name = t.read(f'(//span[@class="itemTitle"])[{i}]')
img = t.read(
f'(//a[@class="itemImage"]/picture/img/@srcset)[{i}]')
link = "https://www.jdsports.com.sg" + t.read(
f'(//span[@class = "itemTitle"])[{i}]/a/@href')
details.append({
"email": email,
"name": name,
"price": price,
"img": img,
"Company": "JD",
"link": link
})
else:
details.append({
"email": email,
"name": "NA",
"price": "NA",
"img": "NA",
"Company": "JD",
"link": "NA"
})
else:
count = t.count(
'//ul[@id="productListMain"]//li[@class="productListItem "]')
t.wait(3)
if count != 0:
for i in range(1, min(count, 4)):
price = t.read(f'(//span[@class="pri"])[{i}]')
name = t.read(f'(//span[@class="itemTitle"])[{i}]')
img = t.read(
f'(//a[@class="itemImage"]/picture/img/@srcset)[{i}]')
link = "https://www.jdsports.com.sg" + t.read(
f'(//span[@class = "itemTitle"])[{i}]/a/@href')
details.append({
"email": email,
"name": name,
"price": price,
"img": img,
"Company": "JD",
"link": link
})
else:
details.append({
"email": email,
"name": "NA",
"price": "NA",
"img": "NA",
"Company": "JD",
"link": "NA"
})
#t.close()
if len(details) == 0:
details.append({
"email": email,
"name": "NA",
"price": "NA",
"img": "NA",
"Company": "JD",
"link": "NA"
})
# print("JD BOT",details)
return details
def history_data_daily(url_prefix, start_page=1):
curr_page = 1
curr_doc = 1
today = datetime.datetime.today()
today = str(today.date())
try:
t.init()
t.wait(5)
page_file = get_max_page(url_prefix)
with open(page_file, 'r') as f:
max_page = int(f.read()) + 1 # 拿到最大page，加1因为python index是开区间；
os.remove(page_file)
for page_num in range(start_page, max_page):
curr_page = page_num
count_values_file = get_count_values(page_num, url_prefix, today)
if count_values_file == '今日无增量':
t.close()
return True, '今日无增量'
with open(count_values_file, 'r') as f: # 拿到每一页的item数量；
count_values = int(f.read().split(':')[-1]) + 1
os.remove(count_values_file)
for i in range(1, count_values):
if os.path.exists('complete_log' +
str(url_prefix.split('/')[-2]) + '.txt'):
with open(
'complete_log' + str(url_prefix.split('/')[-2]) +
'.txt', 'r') as f:
start_doc = f.read().split(',')[1]
if i < int(start_doc):
continue
else:
pass
curr_doc = i
flag, time, content_url, file_name = read_content(
page_num, url_prefix, i, today)
if flag == '' and time == '' and content_url == '' and file_name == '':
if i == 1:
t.close()
return True, '今日无增量'
else:
return True, '今日增量'
if '.html' not in flag:
# 当直接跳到需要下载的文件的时候：需要提供 当前url，time后缀，目前的文件index
direct_download(content_url, time, i)
else: # 当没有直接下载的时候,需要读取网页
# 读取网页
read_text_content(content_url, file_name, page_num, i,
time, url_prefix)
#顺利完成了item的循环，当前页完成，complete log翻页，start doc放在1;如果page_num已经是count - 1，就不用做事情了。
if page_num != max_page - 1:
with open(
'complete_log' + str(url_prefix.split('/')[-2]) +
'.txt', 'w') as f:
f.write(str(page_num + 1) + ',' + str(1))
else:
pass
t.close()
return True
except: #如果检测到错误
with open('complete_log' + str(url_prefix.split('/')[-2]) + '.txt',
'w') as f:
f.write(str(curr_page) + ',' + str(curr_doc)) #留点
t.close()
return False
]
for col_name in name_list:
value_dict.setdefault(col_name, []) #初始化空数据集
#当可以翻页，或数据只有一页的时候，进行循环
while t.read(element_identifier='//*[@class = "pf-disabled"][last()]'
) != '下一页>>' or (page_curr == 1):
#每页的数据量大小（row number）
count_values = int(
t.count(element_identifier='//tbody[@id = "content"]//tr')
) + 1 # python从0开始
#print输出完成
print("count value for this page finishes..")
t.wait(1)
for i in range(1, count_values):
# 判定条件：如果是今天刚发行的，拿到所有主页面上的数据，
if str(
t.read(element_identifier='//tbody[@id = "content"]//tr[' +
str(i) + ']//td[@class = "px"]')) == str_to_append:
#序号
value_dict[name_list[0]].append(
t.read(element_identifier='//tbody[@id = "content"]//tr[' +
str(i) + ']/td[2]'))
#产品名称
value_dict[name_list[1]].append(
t.read(element_identifier='//tbody[@id = "content"]//tr[' +
str(i) + ']/td[3]'))
#发行银行
value_dict[name_list[2]].append(
def read_text_content(content_url, file_name, page_num, i, time, url_prefix):
# 读取网页
if 'cnhttp' in content_url:
content_url = content_url[21:] # 不知道为什么会出错这个
t.url(content_url)
# 启动很慢
else:
t.url(content_url)
# 启动很慢
# 获取pdf的数量，pdf的名字和pdf应该有的名字
t.wait(2)
pdf_count = t.count(element_identifier='//div[@id = "zoom"]//a/@href')
if pdf_count == 0: ##如果是正常的txt文件
# 取到列表
print("文件{} 是文档。".format(i))
# 取text
if t.read(element_identifier='//div[@id = "zoom"]') != '':
text = t.read(element_identifier='//div[@id = "zoom"]')
try:
with open(file_name, 'w', encoding='utf-8') as f:
f.write(text)
except:
with open('实施《全国企业兼并破产和职工再就业工作计划》银行呆、坏帐准备金核销办法_1997-10-01.txt',
'w',
encoding='utf-8') as f:
f.write(text)
elif t.read(element_identifier='//td[@class = "p1"]') != '':
text = t.read(element_identifier='//td[@class = "p1"]')
with open(file_name, 'w', encoding='utf-8') as f:
f.write(text)
else:
with open('wrong_log.txt', 'a', encoding='utf-8') as f:
string = 'page {} doc {} didnt write in '.format(page_num, i)
f.write(string)
f.write("\n")
print("write files fails...")
else:
# 取text
if t.read(element_identifier='//div[@id = "zoom"]') != '':
text = t.read(element_identifier='//div[@id = "zoom"]')
with open(file_name, 'w', encoding='utf-8') as f:
f.write(text)
elif t.read(element_identifier='//td[@class = "p1"]') != '':
text = t.read(element_identifier='//td[@class = "p1"]')
with open(file_name, 'w', encoding='utf-8') as f:
f.write(text)
else:
with open('wrong_log.txt', 'a', encoding='utf-8') as f:
string = 'page {} doc {} didnt write in '.format(page_num, i)
f.write(string)
f.write("\n")
print("write files fails...")
print("文件{} 含有 {} 个文件要下载。".format(i, pdf_count))
pdf_count += 1 # python从0开始，所以至少有一个pdf count
current_count = 0
for j in range(1, pdf_count):
# 取pdf的名字
if '.htm' not in t.read(
element_identifier='//div[@id = "zoom"]//p//a/@href'):
print("当前是第{}个文件。。".format(j))
p_count = t.count(element_identifier='//div[@id = "zoom"]//p')
while current_count <= p_count:
try:
if t.read(element_identifier=
'//div[@id = "zoom"]//p[last()-' +
str(current_count) + ']//a') != '':
# 如果取到了
print("这个p有!")
pdf_name = t.read(
element_identifier=
'//div[@id = "zoom"]//p[last()-' +
str(current_count) + ']//a/@href')
# 取合规名
pdf_name_to_change = t.read(
element_identifier=
'//div[@id = "zoom"]//p[last()-' +
str(current_count) + ']//a')
# 下载
suffix = pdf_name.split('.')[-1]
pdf_name = pdf_name.split('/')[-1]
prefix = 'http://www.pbc.gov.cn'
download_link = prefix + t.read(
element_identifier=
'//div[@id = "zoom"]//p[last()-' +
str(current_count) + ']//a/@href')
if 'cnhttp' in download_link:
t.url(
t.read(element_identifier=
'//div[@id = "zoom"]//p[last()-' +
str(current_count) + ']//a/@href'))
# 启动很慢
else:
t.url(download_link)
# 启动很慢
wait_seconds = 1
total_seconds = 0
while os.path.exists(pdf_name) == False:
t.wait(wait_seconds)
total_seconds += wait_seconds
if os.path.exists(pdf_name_to_change):
break
if total_seconds > MAX_WAIT:
print('download fails')
with open('download_log.txt',
'a',
encoding='utf-8') as f:
string = 'page {} doc {} file {} didnt download '.format(
page_num, i, j)
f.write(string)
f.write("\n")
break
if os.path.exists(pdf_name_to_change):
pass
else:
os.rename(pdf_name, pdf_name_to_change) # 改名
os.rename(
pdf_name_to_change,
pdf_name_to_change[:-(len(suffix) + 1)] +
'_' + time +
pdf_name_to_change[-(len(suffix) + 1):])
t.url(content_url) # 返回二级目录
# 启动很慢
current_count += 1
break
else:
current_count += 1
print("这个p没有")
except:
print('some error occurs, nvm')
continue
else:
print("是个网页，当文档处理！")
prefix = 'http://www.pbc.gov.cn'
download_link = prefix + t.read(
element_identifier='//div[@id = "zoom"]//p[' + str(j) +
']//a/@href')
if 'cnhttp' in download_link:
t.url(
t.read(element_identifier='//div[@id = "zoom"]//p[' +
str(j) + ']//a/@href'))
# 启动很慢
else:
t.url(download_link)
# 启动很慢
# 取text
if t.read(element_identifier='//div[@id = "zoom"]') != '':
text = t.read(element_identifier='//div[@id = "zoom"]')
with open(file_name, 'w', encoding='utf-8') as f:
f.write(text)
elif t.read(element_identifier='//td[@class = "p1"]') != '':
text = t.read(element_identifier='//td[@class = "p1"]')
with open(file_name, 'w', encoding='utf-8') as f:
f.write(text)
else:
with open('wrong_log.txt', 'a', encoding='utf-8') as f:
string = 'page {} doc {} didnt write in '.format(
page_num, i)
f.write(string)
f.write("\n")
print("write files fails...")
def getdailyincrement(str_to_append):
#初始化页面
t.init()
#输入url进入
t.url("http://bank.jrj.com.cn/bankpro/data.shtml?type=1")
#等5秒反应
t.wait(15)
#鼠标放上去，点击精简选项
t.hover(element_identifier='//*[@id="fxr"]')
t.click(element_identifier='//*[@id="zksq"]')
#鼠标移动到发行日期上，点击文本栏，输入发行日日期为今日，点击搜索
t.hover(element_identifier='//*[@id="fxr"]')
t.click(element_identifier='//*[@id="fxr"]')
t.type(element_identifier='//*[@id="fxr"]', text_to_type=str_to_append)
#再点击，确保日期不会遮住底下的搜索按钮
t.click(element_identifier='//*[@id="fxr"]')
t.hover(element_identifier='//*[@class="ipf01"]')
t.click(element_identifier='//*[@class="ipf01"]')
#把展示的尺寸设置为50个产品每页：
t.hover(element_identifier='//*[@data-pagesize="50"]')
t.click(element_identifier='//*[@data-pagesize="50"]')
#当下一页没有被disable的时候,有以下超参数
page_curr = 1 #当前页面index
value_dict = {} #存放data
count = 1 #csv 命名用
#存放列名
name_list = ['序号', '综合评级', 'url']
for col_name in name_list:
value_dict.setdefault(col_name, []) #初始化空数据集
#当可以翻页，或数据只有一页的时候，进行循环
while (t.read(element_identifier=
'//div[@id = "pagefoot"]//a[@class = "cur pf-disabled"]')
== str(page_curr)) or (page_curr == 1):
#每页的数据量大小（row number）
count_values = int(
t.count(element_identifier='//tbody[@id = "content"]//tr')
) + 1 # python从0开始
# 爬取页面所有一个table里的值
if str(
t.read(element_identifier='//tbody[@id = "content"]//tr[' +
str(count_values - 1) +
']//td[@class = "px"]')) > str_to_append:
# print("direct continue..")
# 翻页
page_curr += 1
# 鼠标模拟移动，并点击翻页
t.hover(element_identifier='//*[@href="' + str(page_curr) + '"]')
t.click(element_identifier='//*[@href="' + str(page_curr) + '"]')
continue
filename = str(count) + "daily_data.csv"
count += 1
t.wait(1) # 等1秒，万一加载错误了
t.table(
element_identifier='//div[@class = "table-s1 tab-s2 w100"]//table',
filename_to_save=filename)
count_values = int(
t.count(element_identifier='//tbody[@id = "content"]//tr')
) + 1 # python从0开始
for i in range(1, count_values):
# 判定条件：如果是今天刚发行的，拿到所有主页面上的数据；
#如果最下面那条数据都大于今天，就直接翻页
if str(
t.read(element_identifier='//tbody[@id = "content"]//tr[' +
str(count_values - 1) +
']//td[@class = "px"]')) > str_to_append:
# print("direct break..")
break
else:
if str(
t.read(
element_identifier='//tbody[@id = "content"]//tr['
+ str(i) +
']//td[@class = "px"]')) == str_to_append:
#序号
value_dict[name_list[0]].append(
t.read(
element_identifier='//tbody[@id = "content"]//tr['
+ str(i) + ']/td[2]'))
#综合评级
value_dict[name_list[1]].append(
t.read(
element_identifier='//tbody[@id = "content"]//tr['
+ str(i) + ']//td[12]//i/@title'))
#url
value_dict[name_list[2]].append(
t.read(
element_identifier='//tbody[@id = "content"]//tr['
+ str(i) + ']//a/@href'))
else: #如果不是今天增量，什么都不做
pass
# print("turn the page..")
# 翻页
page_curr += 1
# 鼠标模拟移动，并点击翻页
t.hover(element_identifier='//*[@href="' + str(page_curr) + '"]')
t.click(element_identifier='//*[@href="' + str(page_curr) + '"]')
#关闭tagui流
t.close()
#输出格式为："今日日期.csv"
today_data = pd.DataFrame(value_dict)
today_data.to_csv(str_to_append + ".csv", index=False, encoding='UTF-8')
return count - 1
def back_to_homelist(self):
click('//button[@aria-label="Close"]')
t.wait(1)
def getExpFlightPrice(airline, dep_ref, dur_ref):
print(airline)
print(dep_ref)
print(dur_ref)
util.wait_for_pageload('//input[@classes="filter-checkbox"]')
t.wait(3)
t.click(f'//a[@data-content-id="airlineToggleContainer"]')
for i in range(len(dep_ref)):
if i == 0:
if t.present(f'//input[@id="airlineRowContainer_{airline[i]}"]'):
t.wait(3)
t.click(f'//input[@id="airlineRowContainer_{airline[i]}"]')
else:
print('Not match')
return 0, ''
elif airline[i] != airline[i-1]:
if t.present(f'//input[@id="airlineRowContainer_{airline[i]}"]'):
t.wait(1)
t.click(f'//input[@id="airlineRowContainer_{airline[i]}"]')
else:
print('Not match')
return 0, ''
if dep_ref[i][0] == '0':
dep_ref[i] = dep_ref[i][1:]
if dur_ref[i][-1:] == 'h':
dur_ref[i] = dur_ref[i] + ' 0m'
else:
dur_ref[i] = dur_ref[i] + 'm'
print(airline)
print(dep_ref)
print(dur_ref)
util.wait_for_pageload('//button[@data-test-id="select-button"]')
t.wait(5)
for i in range(t.count(f'//ul[@id="flightModuleList"]//li')):
i = i + 1
print(i)
dep = t.read(f'(//span[@class="medium-bold"]//span[@data-test-id="departure-time"])[{i}]')
if len(dur_ref) == 1:
if dep == dep_ref[0]:
print('dep OK')
dur = t.read(f'(//span[@data-test-id="duration"])[{i}]')
t.click(f'(//button[@data-test-id="select-button"])[{i}]')
t.wait(5)
if t.present('//a[@id="forcedChoiceNoThanks"]'):
t.click(f'//a[@id="forcedChoiceNoThanks"]')
t.wait(5)
for x in range(5):
print(x)
if t.popup('Flight-Information?'):
break
else:
t.wait(5)
price = t.read(f'(//span[@class="packagePriceTotal"])[2]')
price = float(price.replace(',', '').replace('SG', '').replace('$', '').replace(' ', ''))
print(price)
url = t.url()
return price, url
else:
return 0, ''
elif len(dur_ref) == 2:
print('trip', len(dur_ref))
if dep == dep_ref[0]:
print('dep OK')
dur = t.read(f'(//span[@data-test-id="duration"])[{i}]')
t.click(f'(//button[@data-test-id="select-button"])[{i}]')
t.wait(5)
util.wait_for_pageload('//button[@data-test-id="select-button"]')
t.click(f'//input[@id="airlineRowContainer_{airline[1]}"]')
t.wait(2)
for j in range(t.count(f'//ul[@id="flightModuleList"]//li')):
j = j + 1
print(j)
dep = t.read(f'(//span[@data-test-id="departure-time"])[{j}+1]')
if dep == dep_ref[1]:
print('return dep ok')
dur = t.read(f'(//span[@data-test-id="duration"])[{j}+1]')
if dur == dur_ref[1]:
t.click(f'(//button[@data-test-id="select-button"])[{j}]')
t.wait(5)
if t.present('//a[@id="forcedChoiceNoThanks"]'):
t.click(f'//a[@id="forcedChoiceNoThanks"]')
t.wait(5)
for x in range(5):
print(x)
if t.popup('Flight-Information?'):
break
else:
t.wait(5)
util.wait_for_pageload('//h1[@class="section-header-main"]')
price = t.read(f'(//span[@class="packagePriceTotal"])[2]')
price = float(price.replace(',', '').replace('SG', '').replace('$', '').replace(' ', ''))
print(price)
url = t.url()
print(url)
return price, url
else:
return 0, ''
elif len(dur_ref) >= 3:
dep_lst = []
dur_lst = []
print('multi-trip ', len(dur_ref))
for k in range(len(dur_ref)):
dep_lst.append(t.read(f'(//span[@data-test-id="departure-time"])[{3*i+k+1}]'))
dur_lst.append(t.read(f'(//span[@data-test-id="duration"])[{3*i+k+1}]'))
print(dep_lst)
print(dep_ref)
if dep_lst == dep_ref:
print(dur_lst)
print(dur_ref)
if dur_lst == dur_ref:
t.click(f'(//button[@data-test-id="select-button"])[{j}]')
t.wait(5)
if t.present('//a[@id="forcedChoiceNoThanks"]'):
t.click(f'//a[@id="forcedChoiceNoThanks"]')
t.wait(5)
for x in range(5):
print(x)
if t.popup('Flight-Information?'):
break
else:
t.wait(5)
price = t.read(f'(//span[@class="packagePriceTotal"])[2]')
price = float(price.replace(',', '').replace('SG', '').replace('$', '').replace(' ', ''))
print(price)
url = t.url()
print(url)
return price, url
else:
return 0, ''
def wait_element(xpath, timeout=7):
for i in range(timeout):
if t.present(xpath):
return True
t.wait(1)
return False
import tagui as t
import datetime
import pandas as pd
import os
import s3_function
#超参数
try:
str_to_append = str(datetime.datetime.today().date())
# 初始化页面
t.init()
# 输入url进入
t.url("http://bank.jrj.com.cn/bankpro/data.shtml?type=1")
# 等15秒反应
t.wait(15)
# 鼠标放上去，点击精简选项
t.hover(element_identifier='//*[@id="fxr"]')
t.click(element_identifier='//*[@id="zksq"]')
# 鼠标移动到发行日期上，点击文本栏，输入发行日日期为今日，点击搜索
t.hover(element_identifier='//*[@id="fxr"]')
t.click(element_identifier='//*[@id="fxr"]')
t.type(element_identifier='//*[@id="fxr"]', text_to_type=str_to_append)
# 再点击，确保日期不会遮住底下的搜索按钮
t.click(element_identifier='//*[@id="fxr"]')
t.hover(element_identifier='//*[@class="ipf01"]')
t.click(element_identifier='//*[@class="ipf01"]')
# 把展示的尺寸设置为50个产品每页：
t.hover(element_identifier='//*[@data-pagesize="50"]')
t.click(element_identifier='//*[@data-pagesize="50"]')
# 当下一页没有被disable的时候,有以下超参数
page_curr = 1 # 当前页面index
