def parse_html(html):
soup = BeautifulSoup(html)
data = {
"last_updated": convert_date(soup.p.string, "(%d.%m.%Y, %H.%M Uhr)"),
"lots": []
}
# get all lots
raw_lots = soup.find_all("tr")
for lot in raw_lots:
elements = lot.find_all("td")
lot_name = elements[0].text
data["lots"].append({
"name": lot_name,
"free": int(elements[1].text),
"count": total_number_map.get(lot_name, 0),
"type": type_map.get(lot_name, "unbekannt"),
"coords": geodata.coords(lot_name)
})
return data
def parse_html(html):
soup = BeautifulSoup(html)
# Do everything necessary to scrape the contents of the html
# into a dictionary of the format specified by the schema.
data = {
"last_updated": convert_date("date_string", "date_format"),
"lots": []
}
return data
def parse_html(html):
soup = BeautifulSoup(html)
data = {
"lots": []
}
# Letzte Aktualisierung auslesen, ich liebe html parsing m(
last_updated = soup.find("ul", {"class": "links"}).findNext("p").text.strip()
data["last_updated"] = convert_date(last_updated, "%d.%m.%Y %H.%M Uhr")
# Die einzelnen Stadteile sind in einzelne tables gegliedert
section_tables = soup.find_all("tbody")
for table in section_tables:
# jeder parkplatz steckt in einer eigenen row
rows = table.find_all("tr")
for row in rows:
raw_lot_data = row.find_all("td")
name = raw_lot_data[0].find("a").text
id = raw_lot_data[0].find("a")["href"][-4:]
state = status_image_map.get(raw_lot_data[0].find("img")["src"], "nodata")
coords = geodata.coords(name)
count = raw_lot_data[1].text
count = count.strip()
if count == "":
count = 0
count = int(count)
free = raw_lot_data[2].text
free = free.strip()
if free == "":
free = 0
free = int(free)
data["lots"].append({
"name": name,
"coords": coords,
"id": id,
"state": state,
"free": free,
"count": count
})
return data
def calendar_event_create(self, **info):
date = util.convert_date(info["publisher"], info["date"], "google")
event = {
"summary": info["title"],
"description" : "added by comiCal.py",
"start": {
"date" : date
},
"end": {
"date" : date
}
}
created_event = self.g_api_obj.events().insert(calendarId=self.comiCal_calendar_id, body=event).execute()
return created_event['id']
def parse_html(html):
soup = BeautifulSoup(html)
data = {
"last_updated": convert_date(soup.find("tr").find("strong").text, "Stand: %d.%m.%Y, %H:%M Uhr"),
"lots": []
}
rows = soup.find_all("tr")
rows = rows[1:]
region_header = ""
for row in rows:
if len(row.find_all("th")) > 0:
# This is a header row, save it for later
region_header = row.find("th", {"class": "head1"}).text
else:
if row.find("td").text == "Gesamt":
continue
# This is a parking lot row
raw_lot_data = row.find_all("td")
if len(raw_lot_data) == 2:
type_and_name = process_name(raw_lot_data[0].text)
data["lots"].append({
"name": type_and_name[1],
"type": type_and_name[0],
"count": 0,
"free": 0,
"region": region_header,
"state": process_state_map.get(raw_lot_data[1].text, "")
})
elif len(raw_lot_data) == 4:
type_and_name = process_name(raw_lot_data[0].text)
data["lots"].append({
"name": type_and_name[1],
"type": type_and_name[0],
"count": int(raw_lot_data[1].text),
"free": int(raw_lot_data[2].text),
"region": region_header,
"state": "open"
})
return data
def parse_html(html):
soup = BeautifulSoup(html)
lot_table_trs = soup.select("table[cellpadding=5]")[0].find_all("tr")
data = {
"last_updated": convert_date(lot_table_trs[-1].text.strip(), "%d.%m.%Y %H:%M Uhr"),
"lots": []
}
for tr in lot_table_trs[1:-1]:
tds = tr.find_all("td")
type_and_name = process_name(tds[0].text)
data["lots"].append({
"name": type_and_name[1],
"type": type_and_name[0],
"free": int(tds[1].text),
"count": total_number_map.get(tds[0].text, 0),
"state": state_map.get(tds[2].text, "")
})
return data
def calendar_search(self, publisher, title, latest_release_date):
try:
latest_release_date_gcal = util.convert_date(publisher, latest_release_date, "google")
results = self.g_api_obj.events().list(calendarId=self.comiCal_calendar_id,
q=title).execute()
# ensure correct event
result = None
for found in results["items"]:
if title == found["summary"]:
result = found
break
result_title = result["summary"]
result_date = result["start"]['date']
if result != None and latest_release_date_gcal != result_date:
return {
"action" : "update",
"new_date": latest_release_date_gcal,
"event_id": result["id"]
}
else:
return {
"action": None,
"date" : result_date
}
except (IndexError, TypeError) as e:
return {
"action": "create",
"title" : title,
"date" : latest_release_date_gcal
}
return e
except Exception as e:
print "unknown exception in calendar_search"
print e
def make_table(race_no, race_info, table_results, table_awards, table_racecard,
bet_info):
table = table_results
# -----------------
# combine race info
# -----------------
for i, row in enumerate(table):
if i == 0:
table[i].insert(0, "賽道")
table[i].insert(0, "場地")
table[i].insert(0, "分數範圍")
table[i].insert(0, "長度")
table[i].insert(0, "班次")
table[i].insert(0, "場次")
table[i].insert(0, "日期")
else:
tags = race_info["tag"].split(" - ")
y, m, d = util.convert_date(bet_info["date"])
# 賽道
combined = ""
if bet_info["place"] == "ST":
combined += "田"
else:
combined += "谷"
track = race_info["track"][5:]
if track[0] == "草":
combined += "草"
combined += track.split("\"")[1]
else:
combined += "泥"
table[i].insert(0, combined)
# 場地
condition = race_info["cond"][7:]
if condition == "好地":
table[i].insert(0, "好")
elif condition == "好地至快地":
table[i].insert(0, "好至快")
elif condition == "好地至黏地":
table[i].insert(0, "好至黏")
elif condition == "濕慢地":
table[i].insert(0, "濕慢")
else:
table[i].insert(0, condition)
# 分數範圍
if len(tags) > 2:
table[i].insert(0, tags[2])
else:
table[i].insert(0, '')
# 長度
table[i].insert(0, tags[1])
# 班次
if tags[0][0] == "第":
if len(tags[0]) == 3:
table[i].insert(0, tags[0][1:2])
else:
table[i].insert(0, tags[0][1:2] + "*")
elif tags[0][0] == "國":
table[i].insert(0, tags[0][2:])
elif tags[0][0] == "條":
table[i].insert(0, "*")
else:
table[i].insert(0, tags[0])
# 場次
table[i].insert(0, race_no)
# 日期
table[i].insert(0, "{}/{}/{}".format(y, m, d))
# ----------------
# combine hot info
# ----------------
sort_arr = []
for row in table:
if util.is_float(row[-1]):
sort_arr.append(float(row[-1]))
list.sort(sort_arr)
hot_flag = False
for i, row in enumerate(table):
if i == 0:
table[i].append("熱門")
else:
if util.is_float(row[-1]):
if math.isclose(float(row[-1]), sort_arr[0],
rel_tol=1e-9) and not hot_flag: # 1st hot
table[i].append("1st Hot")
hot_flag = True
elif math.isclose(float(row[-1]), sort_arr[1],
rel_tol=1e-9): # 2nd hot
table[i].append("2nd Hot")
else:
table[i].append("-")
else:
table[i].append("-")
# ----------------
# combine bet info
# ----------------
have_bet = False
thead = table[0]
for i, bet in enumerate(bet_info["bet"]):
if bet["id"] == race_no:
have_bet = True
for j, row in enumerate(table):
# append bet for this row
horse_number = row[thead.index("馬號")]
if j == 0:
table[j].append("投注")
elif not util.is_int(horse_number):
table[j].append("-")
elif int(horse_number) == bet["WP"]:
table[j].append("W P")
# see which WP according to number of Bigs
if len(bet["Big"]) > 1:
for k in range(len(bet["Big"])):
table[j][-1] += " Big{}(PQ)".format(k + 1)
elif len(bet["Big"]) != 0:
table[j][-1] += " Big(PQ)"
elif int(horse_number) in bet["Big"]:
# see which Big it is
if len(bet["Big"]) != 1:
for k in range(len(bet["Big"])):
if int(horse_number) == bet["Big"][k]:
table[j].append("Big{}(PQ)".format(k + 1))
else:
table[j].append("Big(PQ)")
else:
table[j].append("-")
if not have_bet:
for j, row in enumerate(table):
if j == 0:
table[j].append("投注")
else:
table[j].append("-")
# ---------------------
# combine racecard info
# ---------------------
thead = table[0]
col_horse_no = thead.index("馬號")
for i, row in enumerate(table):
if i == 0:
table[i].append("皇牌")
table[i].append("配備")
table[i].append("馬齡")
table[i].append("評分")
table[i].append("評分+/-")
else:
# print(table_racecard[horse_number])
if row[1] != '' and util.is_int(row[col_horse_no]):
horse_number = int(row[col_horse_no])
table[i].append(table_racecard[horse_number][-6]) # 優先參賽次序
table[i].append(table_racecard[horse_number][-5]) # 配備
else:
table[i].append('-')
table[i].append('-')
horse_id = table[i][9].split('(')[1][:-1]
table[i].append(get_age(link_horseinfo, horse_id)) # 馬齡
if row[1] != '' and util.is_int(row[col_horse_no]):
horse_number = int(row[col_horse_no])
table[i].append(table_racecard[horse_number][10]) # 優先參賽次序
table[i].append(table_racecard[horse_number][11]) # 配備
else:
table[i].append('-')
table[i].append('-')
# -------------------
# combine place & ddy
# -------------------
for i, row in enumerate(table):
if i == 0:
table[i].append("地點")
table[i].append("度地儀")
else:
if bet_info["place"] == "ST":
table[i].append("沙田")
else:
table[i].append("跑馬地")
table[i].append(bet_info["ddy"])
# ------------
# combine odds
# ------------
for i, row in enumerate(table):
if i == 0:
table[i].append("P賠率")
table[i].append("P賠率2")
table[i].append("P賠率3")
table[i].append("Queue賠率")
table[i].append("PQ賠率")
table[i].append("PQ賠率2")
table[i].append("PQ賠率3")
else:
p_awards = table_awards[2][1]
q_awards = table_awards[3][1]
pq_awards = table_awards[4][1]
# P1/2/3
for j in range(len(p_awards)):
if p_awards[j][0] == table[i][col_horse_no]:
table[i].append(util.str_to_float(p_awards[j][1]))
else:
table[i].append('')
# Queue
for j in range(len(q_awards)):
horse_number = q_awards[j][0].split(',')
if horse_number[0] == table[i][col_horse_no] or horse_number[
1] == table[i][col_horse_no]:
table[i].append(util.str_to_float(q_awards[j][1]))
else:
table[i].append('')
# Pos-Queue
for j in range(len(pq_awards)):
horse_number = pq_awards[j][0].split(',')
if horse_number[0] == table[i][col_horse_no] or horse_number[
1] == table[i][col_horse_no]:
table[i].append(util.str_to_float(pq_awards[j][1]))
else:
table[i].append('')
return table
def main():
print("* get bet info")
bet_info = race_io.get_betinfo('bet.json')
race_date, race_place, number_of_races = bet_info['date'], bet_info[
'place'], bet_info['races']
print("* get race info from", link_raceinfo)
# all_race_info = race_io.get_raceinfo(link_results)
tables_main = []
tables_report = []
for i in range(1, number_of_races + 1):
link_i_results = link_results + "/" + race_date + "/" + race_place + "/" + str(
i)
link_i_racecard = link_racecard + "/" + race_date + "/" + race_place + "/" + str(
i)
# input
print("* get results from", link_i_results)
race_info, table_results, table_awards = race_io.get_results(
link_i_results)
print("* get racecard from", link_i_racecard)
table_racecard = race_io.get_racecard(link_i_racecard)
# DEBUG: show input
# print(race_info)
# util.print_table(table_results)
# util.print_table(table_awards)
# util.print_table(table_racecard)
# process
print("* processing main table {}".format(i))
table_main = make_main.make_table(race_no=i,
race_info=race_info,
bet_info=bet_info,
table_results=table_results,
table_awards=table_awards,
table_racecard=table_racecard)
print("* processing report {}".format(i))
table_report, result_info = make_report.make_table(
race_no=i,
race_info=race_info,
bet_info=bet_info,
table_awards=table_awards,
table_main=table_main)
# DEBUG: show processing output
# util.write_table(table_main, "table_main.csv")
# util.write_table(table_report, "table_report.csv")
# util.print_table(table_main)
# util.print_table(table_report)
# output tables
print("* append table to the table list")
tables_main.append(table_main)
tables_report.append((table_report, result_info))
print("* combine all tables")
combined_main = make_main.combine_tables(tables_main)
combined_report = make_report.combine_tables(tables_report, bet_info)
# DEBUG: show combined tables
# util.write_table(combined_main, "table_main.csv")
# util.write_table(combined_report, "table_report.csv")
print("* output main & report")
y, m, d = util.convert_date(bet_info["date"])
# backup files
shutil.copyfile("output/Data Base (2018-2019).csv",
"backup/Data Base (2018-2019) pre-{}-{}.csv".format(
m, d)) # main table
make_main.output(combined_main, "output/Data Base (2018-2019).csv")
make_report.output(combined_report,
"output/Horse Report {}-{}-{}.csv".format(y, m, d))
def handle_text_message(event):
conn = model.Conn()
line_user_id = event.source.user_id
text = event.message.text
session_bytes = redis.get(line_user_id)
session = {}
if session_bytes is not None:
session = json.loads(session_bytes.decode("utf-8"))
print('\n\n\nHERE, request event:', event)
print("\n\n\nHERE, session:", session)
if session == {}:
print("\n\n\nHERE # USER PERTAMA KALI BUKA")
redis.set(line_user_id, json.dumps({'status': 'login'}))
remove_rich_menu(line_user_id)
line_bot_api.reply_message(event.reply_token, [
TextMessage(text=constant.WELCOME_APP),
TextMessage(text=constant.LOGIN)
])
else:
if 'login' in session['status']:
print("\n\n\nHERE # PROSES LOGIN")
text = text.replace(' ', '')
texts = text.split('-')
if len(texts) != 2: # VALIDASI LOGIN GAGAL
line_bot_api.reply_message(event.reply_token, [
TextMessage(text=constant.LOGIN_VALIDATION_FAIL),
TextMessage(text=constant.LOGIN)
])
else:
nisn = texts[0]
dob = texts[1]
if util.validate_date(dob, '%d%m%Y'):
query_select = 'SELECT * FROM student WHERE nisn = %s AND dob = %s LIMIT 1'
conn.query(
query_select,
(nisn, util.convert_date(dob, '%d%m%Y', '%Y-%m-%d')))
row = conn.cursor.fetchone()
if row == None: # LOGIN GAGAL
line_bot_api.reply_message(event.reply_token, [
TextMessage(text=constant.LOGIN_FAIL),
TextMessage(text=constant.LOGIN)
])
else:
print("\n\n\nHERE # LOGIN BERHASIL")
# create rich menu
rich_menu = create_rich_menu(line_user_id)
line_bot_api.link_rich_menu_to_user(
line_user_id, rich_menu['home'])
redis.set(
line_user_id,
json.dumps({
'user_id': row['id'],
'nisn': row['nisn'],
'name': row['name'],
'class_id': row['class_id'],
'status': 'home',
'rich_menu': rich_menu
}))
line_bot_api.reply_message(event.reply_token, [
TextMessage(text=constant.WELCOME_HOME %
(row['name']), )
])
else:
print("\n\n\nHERE # VALIDASI LOGIN GAGAL")
line_bot_api.reply_message(event.reply_token, [
TextMessage(text=constant.LOGIN_VALIDATION_FAIL),
TextMessage(text=constant.LOGIN)
])
else:
# to handle save reply on discussion
if 'material_discussion' in session:
query_insert_discussion = 'INSERT INTO class_discussion_detail (class_discussion_id, description, user_id, student_id, date) VALUES (%s, %s, %s, %s, NOW())'
insert_discussion = (
session['material_discussion']['class_discussion_id'],
text, session['material_discussion']['user_id'],
session['material_discussion']['student_id'])
conn.query(query_insert_discussion, insert_discussion)
conn.commit()
redis.set(
line_user_id,
json.dumps({
'user_id': session['user_id'],
'nisn': session['nisn'],
'name': session['name'],
'class_id': session['class_id'],
'status': 'home',
'rich_menu': session['rich_menu']
}))
flex_messages = []
flex_message_material_topic = show_material_topic(
event, conn, session['material_discussion'])
flex_messages.append(flex_message_material_topic)
line_bot_api.reply_message(event.reply_token, flex_messages)
try:
convert_SEARCH_COUNT = util.convert_search_count(
data[b'0:SEARCH_COUNT'])
store_file['SEARCH_COUNT'] = convert_SEARCH_COUNT
# store_file['SEARCH_COUNT'] = data[b'0:SEARCH_COUNT']
except:
store_file['SEARCH_COUNT'] = "NaN"
try:
store_file['SEARCH_KEYWORD'] = data[b'0:SEARCH_KEYWORD'].decode(
"utf-8")
except:
store_file['SEARCH_KEYWORD'] = "NaN"
try:
convert_SEARCH_DATE = util.convert_date(data[b'0:SEARCH_DATE'])
store_file['SEARCH_DATE'] = convert_SEARCH_DATE
except:
store_file['SEARCH_DATE'] = "NaN"
try:
convert_SYSUPDATE_DATE = util.convert_date(data[b'0:SYSUPDATE_DATE'])
store_file['SYSUPDATE_DATE'] = convert_SYSUPDATE_DATE
except:
store_file['SYSUPDATE_DATE'] = "NaN"
try:
store_file['SEARCH_ACTION'] = data[b'0:SEARCH_ACTION'].decode("utf-8")
except:
store_file['SEARCH_ACTION'] = "NaN"
json_output.append(store_file.copy())
def make_table(race_no, race_info, table_results, table_awards, table_racecard, bet_info):
table = table_results
# -----------------
# combine race info
# -----------------
for i, row in enumerate(table):
if i == 0:
table[i].insert(0, "賽道")
table[i].insert(0, "場地")
table[i].insert(0, "分數範圍")
table[i].insert(0, "長度")
table[i].insert(0, "班次")
table[i].insert(0, "場次")
table[i].insert(0, "日期")
else:
tags = race_info["tag"].split(" - ")
y, m, d = util.convert_date(bet_info["date"])
table[i].insert(0, race_info["track"][5:])
table[i].insert(0, race_info["cond"][7:])
# 分數範圍 sometimes does not exist
if len(tags) > 2:
table[i].insert(0, tags[2])
else:
table[i].insert(0, '')
table[i].insert(0, tags[1])
table[i].insert(0, tags[0])
table[i].insert(0, race_no)
table[i].insert(0, "{}/{}/{}".format(y, m, d))
# ----------------
# combine hot info
# ----------------
sort_arr = []
for row in table:
if util.is_float(row[-1]):
sort_arr.append(float(row[-1]))
list.sort(sort_arr)
hot_flag = False
for i, row in enumerate(table):
if i == 0:
table[i].append("熱門")
else:
if util.is_float(row[-1]):
if math.isclose(float(row[-1]), sort_arr[0], rel_tol=1e-9) and not hot_flag: # 1st hot
table[i].append("1st Hot")
hot_flag = True
elif math.isclose(float(row[-1]), sort_arr[1], rel_tol=1e-9): # 2nd hot
table[i].append("2nd Hot")
else:
table[i].append("-")
else:
table[i].append("-")
# ----------------
# combine bet info
# ----------------
have_bet = False
thead = table[0]
for i, bet in enumerate(bet_info["bet"]):
if bet["id"] == race_no:
have_bet = True
for j, row in enumerate(table):
# append bet for this row
horse_number = row[thead.index("馬號")]
if j == 0:
table[j].append("投注")
elif not util.is_int(horse_number):
table[j].append("-")
elif int(horse_number) == bet["WP"]:
table[j].append("W P")
# see which WP according to number of Bigs
if len(bet["Big"]) > 1:
for k in range(len(bet["Big"])):
table[j][-1] += " Big{}(PQ)".format(k+1)
elif len(bet["Big"]) != 0:
table[j][-1] += " Big(PQ)"
elif int(horse_number) in bet["Big"]:
# see which Big it is
if len(bet["Big"]) != 1:
for k in range(len(bet["Big"])):
if int(horse_number) == bet["Big"][k]:
table[j].append("Big{}(PQ)".format(k+1))
else:
table[j].append("Big(PQ)")
else:
table[j].append("-")
if not have_bet:
for j, row in enumerate(table):
if j == 0:
table[j].append("投注")
else:
table[j].append("-")
# ---------------------
# combine racecard info
# ---------------------
thead = table[0]
col_horse_no = thead.index("馬號")
for i, row in enumerate(table):
if i == 0:
table[i].append("皇牌")
table[i].append("配備")
table[i].append("操練")
else:
if row[1] != '' and util.is_int(row[col_horse_no]):
horse_number = int(row[col_horse_no])
table[i].append(table_racecard[horse_number][-6]) # 優先參賽次序
table[i].append(table_racecard[horse_number][-5]) # 配備
else:
table[i].append('-')
table[i].append('-')
# TODO TODO TODO
table[i].append('') # 操練
# -------------------
# combine place & ddy
# -------------------
for i, row in enumerate(table):
if i == 0:
table[i].append("地點")
table[i].append("度地儀")
else:
if bet_info["place"] == "ST":
table[i].append("沙田")
else:
table[i].append("跑馬地")
table[i].append(bet_info["ddy"])
# ------------
# combine odds
# ------------
for i, row in enumerate(table):
if i == 0:
table[i].append("P賠率")
table[i].append("P賠率2")
table[i].append("P賠率3")
table[i].append("Queue賠率")
table[i].append("PQ賠率")
table[i].append("PQ賠率2")
table[i].append("PQ賠率3")
else:
p_awards = table_awards[2][1]
q_awards = table_awards[3][1]
pq_awards = table_awards[4][1]
# P1/2/3
for j in range(len(p_awards)):
if p_awards[j][0] == table[i][col_horse_no]:
table[i].append(util.str_to_float(p_awards[j][1]))
else: table[i].append('')
# Queue
for j in range(len(q_awards)):
horse_number = q_awards[j][0].split(',')
if horse_number[0] == table[i][col_horse_no] or horse_number[1] == table[i][col_horse_no]:
table[i].append(util.str_to_float(q_awards[j][1]))
else: table[i].append('')
# Pos-Queue
for j in range(len(pq_awards)):
horse_number = pq_awards[j][0].split(',')
if horse_number[0] == table[i][col_horse_no] or horse_number[1] == table[i][col_horse_no]:
table[i].append(util.str_to_float(pq_awards[j][1]))
else: table[i].append('')
return table