def newMax(type1):
nonlocal maxVar
if maxVar == None or mtl.typeRank(type1) > mtl.typeRank(maxVar):
maxVar = type1
def contextizeVar(varStatement):
if utl.nextStatement().kind == "keyword":
typ = utl.nextToken()
name = utl.nextToken()
if name in varTable:
raise utl.ContextException(name + " declared twice")
varTable[name] = typ
utl.nextToken() #skip the "="
utl.nextStatement() #skip to expression statement
expType = contextizeExpression(utl.nextStatement())
if mtl.typeRank(expType) > mtl.typeRank(typ):
mtl.compilerWarnings.append(
"possible loss of information when initializing " + name)
else:
name = utl.nextToken()
utl.nextToken() #skip the "="
utl.nextStatement() #skip to expression statement
expType = contextizeExpression(utl.nextStatement())
# case for assigning new value to old variable
if name in varTable:
if mtl.typeRank(expType) > mtl.typeRank(varTable[name]):
mtl.compilerWarnings.append(
"possible loss of information when assigning value to " +
name)
# case for creation of new variable
else:
#add type token to var dec
s = Statement("keyword")
s.subList.append(expType)
s.parent = varStatement
varStatement.subList.insert(0, s)
#add to symbol table
varTable[name] = expType
def readFile(fileName):
rawInput.clear()
try:
file = open(fileName, "r")
except FileNotFoundError:
raise mtl.CompileException("File not found")
for line in file:
for char in line:
rawInput.append(char)
rawInput.append("EOF")
import json
import MainUtil
resources_file_path = '/resources/airplane/airportNameList.ini'
scratch_url_old = 'https://data.variflight.com/profiles/profilesapi/search'
scratch_url = 'https://data.variflight.com/analytics/codeapi/initialList'
get_city_url = 'https://data.variflight.com/profiles/Airports/%s'
# 传入查找网页的url和旧数据，然后本方法会比对原数据中是否有新的条目，如果有则不加入，如果没有则重新加入，最后返回新数据
def scratch_airport_name(scratch_url, old_airports):
new_airports = []
data = requests.get(scratch_url).text
all_airport_json = json.loads(data)['data']
for airport_by_word in all_airport_json.values():
for airport in airport_by_word:
if airport['fn'] not in old_airports:
get_city_uri = get_city_url % airport['id']
data2 = requests.get(get_city_uri).text
soup = bs4.BeautifulSoup(data2, "html.parser")
city = soup.find("span",class_="heig").find_next_sibling()['title']
new_airports.append(city + ',' + airport['fn'])
print(city + "," + airport['fn'])
return new_airports
# main方法，执行这个py，默认调用main方法，相当于java的main
if __name__ == '__main__':
MainUtil.main(resources_file_path, scratch_url, scratch_airport_name)
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import requests
import bs4
import MainUtil
resources_file_path = '/resources/airplane/flightNameList.ini'
scratch_url = 'http://www.variflight.com/sitemap.html?AE71649A58c77='
def scratch_flight_number(scratch_url, old_flights):
new_flights = []
data = requests.get(scratch_url).text
soup = bs4.BeautifulSoup(data, "html.parser")
a_flights = soup.find('div', class_='list').find_all('a', recursive=False)
for flight in a_flights:
if flight.text not in old_flights and flight.text != '国内航段列表':
new_flights.append(flight.text)
return new_flights
if __name__ == '__main__':
MainUtil.main(resources_file_path, scratch_url, scratch_flight_number)
scratch_url = 'http://www.smskb.com/train/'
def scratch_train_station(scratch_url, old_stations):
new_stations = []
provinces_eng = ("Anhui", "Beijing", "Chongqing", "Fujian", "Gansu",
"Guangdong", "Guangxi", "Guizhou", "Hainan", "Hebei",
"Heilongjiang", "Henan", "Hubei", "Hunan", "Jiangsu",
"Jiangxi", "Jilin", "Liaoning", "Ningxia", "Qinghai",
"Shandong", "Shanghai", "Shanxi", "Shanxisheng",
"Sichuan", "Tianjin", "Neimenggu", "Xianggang",
"Xinjiang", "Xizang", "Yunnan", "Zhejiang")
provinces_chi = ("安徽", "北京", "重庆", "福建", "甘肃", "广东", "广西", "贵州", "海南",
"河北", "黑龙江", "河南", "湖北", "湖南", "江苏", "江西", "吉林", "辽宁",
"宁夏", "青海", "山东", "上海", "陕西", "山西", "四川", "天津", "内蒙古",
"香港", "新疆", "西藏", "云南", "浙江")
for i in range(0, provinces_eng.__len__(), 1):
cur_url = scratch_url + provinces_eng[i] + ".htm"
resp = requests.get(cur_url)
data = resp.text.encode(resp.encoding).decode('gbk')
soup = bs4.BeautifulSoup(data, "html.parser")
a_stations = soup.find('left').find('table').find_all('a')
for station in a_stations:
if station.text not in old_stations:
new_stations.append(provinces_chi[i] + ',' + station.text)
return new_stations
if __name__ == '__main__':
MainUtil.main(resources_file_path, scratch_url, scratch_train_station)
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import requests
import bs4
import json
import MainUtil
resources_file_path = '/resources/train/trainNameList.ini'
scratch_url = 'http://www.59178.com/checi/'
def scratch_train_number(scratch_url, old_trains):
new_trains = []
resp = requests.get(scratch_url)
data = resp.text.encode(resp.encoding).decode('gb2312')
soup = bs4.BeautifulSoup(data, "html.parser")
a_trains = soup.find('table').find_all('a')
for train in a_trains:
if train.text not in old_trains and train.text:
new_trains.append(train.text)
return new_trains
if __name__ == '__main__':
MainUtil.main(resources_file_path, scratch_url, scratch_train_number)
# 相对路径，也是需要将此路径存入数据库
resources_file_path = '/resources/movie/cinemaNameList.ini'
scratch_url = 'http://theater.mtime.com/China_Beijing/'
# scratch data with define url
def scratch_latest_movies(scratch_url, old_movies):
data = requests.get(scratch_url).text
soup = bs4.BeautifulSoup(data, "html.parser")
new_movies = []
new_movies_json = json.loads(
soup.find('script', text=re.compile("var hotplaySvList")).text.split("=")[1].replace(";", ""))
coming_movies_data = soup.find_all('li', class_='i_wantmovie')
# 上映的电影
for movie in new_movies_json:
move_name = movie['Title']
if move_name not in old_movies:
new_movies.append(movie['Title'])
print("上映中:" + movie['Title'])
# 即将上映的电影
for coming_movie in coming_movies_data:
coming_movie_name = coming_movie.h3.a.text
if coming_movie_name not in old_movies and coming_movie_name not in new_movies:
new_movies.append(coming_movie_name)
print("即将上映：" + coming_movie_name)
return new_movies
if __name__ == '__main__':
MainUtil.main(resources_file_path, scratch_url, scratch_latest_movies)
import MainUtil
# 相对路径，也是需要将此路径存入数据库
resources_file_path = '/resources/movie/cinemaNameList.ini'
scratch_url = 'http://theater.mtime.com/China_Beijing/'
# scratch data with define url
def scratch_latest_movies(scratch_url, old_movies):
data = requests.get(scratch_url).text
soup = bs4.BeautifulSoup(data, "html.parser")
new_movies = []
new_movies_json = json.loads(
soup.find('script', text=re.compile("var hotplaySvList")).text.split("=")[1].replace(";", ""))
coming_movies_data = soup.find_all('li', class_='i_wantmovie')
# 上映的电影
for movie in new_movies_json:
move_name = movie['Title']
if move_name not in old_movies:
new_movies.append(movie['Title'])
# 即将上映的电影
for coming_movie in coming_movies_data:
coming_movie_name = coming_movie.h3.a.text
if coming_movie_name not in old_movies and coming_movie_name not in new_movies:
new_movies.append(coming_movie_name)
return new_movies
if __name__ == '__main__':
MainUtil.main(resources_file_path, scratch_url, scratch_latest_movies)
# -*- coding: UTF-8 -*-
import requests
import bs4
import json
import MainUtil
resources_file_path = '/resources/airplane/airportNameList.ini'
scratch_url = 'https://data.variflight.com/profiles/profilesapi/search'
def scratch_airport_name(scratch_url, old_airports):
new_airports = []
data = requests.get(scratch_url).text
all_airport_json = json.loads(data)['data']['list']
for airport_by_word in all_airport_json.values():
for airport in airport_by_word:
if airport['profiles_city'] and airport['profiles_name'] not in old_airports:
new_airports.append(airport['profiles_city'] + ',' + airport['profiles_name'])
return new_airports
if __name__ == '__main__':
MainUtil.main(resources_file_path, scratch_url, scratch_airport_name)
scratch_url = 'http://www.smskb.com/train/'
def scratch_train_station(scratch_url, old_stations):
new_stations = []
provinces_eng = (
"Anhui", "Beijing", "Chongqing", "Fujian", "Gansu", "Guangdong", "Guangxi", "Guizhou", "Hainan", "Hebei",
"Heilongjiang", "Henan", "Hubei", "Hunan", "Jiangsu", "Jiangxi", "Jilin", "Liaoning", "Ningxia", "Qinghai",
"Shandong", "Shanghai", "Shanxi", "Shanxisheng", "Sichuan", "Tianjin", "Neimenggu", "Xianggang", "Xinjiang",
"Xizang",
"Yunnan", "Zhejiang")
provinces_chi = (
"安徽", "北京", "重庆", "福建", "甘肃", "广东", "广西", "贵州", "海南", "河北",
"黑龙江", "河南", "湖北", "湖南", "江苏", "江西", "吉林", "辽宁", "宁夏", "青海",
"山东", "上海", "陕西", "山西", "四川", "天津", "内蒙古", "香港", "新疆", "西藏",
"云南", "浙江")
for i in range(0, provinces_eng.__len__(), 1):
cur_url = scratch_url + provinces_eng[i] + ".htm"
resp = requests.get(cur_url)
data = resp.text.encode(resp.encoding).decode('gbk')
soup = bs4.BeautifulSoup(data, "html.parser")
a_stations = soup.find('left').find('table').find_all('a')
for station in a_stations:
if station.text not in old_stations:
new_stations.append(provinces_chi[i] + ',' + station.text)
return new_stations
if __name__ == '__main__':
MainUtil.main(resources_file_path, scratch_url, scratch_train_station)