def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
response = requests.get("https://gotoeat-akita.com/csv/list.csv")
response.encoding = response.apparent_encoding
reader = pd.read_csv(io.BytesIO(response.content),
names=("merchant_name", "merchant_area", "merchant_address", "merchant_tel", "merchant_url"))
for row in reader.iterrows():
merchant_name = row[1]["merchant_name"]
merchant_name = str(re.sub(r"<!--.+-->", "", merchant_name)).strip()
merchant_area = str(row[1]["merchant_area"]).strip()
merchant_address = str(row[1]["merchant_address"]).strip()
if type(row[1]["merchant_tel"]) is str:
merchant_tel = str(row[1]["merchant_tel"]).strip()
else:
merchant_tel = None
if type(row[1]["merchant_url"]) is str:
merchant_url = str(row[1]["merchant_url"]).strip()
else:
merchant_url = None
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"area": merchant_area,
"address": merchant_address,
"tel": merchant_tel,
"url": merchant_url,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
time.sleep(1)
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://gotoeat-fukushima.jp/shop/page/{page}/?s".format(
page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
if (soup.find("ul", {"class": "list_search-result"}) == None):
break
lists = soup.find("ul", {
"class": "list_search-result"
}).findChildren("li", recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("h3", {"class": "result-name"}).text
merchant_address = merchant.find("p", {
"class": "result-address"
}).text
merchant_type = merchant.find("p", {"class": "result-cat"}).text
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
response = requests.get(
"https://static.batchgeo.com/map/json/bb0346d09d7761e1888e982c45c09cc9/1603655201"
)
startIndex = str(response.text).find("{")
if startIndex == -1:
print("Invalid JSON")
return
responseJson = str(response.text[startIndex:-1])
responseJson = json.loads(responseJson)
for key in range(len(responseJson["mapRS"])):
merchant_address = responseJson["mapRS"][key]["addr"]
merchant_address = re.sub(r"^([0-9\-]+) (.+)$", r"\2",
merchant_address)
lat = responseJson["mapRS"][key]["lt"]
lng = responseJson["mapRS"][key]["ln"]
merchant_name = responseJson["mapRS"][key]["t"]
merchant_name = responseJson["dataRS"][key][0]
merchant_postal_code = responseJson["dataRS"][key][1]
merchant_tel = responseJson["dataRS"][key][5]
merchant_time = responseJson["dataRS"][key][7]
merchant_regular_holiday = responseJson["dataRS"][key][8]
merchant_type = responseJson["dataRS"][key][9]
print(merchant_name + " " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"time": merchant_time,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
response = requests.get(
"https://gotoeat-kanagawa.liny.jp/map/api/data.json?x1=33&x2=36&y1=137&y2=141")
response.encoding = response.apparent_encoding
try:
jsondata = response.json()
except Exception as e:
print(e)
print(response.text)
exit(1)
for merchant in jsondata["data"]:
merchant_name = merchant["name"]
merchant_address = merchant["address"]
merchant_tel = merchant["tel"]
lat = merchant["latlng"]["lat"]
lng = merchant["latlng"]["lng"]
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"address": merchant_address,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": [], "categories": []}
findMerchants = []
response = requests.get("https://www.gotoeat-kochi.com/js/shop_list.php")
try:
jsondata = response.json()
except Exception as e:
print(e)
print(response.text)
exit(1)
for merchant in jsondata:
merchant_name = merchant[5]
merchant_area = merchant[1]
merchant_address = merchant[7]
merchant_type = merchant[3]
merchant_tel = merchant[8]
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"area": merchant_area,
"address": merchant_address,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
ret = get_fujinokuni(merchants, findMerchants)
merchants = ret["merchants"]
findMerchants = ret["findMerchants"]
ret = get_fujinokuni(merchants, findMerchants)
merchants = ret["merchants"]
findMerchants = ret["findMerchants"]
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
response = requests.get(
"https://gotoeat-fukuoka.jp/csv/fk_gotoeat_UTF-8.csv")
response.encoding = response.apparent_encoding
reader = pd.read_csv(io.BytesIO(response.content),
names=("merchant_id", "merchant_name", "merchant_name_kana", "merchant_type", "merchant_postal_code", "merchant_prefecture", "merchant_area", "merchant_address", "merchant_building_name", "merchant_tel", "merchant_url", "merchant_addedDate"))
for row in reader.iterrows():
if row[1]["merchant_id"] == "id":
continue
merchant_name = str(row[1]["merchant_name"])
merchant_type = str(row[1]["merchant_type"])
merchant_postal_code = str(row[1]["merchant_postal_code"])
merchant_area = str(row[1]["merchant_area"])
merchant_address = str(row[1]["merchant_prefecture"]) + \
str(row[1]["merchant_area"]) + \
str(row[1]["merchant_address"])
if type(row[1]["merchant_building_name"]) is str:
merchant_address += row[1]["merchant_building_name"]
merchant_tel = str(row[1]["merchant_tel"])
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"area": merchant_area,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
html = requests.post(
"https://gotoeat-fukui.com/shop/search.php", {"Keyword": "", "Action": "text_search"})
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find("div", {"class": "result"}).find("ul").findChildren(
"li", recursive=False)
merchant_ids = []
for merchant in lists:
merchant_id = re.sub(
r"^\.\/\?id=([0-9]+)$", r"\1", merchant.find("a").get("href"))
if merchant_id == merchant.find("a").get("href"):
print("Failed to get ID: {href}".format(href=merchant.find("a").get("href")))
exit(1)
if merchant.find("a").find("strong").text.strip() != None:
if merchant.find("a").find("strong").text.strip() in merchants["names"]:
continue
merchant_ids.append(merchant_id)
print(merchant_ids)
for merchant_id in merchant_ids:
time.sleep(1)
html = requests.get(
"https://gotoeat-fukui.com/shop/?id={merchant_id}".format(merchant_id=merchant_id))
soup = BeautifulSoup(html.content, "html.parser")
merchant_name = soup.find("h3").text.strip()
merchant_area = soup.find("span", {"class": "area"}).text.strip()
dds = soup.find("dl").findChildren("dd", recursive=False)
merchant_type = dds[0].text
merchant_tel = dds[1].text
merchant_address_gmap = str(dds[2])
merchant_address = re.sub(r"^<dd>(.+)<br/><a class=\"gmap\" href=\"(.+?)\".+$", r"\1", merchant_address_gmap)
merchant_gmap_url = re.sub(r"^<dd>(.+)<br/><a class=\"gmap\" href=\"(.+?)\".+$", r"\2", merchant_address_gmap)
lat = re.sub(
r"^.*q=([0-9\.]+),([0-9\.]+)$",
r"\1",
merchant_gmap_url)
if lat == merchant_gmap_url:
lat = None
lng = re.sub(
r"^.*q=([0-9\.]+),([0-9\.]+)$",
r"\2",
merchant_gmap_url)
if lng == merchant_gmap_url:
lng = None
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if lat == None or lng == None:
print(merchant_gmap_url)
if merchant_name in merchants["names"]:
continue
if lat == None or lng == None:
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"area": merchant_area,
"address": merchant_address,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
# Get Municipalities
municipalitys = []
html = requests.get("https://gotoeat-hokkaido.jp/general/particStores")
soup = BeautifulSoup(html.content, "html.parser")
municipalitys_tag = soup.find("form").find("select", {
"class": "wide02"
}).findChildren("option")
for municipality in municipalitys_tag:
if municipality.text == "選択してください":
continue
if municipality.get("value") != None:
municipalitys.append(municipality.get("value"))
else:
municipalitys.append(municipality.text)
print(municipalitys)
for municipality in municipalitys:
time.sleep(1)
session = requests.Session()
html = session.get("https://gotoeat-hokkaido.jp/general/particStores")
soup = BeautifulSoup(html.content, "html.parser")
_token = soup.find("form").find("input", {
"name": "_token"
}).get("value").strip()
html = session.post(
"https://gotoeat-hokkaido.jp/general/particStores/search",
data={
"store_area": municipality,
"store_address1": "",
"division1_id": "",
"store_name": "",
"_token": _token
})
page = 0
while True:
page += 1
html = session.get(
"https://gotoeat-hokkaido.jp/general/particStores?page={page}".
format(page=page))
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find("div", {
"class": "results"
}).find("ul").findChildren("li", recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("h4", {
"class": "results-tit"
}).text.strip()
merchant_address = merchant.find("p", {
"class": "results-txt01"
}).text.strip()
merchant_type = merchant.find("p", {
"class": "results-txt02"
}).text.strip()
merchant_tel = merchant.find("p", {
"class": "results-txt03"
}).text.strip()
merchant_tel = re.sub(r"TEL\.([0-9\-]+)", r"\1", merchant_tel)
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"area": municipality,
"address": merchant_address,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4,
ensure_ascii=False))
if (soup.find("a", {"class": "nextpostslink"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://gotoeat-yamaguchi.com/use/page/{page}/?post_type=post&s".format(page=page))
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find("ul", {"id": "shop-list"}
).findChildren("li", recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find(
"h3").text.strip()
merchant_time = None
merchant_regular_holiday = None
merchant_tel = None
merchant_data = merchant.find("div", {"class": "break"}).findChildren("p")
for data in merchant_data:
key = data.find("strong").text.strip()
if key.strip() == "［住所］":
merchant_address = data.text.strip(
)[data.text.strip().find("］")+1:].strip()
elif key.strip() == "［営業時間］":
merchant_time = data.text.strip(
)[data.text.strip().find("］")+1:].strip()
elif key.strip() == "［定休日］":
merchant_regular_holiday = data.text.strip(
)[data.text.strip().find("］")+1:].strip()
elif key.strip() == "［TEL］":
merchant_tel = data.text.strip(
)[data.text.strip().find("］")+1:].strip()
merchant_type = []
categories = merchant.find(
"p", {"class": "type"}).findChildren("a")
for category in categories:
merchant_type.append(category.text[1:].strip())
merchant_corona_measures = []
corona_measures = merchant.find(
"ul", {"class": "measures_icon"}).findChildren("li")
for corona_measure in corona_measures:
merchant_corona_measures.append(
corona_measure.find("img").get("alt"))
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"time": merchant_time,
"regular_holiday": merchant_regular_holiday,
"tel": merchant_tel,
"corona_measures": merchant_corona_measures,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("a", {"class": "nextpostslink"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://www.gotoeat-tochigi.jp/merchant/index.php?word=&sort=2&page={page}"
.format(page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find("ul", {
"class": "serch_result"
}).findChildren("li", recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("p", {"class": "name"}).text
merchant_type = merchant.find("p", {
"class": "name"
}).find("span").text
merchant_name = re.sub(
r"{merchant_type}$".format(merchant_type=merchant_type), "",
merchant_name)
_merchant_address = merchant.find("div", {
"class": "add"
}).findAll("p")[0].text
merchant_postal_code = re.sub(r"所在地〒([0-9\-]+) (.+)", r"\1",
_merchant_address)
merchant_address = re.sub(r"所在地〒([0-9\-]+) (.+)", r"\2",
_merchant_address)
if len(merchant.find("div", {"class": "add"}).findAll("p")) >= 2:
merchant_tel = merchant.find("div", {
"class": "add"
}).findAll("p")[1].text
merchant_tel = re.sub(r"TEL(.+)", r"\1", merchant_tel)
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("li", {"class": "next"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://niigata-gte.com/shop/page/{page}/".format(page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
if soup.find("div", {"id": "result"}) == None:
break
lists = soup.find("div", {
"id": "result"
}).findChildren("div", {"class": "cont"}, recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("h4").text.strip()
merchant_area = merchant.find("div", {
"class": "tag"
}).findChildren("span", recursive=False)[0].text.strip()
merchant_type = merchant.find("div", {
"class": "tag"
}).findChildren("span", recursive=False)[1].text.strip()
_merchant_address = merchant.find("p", {"class": "add"}).text
merchant_postal_code = re.sub(r"〒([0-9\-]+) (.+)MAP", r"\1",
_merchant_address).strip()
merchant_address = re.sub(r"〒([0-9\-]+) (.+)MAP", r"\2",
_merchant_address).strip()
merchant_tel = None
if merchant.find("p", {"class": "tel"}) != None:
merchant_tel = merchant.find("p", {
"class": "tel"
}).text.strip()
print(merchant_name + " - " + merchant_address)
if merchant_name in merchants["names"]:
continue
if merchant.find("p", {"class": "add"}).find("a") != None:
latlng = merchant.find("p", {
"class": "add"
}).find("a").get("href")
lat = re.sub(
r"^.+maps\/(?:place|search)\/(.+)\/@([0-9\.]+),([0-9\.]+),([0-9\.]+)z.*$",
r"\2", latlng)
if lat == latlng:
lat = None
lng = re.sub(
r"^.+maps\/(?:place|search)\/(.+)\/@([0-9\.]+),([0-9\.]+),([0-9\.]+)z.*$",
r"\3", latlng)
if lng == latlng:
lng = None
if lat == None or lng == None:
print(latlng)
else:
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
findMerchants.append(merchant_name)
if lat == None or lng == None:
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"area": merchant_area,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("li", {"class": "next"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://premium-gift.jp/gotoeatmiyazaki/use_store?events=page&id={page}"
.format(page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
if soup.find("nav", {
"class": "pagenation"
}).find("span", {
"class": "is-current"
}).text != str(page):
break
lists = soup.find("div", {
"class": "store-card"
}).findChildren("div", {"class": "store-card__item"}, recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("h3", {
"class": "store-card__title"
}).text.replace(u"\xa0", " ").strip()
merchant_type = merchant.find("p", {
"class": "store-card__tag"
}).text.strip()
rows = merchant.find("table", {
"class": "store-card__table"
}).findChildren("tr")
for row in rows:
th = row.find("th").text.strip()
td = row.find("td").text.replace(u"\xa0", " ").strip()
if th == "住所：":
merchant_postal_code = re.sub(r"〒([0-9\-]+) (.+)", r"\1",
td)
merchant_address = re.sub(r"〒([0-9\-]+) (.+)", r"\2", td)
elif th == "電話番号：":
merchant_tel = td
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
# Get Municipalities
municipalitys = []
html = requests.get("https://saitama-goto-eat.com/store.html")
soup = BeautifulSoup(html.content, "html.parser")
municipalitys_tag = soup.find("select", {
"id": "round"
}).findChildren("option")
for municipality in municipalitys_tag:
if municipality.text == "選択してください。":
continue
if municipality.get("value") != None:
municipalitys.append(municipality.get("value"))
else:
municipalitys.append(municipality.text)
print(municipalitys)
for municipality in municipalitys:
time.sleep(1)
html = requests.get(
"https://saitama-goto-eat.com/store/{municipality}.html".format(
municipality=municipality))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.findAll("div", {"class": "storebox"})
for merchant in lists:
spans = merchant.findChildren("span", recursive=False)
merchant_name = spans[0].text
merchant_postal_code = spans[2].text
merchant_address = spans[3].text
merchant_tel = spans[4].text
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://www.goto-eat-ehime.com/page/{page}/?s&shop_area&shop_city"
.format(page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find("ul", {
"class": "shop_list"
}).findChildren("li", recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("dt").text.strip()
merchant_type = []
categories = merchant.find("p", {
"class": "shop_cat"
}).findChildren("span")
for category in categories:
merchant_type.append(category.text.strip())
rows = merchant.find("dd").find("ul").findChildren("li")
for row in rows:
key = row.findChildren("span")[0].text.strip()
value = row.findChildren("span")[1].text.strip()
if key == "住所":
merchant_address = value
elif key == "TEL":
merchant_tel = value
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("a", {"class": "nextpostslink"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://www.gotoeat-aichi-shop.jp/shop/page/{page}/".format(
page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find("ul", {
"class": "lcl-shop"
}).findChildren("li", {"class": "lcl-shop__item"}, recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("h2", {
"class": "lcl-shop__name"
}).text.strip()
merchant_type = None
if merchant.find("li",
{"class": "lcl-shop-tag__item--cat"}) != None:
merchant_type = merchant.find(
"li", {
"class": "lcl-shop-tag__item--cat"
}).text.strip()
merchant_area = None
if merchant.find("li",
{"class": "lcl-shop-tag__item--area"}) != None:
merchant_area = merchant.find(
"li", {
"class": "lcl-shop-tag__item--area"
}).text.strip()
_merchant_address = merchant.find("p", {
"class": "lcl-shop__address"
}).text.strip()
merchant_postal_code = re.sub(r"〒([0-9\-]+) (.+)", r"\1",
_merchant_address)
merchant_address = re.sub(r"〒([0-9\-]+) (.+)", r"\2",
_merchant_address)
merchant_tel = None
if merchant.find("a", {"class": "lcl-shop__link--tel"}) != None:
merchant_tel = merchant.find("a", {
"class": "lcl-shop__link--tel"
}).text.strip()
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("a", {"class": "pagination-btn--next"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://www.toyamagotoeat.jp/shop/page/{page}/".format(page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find("ul", {
"class": "list"
}).findChildren("li", {"class": "item"}, recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("div", {
"class": "item_body_name"
}).text.strip()
merchant_type = merchant.find("div", {
"class": "item_header"
}).text.strip()
merchant_address = merchant.find("div", {
"class": "place"
}).find("div", {
"class": "rig"
}).text.strip()
if merchant.find("div", {"class": "phone"}) != None:
merchant_tel = merchant.find("div", {
"class": "phone"
}).find("div", {
"class": "rig"
}).text.strip()
else:
merchant_tel = None
if merchant.find("div", {"class": "work"}) != None:
merchant_time = merchant.find("div", {
"class": "work"
}).find("div", {
"class": "rig"
}).text.strip()
else:
merchant_time = None
if merchant.find("div", {"class": "off_day"}) != None:
merchant_regular_holiday = merchant.find(
"div", {
"class": "off_day"
}).find("div", {
"class": "rig"
}).text.strip()
else:
merchant_regular_holiday = None
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"tel": merchant_tel,
"time": merchant_time,
"regular_holiday": merchant_regular_holiday,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://www.kagawa-gotoeat.com/gtes/store-list?mode=&p={page}".
format(page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.findChildren("div", {"class": "store-list"})
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("h4").text.strip()
rows = merchant.find("table").findChildren("tr")
for row in rows:
th = row.find("th").text.strip()
td = row.find("td").text.strip()
if th == "エリア":
merchant_area = td
elif th == "料理ジャンル":
merchant_type = td
elif th == "電話番号":
merchant_tel = td
elif th == "住所":
merchant_address = re.sub(r"^([\s\S]+)MAPを見る", r"\1",
td).strip()
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"area": merchant_area,
"address": merchant_address,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("a", {"class": "nextpostslink"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
# Get Municipalities
municipalitys = []
html = requests.get("https://www.iwate-gotoeat.jp/stores/")
soup = BeautifulSoup(html.content, "html.parser")
municipalitys_tag = soup.find("select", {
"name": "area"
}).findChildren("option")
for municipality in municipalitys_tag:
if municipality.text == "市町村を選択してください":
continue
if municipality.get("value") != None:
municipalitys.append(municipality.get("value"))
else:
municipalitys.append(municipality.text)
print(municipalitys)
for municipality in municipalitys:
time.sleep(1)
html = requests.post("https://www.iwate-gotoeat.jp/stores/",
data={"area": municipality})
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.findAll("div", {"class": "stores_box"})
for merchant in lists:
merchant_name = merchant.find("h2", {
"class": "stores_box_name"
}).text.strip()
merchant_address = merchant.find("p", {
"class": "stores_box_add"
}).text.strip()
merchant_tel = merchant.find("p", {
"class": "stores_box_tel"
}).text.strip()
merchant_tel = re.sub(r"TEL：([0-9\-]+)", r"\1", merchant_tel)
merchant_type = merchant.find("p", {
"class": "stores_box_genre"
}).text.strip()
merchant_type = merchant_type.split("・")
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
# Get Municipalities
municipalitys = {}
html = requests.get("https://gte-miyagi.jp/available.html")
soup = BeautifulSoup(html.content, "html.parser")
municipality_tags = soup.find(
"ul", {"class": "sendai"}).findChildren("li", recursive=False)
for municipality in municipality_tags:
municipalitys[municipality.find("a").text.strip()] = re.sub(
r"\.\/", "", municipality.find("a").get("href"))
print(municipalitys)
for merchant_area, municipality in municipalitys.items():
print(municipality)
html = requests.get(
"https://gte-miyagi.jp/{municipality}".format(municipality=municipality))
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find("div", {"class": "SLCont"}).findChildren(
"dl", {"class": "shopList"}, recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("dt").text.strip()
dds = merchant.findChildren("dd", recursive=False)
merchant_type = None
merchant_address = None
merchant_postal_code = None
merchant_tel = None
for dd in dds:
spans = dd.findChildren("span", recursive=False)
if (len(spans) != 2):
continue
if spans[0].text.strip() == "カテゴリ:":
merchant_type = spans[1].text.strip()
elif spans[0].text.strip() == "住所:":
merchant_postal_code = re.sub(
r"〒([0-9\-]+)(.+)", r"\1", spans[1].text.strip())
merchant_address = re.sub(
r"〒([0-9\-]+)(.+)", r"\2", spans[1].text.strip())
elif spans[0].text.strip() == "電話番号:":
merchant_tel = spans[1].text.strip()
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"area": merchant_area,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://gotoeat.tokushima.jp/?paged={page}&s".format(page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.findChildren("div", {"class": "post-full"})
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find(
"h2", {"class": "entry-title"}).text.strip()
if merchant_name[0:3] == "保護中":
continue
dts = merchant.find("dl").findChildren("dt", recursive=False)
dds = merchant.find("dl").findChildren("dd", recursive=False)
merchant_regular_holiday = None
merchant_time = None
merchant_tel = None
for key in range(len(dts)):
if len(dds) <= key:
break
dt = dts[key].text.strip()
dd = dds[key].text.strip()
if dt == "所在地":
merchant_address = dd
elif dt == "定休日":
merchant_regular_holiday = dd
elif dt == "営業時間":
merchant_time = dd
elif dt == "電話番号":
merchant_tel = dd
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"address": merchant_address,
"time": merchant_time,
"regular_holiday": merchant_regular_holiday,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("a", {"class": "next"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://gunma-gotoeat-campaign.com/shop/page/{page}/?s".format(
page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
if soup.find("section", {"id": "result"}) == None:
break
lists = soup.find("section", {
"id": "result"
}).findChildren("article", recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("h3").text
merchant_type = merchant.find("span", {"class": "shopcat"}).text
_merchant_address = merchant.find("p", {"class": "shopadr"}).text
merchant_postal_code = re.sub(r"〒([0-9\-]+)(.+)", r"\1",
_merchant_address)
merchant_address = re.sub(r"〒([0-9\-]+)(.+)", r"\2",
_merchant_address)
if merchant.find("p", {"class": "shoptel"}) != None:
merchant_tel = merchant.find("p", {"class": "shoptel"}).text
merchant_tel = re.sub(r"TEL\.(.+)", r"\1", merchant_tel)
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("a", {"class": "next"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://goto-eat.weare.osaka-info.jp/gotoeat/page/{page}/?search_element_0_0=2&search_element_0_1=3&search_element_0_2=4&search_element_0_3=5&search_element_0_4=6&search_element_0_5=7&search_element_0_6=8&search_element_0_7=9&search_element_0_8=10&search_element_0_9=11&search_element_0_cnt=10&search_element_1_cnt=17&search_element_2_cnt=1&s_keyword_3&cf_specify_key_3_0=gotoeat_shop_address01&cf_specify_key_3_1=gotoeat_shop_address02&cf_specify_key_3_2=gotoeat_shop_address03&cf_specify_key_length_3=2&csp=search_add&feadvns_max_line_0=4&fe_form_no=0"
.format(page=page))
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find("div", {
"class": "search_result_box"
}).find("ul").findChildren("li", recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("p", {"class": "name"}).text.strip()
if merchant_name == "":
print("Merchant nane is empty...?")
continue
rows = merchant.find("table").findChildren("tr")
merchant_tel = None
merchant_time = None
merchant_regular_holiday = None
for row in rows:
th = row.find("th").text.strip()
td = row.find("td").text.replace(u"\xa0", " ").strip()
if th == "住所":
merchant_postal_code = td.split("\r\n")[0].strip()
merchant_address = td.split("\r\n")[1].replace(" ",
"").strip()
elif th == "TEL":
merchant_tel = td
elif th == "営業時間" and td != "":
merchant_time = td
elif th == "定休日" and td != "":
merchant_regular_holiday = td
merchant_type = []
tag_list = merchant.find("ul", {
"class": "tag_list"
}).findChildren("li", recursive=False)
for tag in tag_list:
merchant_type.append(tag.text.strip())
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"postal_code": merchant_postal_code,
"time": merchant_time,
"regular_holiday": merchant_regular_holiday,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("a", {"class": "nextpostslink"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://ishikawa-gotoeat-cpn.com/?cities&type&s&post_type=member_store&paged={page}"
.format(page=page))
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find("ul", {
"class": "member_list"
}).findChildren("li", {"class": "member_item"}, recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("h4", {"class": "name"}).text.strip()
merchant_type = merchant.find("div", {
"class": "type"
}).text.strip()
merchant_address = merchant.find("div", {
"class": "address"
}).find("div", {
"class": "content"
}).text.strip()
merchant_postal_code = merchant.find("div", {
"class": "address"
}).find("div", {
"class": "post"
}).text.strip()
merchant_tel = None
if merchant.find("div", {"class": "tel"}) != None:
merchant_tel = merchant.find("div", {
"class": "tel"
}).text.strip()
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("div", {
"class": "page_nation"
}).find("span", {"class": "page_next"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
page = 0
while True:
time.sleep(1)
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://area34.smp.ne.jp/area/table/27130/3jFZ4A/M?_limit_27130=100&S=pimgn2lbtind&_page_27130={page}".format(page=page))
soup = BeautifulSoup(html.content, "html.parser",
from_encoding="shift-jis")
table = soup.findAll("table", {"class": "smp-table"})[0]
rows = table.findAll("tr", {"class": "smp-row-data"})
if len(rows) == 0:
break
for row in rows:
cells = row.findAll("td")
if len(cells) != 6:
continue
merchant_type = cells[0].text.strip()
merchant_name = cells[1].text.strip()
merchant_tel = cells[2].text.strip()
merchant_address = cells[3].text.strip() + " " + cells[4].text.strip()
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://gotoeat-saga.jp/consumer/shop.php?page={page}&name=#search_result2".format(page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.findChildren("div", {"class": "shop_detail"})
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("div", {"class": "ttl"}).text.strip()
merchant_type = merchant.find("div", {"class": "genre"}).text.strip()
dts = merchant.findChildren("dt")
dds = merchant.findChildren("dd")
merchant_regular_holiday = None
merchant_time = None
merchant_tel = None
for key in range(len(dts)):
dt = dts[key].text.strip()
dd = dds[key].text.strip()
if dt == "住所":
merchant_address = re.sub(r"^(.+)MAP$", r"\1", dd).strip()
elif dt == "定休日":
merchant_regular_holiday = dd
elif dt == "営業時間":
merchant_time = dd
elif dt == "電話番号":
merchant_tel = dd
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"time": merchant_time,
"regular_holiday": merchant_regular_holiday,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("li", {"class": "next"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
result = requests.post(
"https://yamagata-gotoeat.com/wp/wp-content/themes/gotoeat/search.php", data={"text": "", "page": page})
result = result.json()
if(result.get("html") == ""):
break
soup = BeautifulSoup(result.get("html"), "html.parser")
lists = soup.findChildren("li", recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("h2").text.strip()
divs = merchant.findChildren("div", recursive=False)
merchant_postal_code = re.sub(
r"([0-9\-]+) (.+)", r"\1", divs[0].text.strip())
merchant_address = re.sub(
r"([0-9\-]+) (.+)", r"\2", divs[0].text.strip())
merchant_tel = re.sub(
r"TEL : ([0-9\-]+)", r"\1", divs[1].text.strip())
tags = merchant.find("ul", {"class": "search__result__tag"}).findChildren(
"li", recursive=False)
merchant_area = tags[0].text.strip()
merchant_type = tags[1].text.strip()
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"area": merchant_area,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": [], "categories": []}
findMerchants = []
html = requests.get("https://platinumaps.jp/maps/gotoeat-okayama",
headers={"Accept-Language": "ja"})
soup = BeautifulSoup(html.content, "html.parser")
li_categories = soup.find("div", {
"class": "map__categoryselect"
}).find("ul").findChildren("li", recursive=False)
categories = {}
for category in li_categories:
category_key = category.get("data-category")
category_value = category.find("div", {
"class": "category__name"
}).text.strip()
categories[category_key] = category_value
merchants["categories"] = categories
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
response = requests.get("https://platinumaps.jp/maps/88/spots",
headers={"Accept-Language": "ja"})
try:
jsondata = response.json()
except Exception as e:
print(e)
print(response.text)
exit(1)
for merchant in jsondata["spots"]:
merchant_name = merchant["title"]
merchant_address = merchant["address"]
merchant_type = merchant["categories"]
merchant_tel = merchant["phoneNumber"]
merchant_regular_holiday = merchant["holiday"]
lat = merchant["lat"]
lng = merchant["lng"]
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"type": merchant_type,
"address": merchant_address,
"tel": merchant_tel,
"regular_holiday": merchant_regular_holiday,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {"data": [], "names": []}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://gotoeat-kumamoto.jp/shop/page/{page}/".format(page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.findChildren("article", {"class": "shop"})
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find("h3").text.strip()
merchant_area = merchant.find("p", {
"class": "cat"
}).find("a").text.strip()
_merchant_address = merchant.find("p").text.strip()
merchant_postal_code = re.sub(r"〒([0-9\-]+) (.+)", r"\1",
_merchant_address)
merchant_address = re.sub(r"〒([0-9\-]+) (.+)", r"\2",
_merchant_address).replace(" ",
"").strip()
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"area": merchant_area,
"address": merchant_address,
"postal_code": merchant_postal_code,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("a", {"class": "next"}) == None):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
def main():
merchantFilePath = os.path.dirname(
os.path.abspath(__file__)) + "/merchants.json"
if os.path.exists(merchantFilePath):
json_open = open(merchantFilePath, "r", encoding="utf8")
merchants = json.load(json_open)
else:
merchants = {
"data": [],
"names": []
}
findMerchants = []
page = 0
while True:
page += 1
print("----- Page {page} -----".format(page=page))
html = requests.get(
"https://gotoeat-hyogo.com/search/result?page={page}".format(page=page))
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.content, "html.parser")
lists = soup.find(
"ul", {"class": "search-results-list"}).findChildren("li", recursive=False)
if (len(lists) == 0):
break
for merchant in lists:
merchant_name = merchant.find(
"p", {"class": "search-results-list-name"}).text.strip()
rows = merchant.findChildren(
"p", {"class": "search-results-list-p01"})
merchant_tel = None
for row in rows:
key = row.findChildren("span", recursive=False)[0].text.strip()
value = row.findChildren("span", recursive=False)[
1].text.strip()
if key == "住所：":
merchant_postal_code = re.sub(
r"〒([0-9\-]+)([\s\S]+)", r"\1", value)
merchant_address = re.sub(
r"〒([0-9\-]+)([\s\S]+)", r"\2", value).replace(
"\n", "").replace(" ", "").strip()
elif key == "TEL：":
merchant_tel = value
print(merchant_name + " - " + merchant_address)
findMerchants.append(merchant_name)
if merchant_name in merchants["names"]:
continue
lat, lng = getLatLng(merchant_address)
print(str(lat) + " " + str(lng))
merchants["data"].append({
"name": merchant_name,
"address": merchant_address,
"postal_code": merchant_postal_code,
"tel": merchant_tel,
"lat": lat,
"lng": lng
})
merchants["names"].append(merchant_name)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
if (soup.find("p", {"class": "arrow-next"}).has_attr("disabled")):
break
else:
time.sleep(1)
merchants = checkRemovedMerchant(merchants, findMerchants)
with open(merchantFilePath, mode="w", encoding="utf8") as f:
f.write(json.dumps(merchants, indent=4, ensure_ascii=False))
