def scrape_worldOmeter(korea=True):
"""worldOmeter에서 세계 확진환자수, 격리해제수, 사망자수 수집
Args:
world: 대한민국 데이터만 수집하려면, True
세계 데이터를 수집하려면, False
Returns:
(dict) 한국의 확진환자수(cc_sum), 격리해제수(recovered), 사망자수(dead)
"""
html = requests.get("https://www.worldometers.info/coronavirus/")
soup = BeautifulSoup(html.text, "html.parser")
data = soup.select("#main_table_countries > tbody > tr")
world_data = {}
world_cc, world_recovered, world_dead = 0, 0, 0
push = []
for datum in data:
country = datum.find_all("td")[0].text.strip()
cc = datum.find_all("td")[1].text.strip()
recovered = datum.find_all("td")[5].text.strip()
dead = datum.find_all("td")[3].text.strip()
postproc = postprocess([cc, recovered, dead])
cc, recovered, dead = postproc[0], postproc[1], postproc[2]
if cc:
world_cc += cc
if recovered:
world_recovered += recovered
if dead:
world_dead += dead
if korea:
if country != "S. Korea":
continue
korea_patients = patients.copy()
korea_patients["cc_sum"] = cc
korea_patients["recovered"] = recovered
korea_patients["dead"] = dead
push.append(("대한민국", korea_patients))
SlackHandler().add_scraping_msg(
"scrape_korea.py >> scrape_worldOmeter()", push)
return korea_patients
world_data[country] = patients.copy()
world_data[country]["cc_sum"] = cc
world_data[country]["recovered"] = recovered
world_data[country]["dead"] = dead
push.append((country, world_data[country]))
time.sleep(0.2)
world_data["world"] = patients.copy()
world_data["world"]["cc_sum"] = world_cc
world_data["world"]["recovered"] = world_recovered
world_data["world"]["dead"] = world_dead
push.append(("world", world_data["world"]))
SlackHandler().add_scraping_msg(
"scrape_korea.py >> scrape_worldOmeter(korea=False)", push)
return world_data
def create_ticket_form():
data = request.form
Database.create_conversation(db, data["user_id"], data["channel_id"])
view = SlackHandler.create_ticket_view(data)
client.views_open(
trigger_id=data["trigger_id"],
view=view
)
return "Opening up a ticket request..."
def lambda_handler(event, context):
metadata = None
# Make output clearer to read.
print("\n---")
print("\n[lambda] instantiated")
bucket_name = event["Records"][0]["s3"]["bucket"]["name"]
key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'],
encoding='utf-8')
directory = key.rsplit("/", 1)[0]
print(f"[lambda] directory: {directory}\n")
# List and get filepaths within "directory"
filenames = []
for parser_object in s3.Bucket(bucket_name).objects.filter(
Prefix=directory):
filenames.append(parser_object.key)
for parser_output_file in filenames:
# Deal with metadata
if parser_output_file.endswith(".json"):
metadata = load_metadata(parser_output_file, bucket_name)
print(f"[lambda] starting slackhandler\n---")
slack = SlackHandler(metadata)
if metadata is None:
print(
"[lambda] warning: metadata file not parsed. please check the s3 bucket!\n"
)
slack.update("metadata file not parsed - stopping lambda")
return False
# now that metadata's loaded, deal with the actual report
load_report(slack, metadata, key, bucket_name)
# Make output clearer to read.
print("---\n")
return True
def interactions():
user_id="UTG9YH3HV"
id = "CTG9YH7QX"
client.chat_postEphemeral(channel=id, user=user_id, text="hi")
data = request.form
client.chat_postEphemeral(channel=id,user=user_id,text=str(data))
client.chat_postEphemeral(channel=id,user=user_id, text="hi")
data = request.form
channel_id = Database.get_conversation(data["user"]["id"], db=db)
client.chat_postMessage(channel=channel_id,
text="Data: {}".format(str(data)))
if data["type"] == "block_actions":
SlackHandler.handle_interactions(client=client, data=data, db=db)
return jsonify({'ok': True, 'message': 'interaction handled successfully'}), 200
elif data["type"] == "view_closed":
SlackHandler.handle_close(client=client, data=data, db=db)
return jsonify({'ok': True, 'message': 'Closed modal successfully!'}), 200
elif data["type"] == "view_submission":
SlackHandler.handle_submission(client=client, data=data, db=db)
return jsonify({'ok': True, 'message': 'Ticket created successfully!'}), 200
return jsonify({'ok': True, 'message': 'Unknown'}), 200
def scrape_seoul():
"""서울 발생 동향 수집"""
html = requests.get("http://www.seoul.go.kr/coronaV/coronaStatus.do")
soup = BeautifulSoup(html.text, "html.parser")
time = soup.select_one("div.status-seoul > h4 > span").text
var_list = [d.text for d in soup.find_all("p", class_="counter")[:7]]
postproc = postprocess(var_list)
# TODO: 수집한 데이터를 해당하는 키에 할당
seoul = {}
# seoul["increasing"] =
seoul["cc_sum"] = postproc[0]
# seoul["recovered"] =
# seoul["dead"] =
# seoul["incidence"] =
seoul["time"] = time
SlackHandler().add_scraping_msg("scrape_domestic.py >> scrape_seoul()", [("서울", seoul)])
return seoul
def scrape_KCDC_korea():
"""KCDC에서 대한민국 확진환자수, 격리해제수, 사망자수 수집
Returns:
(dict) 한국의 세계 확진환자수(cc), 격리해제수(recovered), 사망자수(dead)
"""
html = requests.get("http://ncov.mohw.go.kr/index_main.jsp")
soup = BeautifulSoup(html.text, "lxml")
data = soup.select("div.co_cur > ul > li > a.num")
regex = re.compile(r"\d[,\d]+")
cc = regex.search(data[0].text).group()
recovered = regex.search(data[1].text).group()
dead = regex.search(data[2].text).group()
postproc = postprocess([cc, recovered, dead])
return_data = patients.copy()
return_data["cc_sum"] = postproc[0]
return_data["recovered"] = postproc[1]
return_data["dead"] = postproc[2]
SlackHandler().add_scraping_msg("scrape_korea.py >> scrape_KCDC_korea()",
[("대한민국", return_data)])
return return_data
def scrape_KCDC_citydo():
"""질병관리본부의 시도별 발생동향 수집"""
request_headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Cookie': '_ga=GA1.3.902460717.1582188059; _gid=GA1.3.1299466237.1583138633; JSESSIONID=hUEn1QgHlDMNI2gSSZJxuN0zYGahJogaUyaeAEvgyXstvqyq4C13pOf4dNGoOdid.mohwwas1_servlet_engine40; NetFunnel_ID=; _gat_gtag_UA_26269343_2=1',
'Host': 'ncov.mohw.go.kr',
'Referer': 'http://ncov.mohw.go.kr/bdBoardList_Real.do?brdId=1&brdGubun=13&ncvContSeq=&contSeq=&board_id=&gubun=&fbclid=IwAR3NoNL_j1phitehSggDQedf7S165308xIEeG8ljACy-VRq-T5efcbcTK_s',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36',
}
session = requests.Session()
session.max_redirects = 6000000
html = session.get("http://ncov.mohw.go.kr/bdBoardList_Real.do?brdId=1&brdGubun=13&ncvContSeq=&contSeq=&board_id=&gubun=", headers=request_headers)
soup = BeautifulSoup(html.text, "html.parser")
time = soup.select_one("p.info > span").text
data = soup.select("table.num > tbody > tr")
push = []
new_data = {}
for datum in data:
name = datum.find("th").text
number = datum.find_all("td")
var_list = []
for i in range(4):
var_list.append(number[i].text)
postproc = postprocess(var_list)
new_data[name] = citydo.copy()
new_data[name]["increasing"] = postproc[0]
new_data[name]["cc_sum"] = postproc[1]
new_data[name]["dead"] = postproc[2]
new_data[name]["incidence"] = postproc[3]
new_data[name]["time"] = time
push.append((name, new_data[name]))
SlackHandler().add_scraping_msg("scrape_domestic.py >> scrape_KCDC_citydo()", push)
return new_data
def run_main():
"""전체 프로세스 실행"""
sleep_interval = 60 * 15
while True:
print("\n##########################################################\n")
now_time = datetime.datetime.now()
print(now_time, end="\n")
push = []
old_domestic = load_json(domestic_path)
old_korea = old_domestic["대한민국"]
old_korea["time"] = None
print("\n================= <대한민국> 업데이트 중")
new_korea = scrape_korea.run_korea()
ko_check = check_korea(old_korea, new_korea)
if ko_check:
push.append(["대한민국", old_korea.copy(), new_korea])
old_domestic["대한민국"].update(new_korea)
print("\n================= <국내 시/도> 업데이트 중")
new_domestic = scrape_domestic.run_domestic()
do_check, up_list = check_update(old_domestic, new_domestic)
if do_check:
for ul in up_list:
key = list(ul.keys())[0]
push.append([key, old_domestic[key].copy(), ul[key]])
old_domestic[key].update(ul[key])
wo_check = False
if now_time.hour in range(0, 24, 3) and now_time.minute in range(
0, 15):
print("\n================= <세계> 업데이트 중")
old_world = load_json(world_path)
new_world = scrape_korea.scrape_worldOmeter(korea=False)
wo_check, up_list = check_update(old_world, new_world)
if wo_check:
for ul in up_list:
key = list(ul.keys())[0]
if key not in old_world.keys():
new = patients.copy()
push.append([key, new, ul[key]])
else:
push.append([key, old_world[key].copy(), ul[key]])
old_world.update(ul)
if ko_check or do_check:
print("\n================= <대한민국 및 시/도> 데이터 업데이트 중")
old_domestic["대한민국"]["time"] = str(datetime.datetime.now())
save_json(old_domestic, domestic_path)
SlackHandler().push_file_msg("./_domestic.json")
if wo_check:
print("\n================= <세계> 데이터 업데이트 중")
save_json(old_world, world_path)
SlackHandler().push_file_msg("./_world.json")
if ko_check or do_check or wo_check:
SlackHandler().add_update_msg(push)
SlackHandler().push_scraping_message()
SlackHandler().push_update_message()
print("\n##########################################################\n")
time.sleep(sleep_interval)
if status_code is not None:
self.status_code = status_code
self.payload = payload
def to_dict(self):
rv = dict(self.payload or ())
rv["message"] = self.message
return rv
cfg = load_config()
meta_data_handler = MetaDataHandler()
db_handler = DBHandler(**cfg["db_handler"])
log_handler = LogHandler(**cfg["log_handler"])
slack_handlers = [SlackHandler(**args) for args in cfg["slack_handlers"]]
app = Flask(__name__)
CORS(app, **cfg["cors"])
@app.route("/")
def index():
return jsonify({})
def get_start() -> int:
start = request.args.get(
"start", "0") # NOTE: set the default value as a string object.
if not start.isdecimal():
raise InvalidUsage('Parameter "start" must be an integer.')