async def download_file(filename, url, session: aiohttp.ClientSession):
file_directory = get_path_from_root(FILES)
path = get_path_from_root(f'{filename}.pdf', file_directory)
if not os.path.exists(file_directory):
os.mkdir(file_directory)
async with session.get(url) as response:
async with aiofiles.open(path, mode='wb') as file:
async for data, _ in response.content.iter_chunks():
await file.write(data)
logger.debug(f'saved new report: {filename}')
async def extract_data():
download_directory_path = get_path_from_root(FILES)
report_directory_path = get_path_from_root(INDIVIDUAL_REPORTS)
complete_report_name = DATA
if not os.path.exists(download_directory_path):
return
analysis = await analyse_all_reports(download_directory_path)
logger.info(f'analysed {len(analysis)} '
f'report{"s" if len(analysis) else ""}')
return await asyncio.gather(
save_individual_reports(analysis, report_directory_path),
save_complete_report(analysis, complete_report_name))
async def save_individual_reports(data, directory):
if not os.path.isdir(directory):
os.mkdir(directory)
for timestamp, analysis in data:
csv_file_path = get_path_from_root(f'{timestamp}.csv', directory)
async with aiofiles.open(csv_file_path, mode='w') as file:
fieldnames = list(analysis[0].keys())
csv_countries = [list(country.values()) for country in analysis]
writer = aiocsv.AsyncWriter(file)
await writer.writerow(fieldnames)
await writer.writerows(csv_countries)
json_file_path = get_path_from_root(f'{timestamp}.json', directory)
async with aiofiles.open(json_file_path, mode='w') as file:
await file.write(json.dumps(analysis, indent=2))
def get_missing_reports(scraped_results):
local_directory = get_path_from_root(INDIVIDUAL_REPORTS)
if not os.path.isdir(local_directory):
return scraped_results
local_files = {item.split('.')[0] for item in os.listdir(local_directory)}
missing_reports = {
key: val
for key, val in scraped_results.items()
if val['date'] not in local_files
}
return missing_reports
def main(debug):
logging.getLogger('pdfminer').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
logging.getLogger('asyncio').setLevel(logging.WARNING)
logging_defaults = {'custom_level': 'DEBUG' if debug else 'INFO'}
logging.config.fileConfig(fname=get_path_from_root(LOGGING_CONFIG),
defaults=logging_defaults,
disable_existing_loggers=False)
try:
loop = asyncio.get_event_loop()
loop.run_until_complete(get_reports())
loop.run_until_complete(extract_data())
loop.close()
remove_downloaded_files()
return 0
except KeyboardInterrupt:
return 0
except RuntimeError:
return 1
async def save_complete_json_report(data, filename):
analysed_dates = [el[0] for el in data]
json_file_path = get_path_from_root(f'{filename}.json')
if not os.path.exists(json_file_path):
logger.info(f'{json_file_path} doesn\'t exist, creating new one')
await write_to_json(data, json_file_path)
else:
async with aiofiles.open(json_file_path, mode='r+') as f:
content = await f.read()
if not content.strip():
logger.debug(f'{json_file_path} is empty, '
f'writing from scratch')
await write_to_json(data, json_file_path)
else:
saved_data = {
k: v
for k, v in json.loads(content).items()
if k not in analysed_dates
}
complete_data = {**saved_data, **parse_data_for_json(data)}
await f.seek(0)
await f.truncate()
await f.write(
json.dumps(complete_data, sort_keys=True, indent=2))
async def save_complete_csv_report(data, filename):
analysed_dates = [el[0] for el in data]
csv_file_path = get_path_from_root(f'{filename}.csv')
if not os.path.exists(csv_file_path):
logger.info(f'{csv_file_path} doesn\'t exist, creating new one')
await write_to_csv(data, csv_file_path)
else:
async with aiofiles.open(csv_file_path, mode='r+') as f:
content = await f.read()
if not content.strip():
logger.debug(f'{csv_file_path} is empty, writing from scratch')
await write_to_csv(data, csv_file_path)
else:
fieldnames, *saved_data = [
line for line in csv.reader(content.splitlines())
if line and line[0] not in analysed_dates
]
complete_data = [*saved_data, *parse_data_for_csv(data)]
complete_data = sorted(complete_data, key=lambda el: el[0])
await f.seek(0)
await f.truncate()
writer = aiocsv.AsyncWriter(f)
await writer.writerow(fieldnames)
await writer.writerows(complete_data)
def remove_downloaded_files(folder=DOWNLOAD_FOLDER):
if not os.path.exists(folder):
return
for file in os.listdir(folder):
os.remove(get_path_from_root(file, folder))
os.rmdir(folder)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
from risikogebiete_api.utils import get_path_from_root
from risikogebiete_api.constants import FILES
DOWNLOAD_FOLDER = get_path_from_root(FILES)
def remove_downloaded_files(folder=DOWNLOAD_FOLDER):
if not os.path.exists(folder):
return
for file in os.listdir(folder):
os.remove(get_path_from_root(file, folder))
os.rmdir(folder)
async def analyse_all_reports(directory):
tasks = [
analyse_report(get_path_from_root(file, directory))
for file in os.listdir(directory)
]
return await asyncio.gather(*tasks)