def exporter():
exporter = DBExporter()
exporter.export(
[
{**SOURCES[0], **RECORDS[0]},
{**SOURCES[1], **RECORDS[1]},
],
fingerprint=FINGERPRINT)
return exporter
def parse(path, url, export):
if not path and not url:
raise click.ClickException('Please provide either --path or --url')
parser_name = DWDPoller().get_parser(os.path.basename(path or url))
parser = getattr(parsers, parser_name)(path=path, url=url)
if url:
parser.download()
logger.info("Parsing %s with %s", path or url, parser_name)
records = parser.parse()
if export:
exporter = DBExporter()
exporter.export(records)
else:
dump_records()
def test_clean_deletes_expired_recent_records(db):
now = datetime.datetime.utcnow().replace(minute=0,
second=0,
microsecond=0,
tzinfo=tzutc())
records = [
{
'observation_type': 'historical',
'timestamp': now - datetime.timedelta(hours=6),
**PLACE,
'temperature': 10.,
},
{
'observation_type': 'recent',
'timestamp': now - datetime.timedelta(hours=7),
**PLACE,
'temperature': 20.,
},
{
'observation_type': 'recent',
'timestamp': now - datetime.timedelta(hours=6),
**PLACE,
'temperature': 30.,
},
]
DBExporter().export(records)
assert len(db.table('weather')) == 3
clean()
assert len(db.table('weather')) == 2
rows = db.fetch('SELECT temperature FROM weather ORDER BY temperature')
assert [r['temperature'] for r in rows] == [10., 30.]
def data(db):
records = RECENT_RECORDS + CURRENT_RECORDS + FORECAST_RECORDS
DBExporter().export(records)
def test_clean_deletes_expired_forecast_current_synop_records(db):
now = datetime.datetime.utcnow().replace(minute=0,
second=0,
microsecond=0,
tzinfo=tzutc())
records = [
{
'observation_type': 'forecast',
'timestamp': now,
**PLACE,
'temperature': 10.,
},
{
'observation_type': 'forecast',
'timestamp': now - datetime.timedelta(hours=6),
**PLACE,
'temperature': 20.,
},
{
'observation_type': 'current',
'timestamp': now,
**PLACE,
'temperature': 30.,
},
{
'observation_type': 'current',
'timestamp': now - datetime.timedelta(hours=6),
**PLACE,
'temperature': 40.,
},
{
'observation_type': 'current',
'timestamp': now - datetime.timedelta(days=3),
**PLACE,
'temperature': 50.,
},
]
synop_records = [
{
'observation_type': 'synop',
'timestamp': now,
**PLACE,
'temperature': 60.,
},
{
'observation_type': 'synop',
'timestamp': now - datetime.timedelta(hours=6),
**PLACE,
'temperature': 70.,
},
{
'observation_type': 'synop',
'timestamp': now - datetime.timedelta(days=3),
**PLACE,
'temperature': 80.,
},
]
DBExporter().export(records)
SYNOPExporter().export(synop_records)
assert len(db.table('weather')) == 5
assert len(db.table('synop')) == 3
clean()
assert len(db.table('weather')) == 3
assert len(db.table('synop')) == 2
rows = db.fetch('SELECT temperature FROM weather ORDER BY temperature')
assert [r['temperature'] for r in rows] == [10., 30., 40.]
rows = db.fetch('SELECT temperature FROM synop ORDER BY temperature')
assert [r['temperature'] for r in rows] == [60., 70.]