def test_dispatch_and_verify():
# WONTFIX
loc = Locations(2010, 1)
csv_text = loc.interim_csv.read_text(encoding='utf-8')
values = list(evaluate(csv_text, UNITS, YAML_DOC))
dfs = {}
for freq in 'aqm':
df = create_dataframe(values, freq)
dfs[freq] = df
verify(df, freq)
def parse(year,
month,
units_dict=UNITS,
yaml_doc=YAML_DOC,
data_root=DATA_ROOT,
validator=verify):
# filesystem
loc = Locations(year, month, data_root)
#parse
text = loc.interim_csv.read_text(encoding='utf-8')
values = list(evaluate(text, units_dict, yaml_doc))
# save three dataframes
dfs = {}
for freq in 'aqm':
df = create_dataframe(values, freq)
dfs[freq] = df
validator(df, freq)
df.to_csv(str(loc.processed_csv(freq)))
return dfs
def isolation1():
doc = SRC.read_text(encoding='utf-8')
return evaluate(doc, UNITS, YAML_DOC)
def save(year, month, dfs, data_root=DATA_ROOT, output_root=OUTPUT_ROOT):
loc = Locations(year, month, data_root)
oloc = OutputLocations(output_root)
if is_latest(year, month):
to_latest(year, month, loc)
save_excel(oloc.xlsx, dfs)
def all_months():
for year, month in reversed(supported_dates()[:-2]):
print("\n"*5, year, month)
dfa, dfq, dfm = unpack(parse(year, month))
#for speed tests
SRC = Locations(2018, 1).interim_csv
TEST_DOC = SRC.read_text(encoding='utf-8')
TEST_VALUES = list(evaluate(TEST_DOC, UNITS, YAML_DOC))
def isolation1():
doc = SRC.read_text(encoding='utf-8')
return evaluate(doc, UNITS, YAML_DOC)
def isolation2():
dfs = {}
for freq in 'aqm':
df = create_dataframe(TEST_VALUES, freq)
dfs[freq] = df
verify(df, freq)
df.to_csv('temp.txt')
return dfs
if __name__ == '__main__':