def download(overwrite_cached=False):
settings = par.read_settings(str(_HERE_ / 'settings.json'))
cached_dd = dl.check_cached(settings['dd_path'], kind='dictionary')
cached_month = dl.check_cached(settings['monthly_path'], kind='data')
dds = dl.all_monthly_files(kind='dictionary')
dds = filter(itemgetter(1), dds) # make sure not None cpsdec!
dds = dl.filter_dds(dds, months=[par._month_to_dd(settings['date_start']),
par._month_to_dd(settings['date_end'])])
data = dl.all_monthly_files()
data = dl.filter_monthly_files(data, months=[[settings['date_start'],
settings['date_end']]])
if not overwrite_cached:
def is_new(x, cache=None):
return dl.rename_cps_monthly(x[1]) not in cache
dds = filter(partial(is_new, cache=cached_dd), dds)
data = filter(partial(is_new, cache=cached_month), data)
for month, renamed in dds:
dl.download_month(month, Path(settings['dd_path']))
logging.info("Downloaded {}".format(renamed))
for month, renamed in data:
dl.download_month(month, Path(settings['monthly_path']))
logging.info("Downloaded {}".format(renamed))
def test_filter_monthly_files_basic(self):
files = [('jan94.zip', 'cpsm1994-01.zip'),
('mar94.zip', 'cpsm1994-03.zip'),
('apr94.zip', 'cpsm1994-04.zip'),
('may94.zip', 'cpsm1994-05.zip')]
months = ['1994-01', '1994-03', '1994-04']
result = list(d.filter_monthly_files(files, months=months))
expected = files[0:3]
self.assertEqual(result, expected)
def test_filter_monthly_files_nested(self):
files = [('jan94.zip', 'cpsm1994-01.zip'),
('mar94.zip', 'cpsm1994-03.zip'),
('apr94.zip', 'cpsm1994-04.zip'),
('nov94.zip', 'cpsm1994-11.zip'),
('dec94.zip', 'cpsm1994-12.zip'),
('jan95.zip', 'cpsm1995-01.zip')]
months = [['1994-01', '1994-04'], ['1994-11', '1995-01']]
result = list(d.filter_monthly_files(files, months=months))
expected = files
self.assertEqual(result, expected)
def test_filter_monthly_files_None(self):
files = [('jan94.zip', 'cpsm1994-01.zip'),
('mar94.zip', 'cpsm1994-03.zip')]
result = list(d.filter_monthly_files(iter(files)))
expected = files
self.assertEqual(expected, list(result))
def test_filter_monthly_files_mix(self):
months = [['1994-01', '1994-04'], '1995-01']
files = iter([('jan94.zip', 'cpsm1994-01.zip'),
('mar94.zip', 'cpsm1994-03.zip')])
with self.assertRaises(ValueError):
list(d.filter_monthly_files(files, months=months))
def test_filter_monthly_files_None(self):
files = [('jan94.zip', 'cpsm1994-01.zip'),
('mar94.zip', 'cpsm1994-03.zip')]
result = list(d.filter_monthly_files(iter(files)))
expected = files
self.assertEqual(expected, list(result))
def test_filter_monthly_files_mix(self):
months = [['1994-01', '1994-04'], '1995-01']
files = iter([('jan94.zip', 'cpsm1994-01.zip'),
('mar94.zip', 'cpsm1994-03.zip')])
with self.assertRaises(ValueError):
list(d.filter_monthly_files(files, months=months))
