def main():
'''
Program wrapper.
'''
print('%s Creating Dataverse instance.' % item('bullet'))
d = Dataverse('dataverse.harvard.edu', 'IFPRI')
print('%s Collecting all content from Dataverse.' % item('bullet'))
contents = d.contents()
#
# Collects data and organizes
# in lists and dictionaries.
#
datasets = []
resources = []
for dataset in contents:
print('%s Collecting data for ID %s.' % (item('bullet'), dataset['id']))
o = Dataset(dataset['id']).info()
if o.get('status') is 'ERROR' or None:
continue
try:
parsed_data = parse_dataset(o)
except ValueError:
print('%s Missing metadata. Not parsing.' % item('warn'))
continue
else:
datasets.append(parsed_data['metadata'])
resources += parsed_data['resources']
#
# Exports JSON files to disk..
#
export_json(datasets, 'data/datasets.json')
export_json(resources, 'data/resources.json')
print('%s Total datasets downloaded %s' % (item('success'), str(len(datasets))))
def LoadConfig(config_path="config/dev.json", verbose=True):
"""Load configuration parameters."""
try:
with open(config_path) as json_file:
config = json.load(json_file)
except Exception as e:
print("%s Couldn't load configuration." % item("error"))
if verbose:
print(e)
return False
return config
def main(verbose=True):
'''
Wrapper to run all the scheduled tasks.
'''
if verbose:
print('%s Running scheduler.' % item('bullet'))
try:
while True:
schedule.run_pending()
time.sleep(1)
except Exception as e:
print(e)
return False
def test_item_returns_correct_type(self):
'''
item() utility function returns the right type.
'''
result = item('bullet')
self.assertIs(type(result), str)
continue
try:
parsed_data = parse_dataset(o)
except ValueError:
print('%s Missing metadata. Not parsing.' % item('warn'))
continue
else:
datasets.append(parsed_data['metadata'])
resources += parsed_data['resources']
#
# Exports JSON files to disk..
#
export_json(datasets, 'data/datasets.json')
export_json(resources, 'data/resources.json')
print('%s Total datasets downloaded %s' % (item('success'), str(len(datasets))))
if __name__ == '__main__':
try:
main()
print('%s Successfully collected IFPRI data.' % item('success'))
scraperwiki.status('ok')
except Exception as e:
print('%s Failed to collected IFPRI data.' % item('error'))
print(e)
scraperwiki.status('error', 'Failed to collect data.')