def main(): ''' Program wrapper. ''' print('%s Creating Dataverse instance.' % item('bullet')) d = Dataverse('dataverse.harvard.edu', 'IFPRI') print('%s Collecting all content from Dataverse.' % item('bullet')) contents = d.contents() # # Collects data and organizes # in lists and dictionaries. # datasets = [] resources = [] for dataset in contents: print('%s Collecting data for ID %s.' % (item('bullet'), dataset['id'])) o = Dataset(dataset['id']).info() if o.get('status') is 'ERROR' or None: continue try: parsed_data = parse_dataset(o) except ValueError: print('%s Missing metadata. Not parsing.' % item('warn')) continue else: datasets.append(parsed_data['metadata']) resources += parsed_data['resources'] # # Exports JSON files to disk.. # export_json(datasets, 'data/datasets.json') export_json(resources, 'data/resources.json') print('%s Total datasets downloaded %s' % (item('success'), str(len(datasets))))
def LoadConfig(config_path="config/dev.json", verbose=True): """Load configuration parameters.""" try: with open(config_path) as json_file: config = json.load(json_file) except Exception as e: print("%s Couldn't load configuration." % item("error")) if verbose: print(e) return False return config
def main(verbose=True): ''' Wrapper to run all the scheduled tasks. ''' if verbose: print('%s Running scheduler.' % item('bullet')) try: while True: schedule.run_pending() time.sleep(1) except Exception as e: print(e) return False
def test_item_returns_correct_type(self): ''' item() utility function returns the right type. ''' result = item('bullet') self.assertIs(type(result), str)
continue try: parsed_data = parse_dataset(o) except ValueError: print('%s Missing metadata. Not parsing.' % item('warn')) continue else: datasets.append(parsed_data['metadata']) resources += parsed_data['resources'] # # Exports JSON files to disk.. # export_json(datasets, 'data/datasets.json') export_json(resources, 'data/resources.json') print('%s Total datasets downloaded %s' % (item('success'), str(len(datasets)))) if __name__ == '__main__': try: main() print('%s Successfully collected IFPRI data.' % item('success')) scraperwiki.status('ok') except Exception as e: print('%s Failed to collected IFPRI data.' % item('error')) print(e) scraperwiki.status('error', 'Failed to collect data.')