def test_value_error_raised_when_metadata_not_present(self):
    '''
    Metadata is not present an exception is raised.

    '''
    d = Dataset(self.dataset_error).info()
    with self.assertRaises(ValueError):
      parse_dataset(d)
  def test_metadata_is_complete(self):
    '''
    Metadata property is complete.

    '''
    d = Dataset(self.dataset_id).info()
    result = parse_dataset(d)

    for key in self.metadata_types.keys():
      print(result['metadata'])
      self.assertIs(type(result['metadata'][key]), self.metadata_types[key])
  def test_parser_returns_metadata_and_resource(self):
    '''
    Dataset and the resource objects are returned.

    '''
    d = Dataset(self.dataset_id).info()
    result = parse_dataset(d)

    self.assertIs(type(result), type({}))

    for key in ['metadata', 'resources']:
      self.assertIn(key, result.keys())
Ejemplo n.º 4
0
def main():
  '''
  Program wrapper.

  '''
  print('%s Creating Dataverse instance.' % item('bullet'))
  d = Dataverse('dataverse.harvard.edu', 'IFPRI')

  print('%s Collecting all content from Dataverse.' % item('bullet'))
  contents = d.contents()

  #
  #  Collects data and organizes
  #  in lists and dictionaries.
  #
  datasets = []
  resources = []
  for dataset in contents:
    print('%s Collecting data for ID %s.' % (item('bullet'), dataset['id']))

    o = Dataset(dataset['id']).info()
    if o.get('status') is 'ERROR' or None:
      continue

    try:
      parsed_data = parse_dataset(o)
    except ValueError:
      print('%s Missing metadata. Not parsing.' % item('warn'))
      continue

    else:
      datasets.append(parsed_data['metadata'])
      resources += parsed_data['resources']

  #
  #  Exports JSON files to disk..
  #
  export_json(datasets, 'data/datasets.json')
  export_json(resources, 'data/resources.json')
  print('%s Total datasets downloaded %s' % (item('success'), str(len(datasets))))