def query(self, statement=None):
        '''
        Makes a query to the mVAM API.

        '''
        if statement is None:
            print('%s No statement provided. Fetching all available records.' % item('warn'))

        results = []
        if statement:
            for page in range(0, self.page_limit):
                print('%s Collecting %s page %s' % (item('bullet'), self.table, page) )
                r = requests.post(self.metadata['url'], data = {'table': self.table, 'where': statement, 'page': page })
                if len(r.json()) == 0:
                    break
                else:
                    results += r.json()

        else:
            for page in range(0, self.page_limit):
                print('%s Collecting %s page %s' % (item('bullet'), self.table, page) )
                r = requests.post(self.metadata['url'], data = {'table': self.table, 'page': page })
                if len(r.json()) == 0:
                    break
                else:
                    results += r.json()

        self.metadata['records'] = len(results)
        return(results)
    def delete(self, gallery_item_data):
        '''
        Deletes a gallery item on HDX.

        '''
        r = requests.post(
            self.url['delete'], data=json.dumps(gallery_item_data),
            headers=self.headers, auth=('dataproject', 'humdata'))

        if r.status_code != 200:
            print("%s failed to delete %s" % (item('error'), self.data['dataset_id']))
            print(r.text)

        else:
            print("%s deleted successfully %s" % (item('success'), self.data['dataset_id']))
    def update(self, resource_package):
        '''
        Updates a dataset on HDX.

        '''
        r = requests.post(
            self.url['update'], data=json.dumps(resource_package),
            headers=self.headers, auth=('dataproject', 'humdata'))

        if r.status_code != 200:
            print("%s failed to create %s" % (item('error'), self.data['name']))
            print(r.text)

        else:
            print("%s updated successfully %s" % (item('success'), self.data['name']))
  def test_item_returns_correct_type(self):
    '''
    item() utility function returns a string.

    '''
    result = item('bullet')
    self.assertIs(type(result), str)
def store_sqlite(data, table):
  '''
  Store records in a SQLite database.

  '''
  print('{bullet} Storing {n} records in database.'.format(bullet=item('bullet'), n=len(data)))

  # for record in data:
  scraperwiki.sqlite.save(data[0].keys(), data, table_name=table)
    def create(self):
        '''
        Creates a gallery item on HDX.

        '''
        if self.state['exists'] is True:
            print("%s Gallery item exists (%s). Updating. %s" % (item('warn'), len(self.state['items']), self.data['dataset_id']))
            for object in self.state['items']:
                self.delete(gallery_item_data = object)

        r = requests.post(
            self.url['create'], data=json.dumps(self.data),
            headers=self.headers, auth=('dataproject', 'humdata'))

        if r.status_code != 200:
            print("%s failed to create %s" % (item('error'), self.data['dataset_id']))
            print(r.text)

        else:
            print("%s gallery item created %s" % (item('success'), self.data['dataset_id']))
    def create(self):
        '''
        Creates a dataset on HDX.

        '''
        if self.state['exists'] is True:
            print("%s Dataset exists. Updating. %s" % (item('warn'), self.data['name']))
            for resource in self.state['items']:
                self.data['id'] = resource['id']
                self.update(resource_package=self.data)

            return

        r = requests.post(
            self.url['create'], data=json.dumps(self.data),
            headers=self.headers, auth=('dataproject', 'humdata'))

        if r.status_code != 200:
            print("%s failed to create %s" % (item('error'), self.data['name']))
            print(r.text)

        else:
            print("%s created successfully %s" % (item('success'), self.data['name']))
Example #8
0
def load_config(config_path='config/config.json', verbose=True):
    '''
  Load configuration parameters.

  '''
    try:
        with open(config_path) as json_file:
            config = json.load(json_file)

    except Exception as e:
        print("%s Couldn't load configuration." % item('error'))
        if verbose:
            print(e)
        return False

    return config
def main(verbose=True):
  '''
  Wrapper to run all the scheduled tasks.

  '''

  if verbose:
    print('%s Running scheduler.' % item('bullet'))

  try:
    while True:
      schedule.run_pending()
      time.sleep(1)

  except Exception as e:
    print(e)
    return False
Example #10
0
def main():
    '''
    Program wrapper.

    '''
    contents = WorldPop().info()

    #
    #  Collects data and organizes
    #  in lists and dictionaries.
    #
    datasets = []
    resources = []
    for dataset in contents['data']['worldPopData']:
        d = parse_dataset(dataset)
        if d is not None:
            datasets.append(d['metadata'])
            resources.append(d['resource'])

    export_json(datasets, 'data/datasets.json')
    export_json(resources, 'data/resources.json')
    print('%s Total datasets downloaded %s' % (item('success'), str(len(datasets))))
Example #11
0

def main():
    """
    Program wrapper.

    """
    tables = ["pblStatsSum", "pblStatsSum4Maps"]
    for t in tables:
        m = mVAM(table=t)

        output = []
        records = m.query()
        for record in records:
            output.append(parse(record))

        store_csv(data=output, path="%s.csv" % t)
        store_sqlite(data=output, table=t)


if __name__ == "__main__":
    try:
        main()
        print("%s Successfully collected mVAM data." % item("success"))
        scraperwiki.status("ok")

    except Exception as e:
        print("%s Failed to collected mVAM data." % item("error"))
        print(e)
        scraperwiki.status("error", "Failed to collect data.")
 def test_item_returns_correct_type(self):
     '''
 Tests that the item() utility function returns the right type.
 '''
     result = item('bullet')
     self.assertIs(type(result), str)
 def test_item_returns_correct_type(self):
   '''
   Tests that the item() utility function returns the right type.
   '''
   result = item('bullet')
   self.assertIs(type(result), str)