def query(self, statement=None): ''' Makes a query to the mVAM API. ''' if statement is None: print('%s No statement provided. Fetching all available records.' % item('warn')) results = [] if statement: for page in range(0, self.page_limit): print('%s Collecting %s page %s' % (item('bullet'), self.table, page) ) r = requests.post(self.metadata['url'], data = {'table': self.table, 'where': statement, 'page': page }) if len(r.json()) == 0: break else: results += r.json() else: for page in range(0, self.page_limit): print('%s Collecting %s page %s' % (item('bullet'), self.table, page) ) r = requests.post(self.metadata['url'], data = {'table': self.table, 'page': page }) if len(r.json()) == 0: break else: results += r.json() self.metadata['records'] = len(results) return(results)
def delete(self, gallery_item_data): ''' Deletes a gallery item on HDX. ''' r = requests.post( self.url['delete'], data=json.dumps(gallery_item_data), headers=self.headers, auth=('dataproject', 'humdata')) if r.status_code != 200: print("%s failed to delete %s" % (item('error'), self.data['dataset_id'])) print(r.text) else: print("%s deleted successfully %s" % (item('success'), self.data['dataset_id']))
def update(self, resource_package): ''' Updates a dataset on HDX. ''' r = requests.post( self.url['update'], data=json.dumps(resource_package), headers=self.headers, auth=('dataproject', 'humdata')) if r.status_code != 200: print("%s failed to create %s" % (item('error'), self.data['name'])) print(r.text) else: print("%s updated successfully %s" % (item('success'), self.data['name']))
def test_item_returns_correct_type(self): ''' item() utility function returns a string. ''' result = item('bullet') self.assertIs(type(result), str)
def store_sqlite(data, table): ''' Store records in a SQLite database. ''' print('{bullet} Storing {n} records in database.'.format(bullet=item('bullet'), n=len(data))) # for record in data: scraperwiki.sqlite.save(data[0].keys(), data, table_name=table)
def create(self): ''' Creates a gallery item on HDX. ''' if self.state['exists'] is True: print("%s Gallery item exists (%s). Updating. %s" % (item('warn'), len(self.state['items']), self.data['dataset_id'])) for object in self.state['items']: self.delete(gallery_item_data = object) r = requests.post( self.url['create'], data=json.dumps(self.data), headers=self.headers, auth=('dataproject', 'humdata')) if r.status_code != 200: print("%s failed to create %s" % (item('error'), self.data['dataset_id'])) print(r.text) else: print("%s gallery item created %s" % (item('success'), self.data['dataset_id']))
def create(self): ''' Creates a dataset on HDX. ''' if self.state['exists'] is True: print("%s Dataset exists. Updating. %s" % (item('warn'), self.data['name'])) for resource in self.state['items']: self.data['id'] = resource['id'] self.update(resource_package=self.data) return r = requests.post( self.url['create'], data=json.dumps(self.data), headers=self.headers, auth=('dataproject', 'humdata')) if r.status_code != 200: print("%s failed to create %s" % (item('error'), self.data['name'])) print(r.text) else: print("%s created successfully %s" % (item('success'), self.data['name']))
def load_config(config_path='config/config.json', verbose=True): ''' Load configuration parameters. ''' try: with open(config_path) as json_file: config = json.load(json_file) except Exception as e: print("%s Couldn't load configuration." % item('error')) if verbose: print(e) return False return config
def main(verbose=True): ''' Wrapper to run all the scheduled tasks. ''' if verbose: print('%s Running scheduler.' % item('bullet')) try: while True: schedule.run_pending() time.sleep(1) except Exception as e: print(e) return False
def main(): ''' Program wrapper. ''' contents = WorldPop().info() # # Collects data and organizes # in lists and dictionaries. # datasets = [] resources = [] for dataset in contents['data']['worldPopData']: d = parse_dataset(dataset) if d is not None: datasets.append(d['metadata']) resources.append(d['resource']) export_json(datasets, 'data/datasets.json') export_json(resources, 'data/resources.json') print('%s Total datasets downloaded %s' % (item('success'), str(len(datasets))))
def main(): """ Program wrapper. """ tables = ["pblStatsSum", "pblStatsSum4Maps"] for t in tables: m = mVAM(table=t) output = [] records = m.query() for record in records: output.append(parse(record)) store_csv(data=output, path="%s.csv" % t) store_sqlite(data=output, table=t) if __name__ == "__main__": try: main() print("%s Successfully collected mVAM data." % item("success")) scraperwiki.status("ok") except Exception as e: print("%s Failed to collected mVAM data." % item("error")) print(e) scraperwiki.status("error", "Failed to collect data.")
def test_item_returns_correct_type(self): ''' Tests that the item() utility function returns the right type. ''' result = item('bullet') self.assertIs(type(result), str)