def query(self, statement=None):
'''
Makes a query to the mVAM API.
'''
if statement is None:
print('%s No statement provided. Fetching all available records.' % item('warn'))
results = []
if statement:
for page in range(0, self.page_limit):
print('%s Collecting %s page %s' % (item('bullet'), self.table, page) )
r = requests.post(self.metadata['url'], data = {'table': self.table, 'where': statement, 'page': page })
if len(r.json()) == 0:
break
else:
results += r.json()
else:
for page in range(0, self.page_limit):
print('%s Collecting %s page %s' % (item('bullet'), self.table, page) )
r = requests.post(self.metadata['url'], data = {'table': self.table, 'page': page })
if len(r.json()) == 0:
break
else:
results += r.json()
self.metadata['records'] = len(results)
return(results)
def delete(self, gallery_item_data):
'''
Deletes a gallery item on HDX.
'''
r = requests.post(
self.url['delete'], data=json.dumps(gallery_item_data),
headers=self.headers, auth=('dataproject', 'humdata'))
if r.status_code != 200:
print("%s failed to delete %s" % (item('error'), self.data['dataset_id']))
print(r.text)
else:
print("%s deleted successfully %s" % (item('success'), self.data['dataset_id']))
def update(self, resource_package):
'''
Updates a dataset on HDX.
'''
r = requests.post(
self.url['update'], data=json.dumps(resource_package),
headers=self.headers, auth=('dataproject', 'humdata'))
if r.status_code != 200:
print("%s failed to create %s" % (item('error'), self.data['name']))
print(r.text)
else:
print("%s updated successfully %s" % (item('success'), self.data['name']))
def test_item_returns_correct_type(self):
'''
item() utility function returns a string.
'''
result = item('bullet')
self.assertIs(type(result), str)
def store_sqlite(data, table):
'''
Store records in a SQLite database.
'''
print('{bullet} Storing {n} records in database.'.format(bullet=item('bullet'), n=len(data)))
# for record in data:
scraperwiki.sqlite.save(data[0].keys(), data, table_name=table)
def create(self):
'''
Creates a gallery item on HDX.
'''
if self.state['exists'] is True:
print("%s Gallery item exists (%s). Updating. %s" % (item('warn'), len(self.state['items']), self.data['dataset_id']))
for object in self.state['items']:
self.delete(gallery_item_data = object)
r = requests.post(
self.url['create'], data=json.dumps(self.data),
headers=self.headers, auth=('dataproject', 'humdata'))
if r.status_code != 200:
print("%s failed to create %s" % (item('error'), self.data['dataset_id']))
print(r.text)
else:
print("%s gallery item created %s" % (item('success'), self.data['dataset_id']))
def create(self):
'''
Creates a dataset on HDX.
'''
if self.state['exists'] is True:
print("%s Dataset exists. Updating. %s" % (item('warn'), self.data['name']))
for resource in self.state['items']:
self.data['id'] = resource['id']
self.update(resource_package=self.data)
return
r = requests.post(
self.url['create'], data=json.dumps(self.data),
headers=self.headers, auth=('dataproject', 'humdata'))
if r.status_code != 200:
print("%s failed to create %s" % (item('error'), self.data['name']))
print(r.text)
else:
print("%s created successfully %s" % (item('success'), self.data['name']))
def load_config(config_path='config/config.json', verbose=True):
'''
Load configuration parameters.
'''
try:
with open(config_path) as json_file:
config = json.load(json_file)
except Exception as e:
print("%s Couldn't load configuration." % item('error'))
if verbose:
print(e)
return False
return config
def main(verbose=True):
'''
Wrapper to run all the scheduled tasks.
'''
if verbose:
print('%s Running scheduler.' % item('bullet'))
try:
while True:
schedule.run_pending()
time.sleep(1)
except Exception as e:
print(e)
return False
def main():
'''
Program wrapper.
'''
contents = WorldPop().info()
#
# Collects data and organizes
# in lists and dictionaries.
#
datasets = []
resources = []
for dataset in contents['data']['worldPopData']:
d = parse_dataset(dataset)
if d is not None:
datasets.append(d['metadata'])
resources.append(d['resource'])
export_json(datasets, 'data/datasets.json')
export_json(resources, 'data/resources.json')
print('%s Total datasets downloaded %s' % (item('success'), str(len(datasets))))
def main():
"""
Program wrapper.
"""
tables = ["pblStatsSum", "pblStatsSum4Maps"]
for t in tables:
m = mVAM(table=t)
output = []
records = m.query()
for record in records:
output.append(parse(record))
store_csv(data=output, path="%s.csv" % t)
store_sqlite(data=output, table=t)
if __name__ == "__main__":
try:
main()
print("%s Successfully collected mVAM data." % item("success"))
scraperwiki.status("ok")
except Exception as e:
print("%s Failed to collected mVAM data." % item("error"))
print(e)
scraperwiki.status("error", "Failed to collect data.")
def test_item_returns_correct_type(self):
'''
Tests that the item() utility function returns the right type.
'''
result = item('bullet')
self.assertIs(type(result), str)
def test_item_returns_correct_type(self):
'''
Tests that the item() utility function returns the right type.
'''
result = item('bullet')
self.assertIs(type(result), str)