def test_type(self):
'''
utilities.load: Test for the type of configuration files.
'''
for file in self.files:
result = Load.loadJSONFile(file)
self.assertIs(type(result), type({}))
def __init__(self, instance='dev'):
instances = {'dev': 'config/dev.json', 'prod': 'config/prod.json'}
config = Load.loadJSONFile(instances.get(instance))
ckan = ckanapi.RemoteCKAN(user_agent='ckanapi/1.0',
apikey=config['ckan']['api'],
address=config['ckan']['url'])
self.ckan = ckan
def test_type(self):
'''
utilities.load: Test for the type of configuration files.
'''
for file in self.files:
result = Load.loadJSONFile(file)
self.assertIs(type(result), type({}))
def test_structure(self):
'''
utilities.load: Test for the structure of configuration files.
'''
keys = ['endpoints', 'database', 'ckan']
for file in self.files:
result = Load.loadJSONFile(file)
for key in result.keys():
self.assertIn(key, keys)
def test_api_key(self):
'''
utilities.load: Test that API key has been properly loaded from environment variable.
'''
key = os.environ.get('CKAN_KEY')
for file in self.files:
result = Load.loadJSONFile(file)
self.assertIn(key, result['ckan']['api'])
def test_api_key(self):
'''
utilities.load: Test that API key has been properly loaded from environment variable.
'''
key = os.environ.get('CKAN_KEY')
for file in self.files:
result = Load.loadJSONFile(file)
self.assertIn(key, result['ckan']['api'])
def createTables(instance='config/dev.json', verbose=True):
'''
Creates tables in a PostgreSQL instance.
'''
#
# Loading database information
# from config file.
#
database = loadJSONFile(instance)['database']
#
# TODO: add environment variables
# to these default values.
#
conn = psycopg2.connect(
host=HOST_DATABASE,
dbname=DB_NAME,
user=DB_USER,
password=DB_PASSWORD
)
cur = conn.cursor()
#
# Build each table.
#
for table in database:
#
# Construct SQL statement.
#
table_sql = ""
for column in table['columns']:
s = '%s %s, ' % (column['field_name'], column['type'])
table_sql += s
statement = 'CREATE TABLE IF NOT EXISTS "%s" (%sPRIMARY KEY (%s))' % (table['name'], table_sql, ", ".join(table['primary_key']))
#
# Make statements to the database.
#
try:
cur.execute(statement)
conn.commit()
print("%s table `%s` created." % (item('bullet'), str(table['name'])))
except Exception as e:
print('%s Table `%s` could not be created.' % (item('error'), table['name']))
if verbose:
print(e)
return False
#
# Close communication.
#
cur.close()
conn.close()
def test_structure(self):
'''
utilities.load: Test for the structure of configuration files.
'''
keys = ['version', 'description', 'repository', 'maintainer', 'ckan', 'database']
for file in self.files:
result = Load.loadJSONFile(file)
for key in result.keys():
self.assertIn(key, keys)
def test_structure(self):
'''
database.configuration.file: Test for the structure of configuration files.
'''
keys = ['name', 'primary_key', 'columns']
for file in self.files:
result = Load.loadJSONFile(file)
for table in result['database']:
for key in table.keys():
self.assertIn(key, keys)
def test_structure(self):
'''
database.configuration.file: Test for the structure of configuration files.
'''
keys = ['name', 'primary_key', 'columns']
for file in self.files:
result = Load.loadJSONFile(file)
for table in result['database']:
for key in table.keys():
self.assertIn(key, keys)
def main():
'''
Application wrapper.
'''
dir_name = os.path.dirname(os.path.realpath(__file__))
file = os.path.join(dir_name, 'config', 'config.json')
config = Load.loadJSONFile(file)
for endpoint in config['endpoints']:
data = collectData(endpoint['url'])
storeData(data, endpoint['name'])
def __init__(self, instance='dev'):
instances = {
'dev': 'config/dev.json',
'prod': 'config/prod.json'
}
config = Load.loadJSONFile(instances.get(instance))
ckan = ckanapi.RemoteCKAN(
user_agent='ckanapi/1.0',
apikey=config['ckan']['api'],
address=config['ckan']['url']
)
self.ckan = ckan
def test_structure(self):
'''
utilities.load: Test for the structure of configuration files.
'''
keys = [
'version', 'description', 'repository', 'maintainer', 'ckan',
'database'
]
for file in self.files:
result = Load.loadJSONFile(file)
for key in result.keys():
self.assertIn(key, keys)
def test_column_names(self):
'''
database.configuration.tables: Test that column names are correct.
'''
for file in self.files:
database = Load.loadJSONFile(file)['database']
for table in database:
#
# Don't test the 'test' table.
#
if table['name'] == 'test':
continue
else:
for column in table['columns']:
self.assertIn(column['field_name'], self.tables.get(table['name']))
def test_column_names(self):
'''
database.configuration.tables: Test that column names are correct.
'''
for file in self.files:
database = Load.loadJSONFile(file)['database']
for table in database:
#
# Don't test the 'test' table.
#
if table['name'] == 'test':
continue
else:
for column in table['columns']:
self.assertIn(column['field_name'],
self.tables.get(table['name']))
def createTables(instance='config/config.json'):
'''
Creates tables a database based on the
tables specified in the configuration file.
'''
#
# Loading database information
# from config file.
#
database = loadJSONFile(instance)['database']
#
# Build each table.
#
for table in database:
#
# Construct SQL statement.
#
table_sql = ""
for column in table['columns']:
s = '%s %s, ' % (column['field_name'], column['type'])
table_sql += s
statement = 'CREATE TABLE IF NOT EXISTS "%s" (%sPRIMARY KEY (%s))' % (table['name'], table_sql, ", ".join(table['primary_key']))
#
# Send statements to the database.
#
try:
scraperwiki.sql.execute(statement)
print("%s table `%s` created." % (item('bullet'), str(table['name'])))
except Exception as e:
print('%s Table `%s` could not be created.' % (item('error'), table['name']))
print(e)
return False
#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Function to load resource information. This
function uses instances from datasets to create
an iterable array of resources ids. Then uses that
array to create instances of resources.
'''
import app.utilities.load as Load
from app.classes.dataset import Dataset
from app.classes.resource import Resource
config = Load.loadJSONFile('config/dev.json')
def _fields(config, key):
'''
Extract field names from the database
property in the configuration file.
'''
result = None
for table in config['database']:
if table['name'] == key:
result = [ i['field_name'] for i in table['columns'] ]
return result
def fetchResourceData(dataset_id):
'''
#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Unit tests for checking test data is
fetched correctly.
'''
import unittest
import app.utilities.load as Load
from app.functions.fetch_class_data import fetchClassData, _fields
config = Load.loadJSONFile('config/dev.json')
class TestFunctionFetchClassData(unittest.TestCase):
'''
Performs test for checking function to retrieve
a specific class data.
'''
def setUp(self):
self.classes = [{
'class': 'country',
'data': 'irq'
}, {
'class': 'dataset',
'data': 'ebola-cases-2014'
}, {
'class': 'gallery_item',
'data': '753ccf05-872f-4c8d-9cc2-8e562f6fc1d5'
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Unit tests for checking test data is
fetched correctly.
"""
import unittest
import app.utilities.load as Load
from app.functions.fetch_class_data import fetchClassData, _fields
config = Load.loadJSONFile("config/dev.json")
class TestFunctionFetchClassData(unittest.TestCase):
"""
Performs test for checking function to retrieve
a specific class data.
"""
def setUp(self):
self.classes = [
{"class": "country", "data": "irq"},
{"class": "dataset", "data": "ebola-cases-2014"},
{"class": "gallery_item", "data": "753ccf05-872f-4c8d-9cc2-8e562f6fc1d5"},
{"class": "resource", "data": "c59b5722-ca4b-41ca-a446-472d6d824d01"},
{"class": "revision", "data": "b1e6f54d-d086-4c03-9739-84ae400e5aa1"},
{"class": "user", "data": "luiscape"},
]