def test_trackable_indexability(session, caching):
"""Test Trackable indexability (via []s)"""
problem_name = 'Test Problem'
problem_key = Problem.create_key(name='Test Problem')
assert len(problem_key) == 1
with pytest.raises(KeyMissingFromRegistryAndDatabase): # a KeyError
Problem[problem_key]
with pytest.raises(KeyError):
Problem[problem_key]
problem = Problem(problem_name) # This registers the key
indexed_problem = Problem[problem_key]
assert indexed_problem is problem
# Unpacked 1-tuples can also be used to index from the registry
indexed_problem = Problem[problem_key.human_id]
assert indexed_problem is problem
session.add(problem)
session.commit()
Trackable.clear_all() # This deregisters the key
# This registers the key since it is found in the database
indexed_problem = Problem[problem_key]
assert indexed_problem is problem
Trackable.clear_all() # This deregisters the key
# Unpacked 1-tuples can also be used to index from the database
indexed_problem = Problem[problem_key.human_id]
assert indexed_problem is problem
def test_uri_formation_and_instantiation_with_null_query_value(
session, caching):
"""Test URI formation and instantiation"""
cls = Community
# Register existing for builders since registry is cleared below
Trackable.register_existing(session)
builder = Builder(cls, optional=False)
inst = builder.build(org=None) # org is a query parameter
uri = inst.uri
assert 'org' not in uri, 'org should not be in uri since it is null'
assert '?' not in uri, 'there should be no query string since org is null'
assert cls.URI_TYPE is UriType.NATURAL
instantiated_via_uri = IntertwineModel.instantiate_uri(uri)
assert instantiated_via_uri is inst
session.add(inst)
session.commit()
uri = inst.uri
Trackable.clear_all() # Deregister key to test retrieve from db
instantiated_from_db_via_uri = IntertwineModel.instantiate_uri(uri)
assert instantiated_from_db_via_uri is inst
def decode(session, json_path, *args, **options):
"""
Load JSON files within a path and return data structures
Given a path to a JSON file or a directory containing JSON files,
returns a dictionary where the keys are classes and the values are
corresponding sets of objects updated from the JSON file(s).
Calls another function to actually decode the json_data. This
other function's name begins with 'decode_' and ends with the last
directory in the absolute json_path: decode_<dir_name>(json_data)
Usage:
>>> json_path = '/data/problems/problems00.json' # load a JSON file
>>> u0 = decode(json_path) # get updates from data load
>>> json_path = '/data/problems/' # load all JSON files in a directory
>>> u1 = decode(json_path) # get updates from next data load
>>> u1_problems = u1['Problem'] # get set of updated problems
>>> u1_connections = u1['ProblemConnection'] # set of updated connections
>>> u1_ratings = u1['ProblemConnectionRating'] # set of updated ratings
>>> p0 = Problem('poverty') # get existing 'Poverty' problem
>>> p1 = Problem('homelessness') # get existing 'Homelessness' problem
>>> p2 = Problem['domestic_violence'] # Problem is subscriptable
>>> for p in Problem: # Problem is iterable
... print(p)
"""
# Gather valid json_paths based on the given file or directory
json_paths = []
if os.path.isfile(json_path):
if (json_path.rsplit('.', 1)[-1].lower() == 'json'
and 'schema' not in os.path.basename(json_path).lower()):
json_paths.append(json_path)
elif os.path.isdir(json_path):
json_paths = [
os.path.join(json_path, f) for f in os.listdir(json_path)
if (os.path.isfile(os.path.join(json_path, f)) and f.rsplit(
'.', 1)[-1].lower() == 'json' and 'schema' not in f.lower())
]
if len(json_paths) == 0:
raise InvalidJSONPath(path=json_path)
# Load raw json_data from each of the json_paths
json_data = []
for path in json_paths:
with io.open(path) as json_file:
# TODO: May need to change this to load incrementally in the future
json_data.append(json.load(json_file))
# Determine the decode function based on directory name and then call it
if os.path.isfile(json_path):
dir_name = os.path.abspath(json_path).rsplit('/', 2)[-2]
else:
dir_name = os.path.abspath(json_path).rsplit('/', 1)[-1]
function_name = 'decode_' + dir_name
module = sys.modules[__name__]
decode_function = getattr(module, function_name)
Trackable.register_existing(session)
return decode_function(json_data)
def test_trackable_tget(session, caching):
"""Test Trackable get (tget)"""
problem_name = 'Test Problem'
problem_key = Problem.create_key(name='Test Problem')
assert len(problem_key) == 1
nada = 'nada'
tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=False)
assert tget_problem == nada
tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=True)
assert tget_problem == nada
problem = Problem(problem_name) # This registers the key
tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=False)
assert tget_problem is problem
tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=True)
assert tget_problem is problem
# Unpacked 1-tuples can also be used to get from the registry
tget_problem = Problem.tget(problem_key.human_id, query_on_miss=False)
assert tget_problem is problem
session.add(problem)
session.commit()
Trackable.clear_all() # This deregisters the key
tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=False)
assert tget_problem == nada
# This registers the key since it is found in the database
tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=True)
assert tget_problem is problem
tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=False)
assert tget_problem is problem
Trackable.clear_all() # This deregisters the key
# Unpacked 1-tuples can also be used to get from the database
tget_problem = Problem.tget(problem_key.human_id, query_on_miss=True)
assert tget_problem is problem
def test_decode_same_data(session, caching):
"""Test decoding incrementally"""
create_geo_data(session)
json_path = os.path.join(PROBLEM_DATA_DIRECTORY, 'problems02.json')
u2 = decode(session, json_path)
for updates in u2.values():
session.add_all(updates)
session.commit()
p2 = Problem.query.filter_by(name='Domestic Violence').one()
assert p2 is Problem['domestic_violence']
# Simulate impact of app restart on Trackable by clearing it:
Trackable.clear_all()
# Try reloading existing data (none should be loaded):
u2_repeat = decode(session, json_path)
for updates in u2_repeat.values():
assert len(updates) == 0
def decode_problems(json_data):
"""
Return entities created from problem JSON data
Takes as input a list of json data loads, each from a separate JSON
file and returns a dictionary where the keys are classes and the
values are corresponding sets of objects updated from the JSON
file(s).
Resets tracking of updates via the Trackable metaclass each time it
is called.
"""
Trackable.clear_updates()
for json_data_load in json_data:
for data_key, data_value in json_data_load.items():
Problem(name=data_key, **data_value)
return Trackable.catalog_updates()
def test_uri_formation_and_instantiation(session, caching):
"""Test URI formation and instantiation"""
for class_name, cls in Trackable._classes.items():
# Register existing for builders since registry is cleared below
Trackable.register_existing(session)
builder = Builder(cls, optional=False)
inst = builder.build()
uri = inst.uri
if cls.URI_TYPE is UriType.NATURAL:
instantiated_via_uri = IntertwineModel.instantiate_uri(uri)
assert instantiated_via_uri is inst
session.add(inst)
session.commit()
uri = inst.uri
Trackable.clear_all() # Deregister key to test retrieve from db
instantiated_from_db_via_uri = IntertwineModel.instantiate_uri(uri)
assert instantiated_from_db_via_uri is inst
def erase_data(session, confirm=None):
"""
Erase all data from database and clear tracking of all instances
For Trackable classes, erases all data from the database and clears
tracking of all instances. Prompts the user to confirm by typing
'ERASE'. Can alternatively take an optional confirm parameter with
a value of 'ERASE' to proceed without a user prompt.
"""
if confirm != 'ERASE':
prompt = ('This will erase *all* data from the database and '
'clear tracking of all instances.\n'
'Type "ERASE" (all caps) to proceed. '
'Anything else will abort.\n>')
confirm_again = raw_input(prompt)
if confirm_again != 'ERASE':
print('Aborting - leaving data untouched.')
return
print('Processing...')
# limit data to Trackable classes with existing tables
engine = session.bind
inspector = Inspector.from_engine(engine)
table_names = set(inspector.get_table_names())
classes = [
x for x in Trackable._classes.values()
if x.__tablename__ in table_names
]
print('Erase Data classes: ', classes)
Trackable.register_existing(session, *classes)
for cls in classes:
for inst in cls:
session.delete(inst)
session.commit()
Trackable.clear_instances()
print('Erase data has completed')
def test_incremental_decode(session, caching):
"""Test decoding multiple files incrementally"""
create_geo_data(session)
# Initial data load:
json_path = os.path.join(PROBLEM_DATA_DIRECTORY, 'problems00.json')
u0 = decode(session, json_path)
for updates in u0.values():
session.add_all(updates)
session.commit()
p0 = Problem.query.filter_by(name='Poverty').one()
assert p0 is Problem['poverty']
# Simulate impact of app restart on Trackable by clearing it:
Trackable.clear_all()
# Next data load:
json_path = os.path.join(PROBLEM_DATA_DIRECTORY, 'problems01.json')
u1 = decode(session, json_path)
for updates in u1.values():
session.add_all(updates)
session.commit()
p0 = Problem.query.filter_by(name='Poverty').one()
assert p0 is Problem['poverty']
p1 = Problem.query.filter_by(name='Homelessness').one()
assert p1 is Problem['homelessness']
# Simulate impact of app restart on Trackable by clearing it:
Trackable.clear_all()
# Next data load:
json_path = os.path.join(PROBLEM_DATA_DIRECTORY, 'problems02.json')
u2 = decode(session, json_path)
for updates in u2.values():
session.add_all(updates)
session.commit()
# Make sure they're still the same problems
p0 = Problem.query.filter_by(name='Poverty').one()
assert p0 is Problem['poverty']
p1 = Problem.query.filter_by(name='Homelessness').one()
assert p1 is Problem['homelessness']
p2 = Problem.query.filter_by(name='Domestic Violence').one()
assert p2 is Problem['domestic_violence']
c1 = ProblemConnection.query.filter(
ProblemConnection.axis == 'scoped',
ProblemConnection.broader == p0,
ProblemConnection.narrower == p1).one()
assert c1.axis == 'scoped'
assert c1.broader is p0
assert c1.narrower is p1
c2 = ProblemConnection.query.filter(
ProblemConnection.axis == 'causal',
ProblemConnection.driver == p2,
ProblemConnection.impact == p1).one()
assert c2.axis == 'causal'
assert c2.driver is p2
assert c2.impact is p1
rs1 = ProblemConnectionRating.query.filter(
ProblemConnectionRating.connection == c1)
assert len(rs1.all()) > 0
for r in rs1:
assert r.connection is c1
geo = Geo['us/tx/austin']
rs2 = ProblemConnectionRating.query.filter(
ProblemConnectionRating.problem == p1,
ProblemConnectionRating.connection == c2,
ProblemConnectionRating.org.is_(None),
ProblemConnectionRating.geo == geo)
assert len(rs2.all()) > 0
for r in rs2:
assert r.problem == p1
assert r.connection == c2
assert r.org is None
assert r.geo is geo
def teardown():
Trackable.clear_all()
transaction.rollback()
connection.close()
session.remove()
def teardown():
Trackable.clear_all()
intertwine_db.drop_all()
if os.path.exists(TESTDB_PATH):
os.unlink(TESTDB_PATH)
def teardown():
Trackable.clear_all()
ctx.pop()
def caching(app, request):
"""Enable caching of model instances"""
with Trackable.caching():
yield