def test_trackable_indexability(session, caching): """Test Trackable indexability (via []s)""" problem_name = 'Test Problem' problem_key = Problem.create_key(name='Test Problem') assert len(problem_key) == 1 with pytest.raises(KeyMissingFromRegistryAndDatabase): # a KeyError Problem[problem_key] with pytest.raises(KeyError): Problem[problem_key] problem = Problem(problem_name) # This registers the key indexed_problem = Problem[problem_key] assert indexed_problem is problem # Unpacked 1-tuples can also be used to index from the registry indexed_problem = Problem[problem_key.human_id] assert indexed_problem is problem session.add(problem) session.commit() Trackable.clear_all() # This deregisters the key # This registers the key since it is found in the database indexed_problem = Problem[problem_key] assert indexed_problem is problem Trackable.clear_all() # This deregisters the key # Unpacked 1-tuples can also be used to index from the database indexed_problem = Problem[problem_key.human_id] assert indexed_problem is problem
def test_uri_formation_and_instantiation_with_null_query_value( session, caching): """Test URI formation and instantiation""" cls = Community # Register existing for builders since registry is cleared below Trackable.register_existing(session) builder = Builder(cls, optional=False) inst = builder.build(org=None) # org is a query parameter uri = inst.uri assert 'org' not in uri, 'org should not be in uri since it is null' assert '?' not in uri, 'there should be no query string since org is null' assert cls.URI_TYPE is UriType.NATURAL instantiated_via_uri = IntertwineModel.instantiate_uri(uri) assert instantiated_via_uri is inst session.add(inst) session.commit() uri = inst.uri Trackable.clear_all() # Deregister key to test retrieve from db instantiated_from_db_via_uri = IntertwineModel.instantiate_uri(uri) assert instantiated_from_db_via_uri is inst
def decode(session, json_path, *args, **options): """ Load JSON files within a path and return data structures Given a path to a JSON file or a directory containing JSON files, returns a dictionary where the keys are classes and the values are corresponding sets of objects updated from the JSON file(s). Calls another function to actually decode the json_data. This other function's name begins with 'decode_' and ends with the last directory in the absolute json_path: decode_<dir_name>(json_data) Usage: >>> json_path = '/data/problems/problems00.json' # load a JSON file >>> u0 = decode(json_path) # get updates from data load >>> json_path = '/data/problems/' # load all JSON files in a directory >>> u1 = decode(json_path) # get updates from next data load >>> u1_problems = u1['Problem'] # get set of updated problems >>> u1_connections = u1['ProblemConnection'] # set of updated connections >>> u1_ratings = u1['ProblemConnectionRating'] # set of updated ratings >>> p0 = Problem('poverty') # get existing 'Poverty' problem >>> p1 = Problem('homelessness') # get existing 'Homelessness' problem >>> p2 = Problem['domestic_violence'] # Problem is subscriptable >>> for p in Problem: # Problem is iterable ... print(p) """ # Gather valid json_paths based on the given file or directory json_paths = [] if os.path.isfile(json_path): if (json_path.rsplit('.', 1)[-1].lower() == 'json' and 'schema' not in os.path.basename(json_path).lower()): json_paths.append(json_path) elif os.path.isdir(json_path): json_paths = [ os.path.join(json_path, f) for f in os.listdir(json_path) if (os.path.isfile(os.path.join(json_path, f)) and f.rsplit( '.', 1)[-1].lower() == 'json' and 'schema' not in f.lower()) ] if len(json_paths) == 0: raise InvalidJSONPath(path=json_path) # Load raw json_data from each of the json_paths json_data = [] for path in json_paths: with io.open(path) as json_file: # TODO: May need to change this to load incrementally in the future json_data.append(json.load(json_file)) # Determine the decode function based on directory name and then call it if os.path.isfile(json_path): dir_name = os.path.abspath(json_path).rsplit('/', 2)[-2] else: dir_name = os.path.abspath(json_path).rsplit('/', 1)[-1] function_name = 'decode_' + dir_name module = sys.modules[__name__] decode_function = getattr(module, function_name) Trackable.register_existing(session) return decode_function(json_data)
def test_trackable_tget(session, caching): """Test Trackable get (tget)""" problem_name = 'Test Problem' problem_key = Problem.create_key(name='Test Problem') assert len(problem_key) == 1 nada = 'nada' tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=False) assert tget_problem == nada tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=True) assert tget_problem == nada problem = Problem(problem_name) # This registers the key tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=False) assert tget_problem is problem tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=True) assert tget_problem is problem # Unpacked 1-tuples can also be used to get from the registry tget_problem = Problem.tget(problem_key.human_id, query_on_miss=False) assert tget_problem is problem session.add(problem) session.commit() Trackable.clear_all() # This deregisters the key tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=False) assert tget_problem == nada # This registers the key since it is found in the database tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=True) assert tget_problem is problem tget_problem = Problem.tget(problem_key, default=nada, query_on_miss=False) assert tget_problem is problem Trackable.clear_all() # This deregisters the key # Unpacked 1-tuples can also be used to get from the database tget_problem = Problem.tget(problem_key.human_id, query_on_miss=True) assert tget_problem is problem
def test_decode_same_data(session, caching): """Test decoding incrementally""" create_geo_data(session) json_path = os.path.join(PROBLEM_DATA_DIRECTORY, 'problems02.json') u2 = decode(session, json_path) for updates in u2.values(): session.add_all(updates) session.commit() p2 = Problem.query.filter_by(name='Domestic Violence').one() assert p2 is Problem['domestic_violence'] # Simulate impact of app restart on Trackable by clearing it: Trackable.clear_all() # Try reloading existing data (none should be loaded): u2_repeat = decode(session, json_path) for updates in u2_repeat.values(): assert len(updates) == 0
def decode_problems(json_data): """ Return entities created from problem JSON data Takes as input a list of json data loads, each from a separate JSON file and returns a dictionary where the keys are classes and the values are corresponding sets of objects updated from the JSON file(s). Resets tracking of updates via the Trackable metaclass each time it is called. """ Trackable.clear_updates() for json_data_load in json_data: for data_key, data_value in json_data_load.items(): Problem(name=data_key, **data_value) return Trackable.catalog_updates()
def test_uri_formation_and_instantiation(session, caching): """Test URI formation and instantiation""" for class_name, cls in Trackable._classes.items(): # Register existing for builders since registry is cleared below Trackable.register_existing(session) builder = Builder(cls, optional=False) inst = builder.build() uri = inst.uri if cls.URI_TYPE is UriType.NATURAL: instantiated_via_uri = IntertwineModel.instantiate_uri(uri) assert instantiated_via_uri is inst session.add(inst) session.commit() uri = inst.uri Trackable.clear_all() # Deregister key to test retrieve from db instantiated_from_db_via_uri = IntertwineModel.instantiate_uri(uri) assert instantiated_from_db_via_uri is inst
def erase_data(session, confirm=None): """ Erase all data from database and clear tracking of all instances For Trackable classes, erases all data from the database and clears tracking of all instances. Prompts the user to confirm by typing 'ERASE'. Can alternatively take an optional confirm parameter with a value of 'ERASE' to proceed without a user prompt. """ if confirm != 'ERASE': prompt = ('This will erase *all* data from the database and ' 'clear tracking of all instances.\n' 'Type "ERASE" (all caps) to proceed. ' 'Anything else will abort.\n>') confirm_again = raw_input(prompt) if confirm_again != 'ERASE': print('Aborting - leaving data untouched.') return print('Processing...') # limit data to Trackable classes with existing tables engine = session.bind inspector = Inspector.from_engine(engine) table_names = set(inspector.get_table_names()) classes = [ x for x in Trackable._classes.values() if x.__tablename__ in table_names ] print('Erase Data classes: ', classes) Trackable.register_existing(session, *classes) for cls in classes: for inst in cls: session.delete(inst) session.commit() Trackable.clear_instances() print('Erase data has completed')
def test_incremental_decode(session, caching): """Test decoding multiple files incrementally""" create_geo_data(session) # Initial data load: json_path = os.path.join(PROBLEM_DATA_DIRECTORY, 'problems00.json') u0 = decode(session, json_path) for updates in u0.values(): session.add_all(updates) session.commit() p0 = Problem.query.filter_by(name='Poverty').one() assert p0 is Problem['poverty'] # Simulate impact of app restart on Trackable by clearing it: Trackable.clear_all() # Next data load: json_path = os.path.join(PROBLEM_DATA_DIRECTORY, 'problems01.json') u1 = decode(session, json_path) for updates in u1.values(): session.add_all(updates) session.commit() p0 = Problem.query.filter_by(name='Poverty').one() assert p0 is Problem['poverty'] p1 = Problem.query.filter_by(name='Homelessness').one() assert p1 is Problem['homelessness'] # Simulate impact of app restart on Trackable by clearing it: Trackable.clear_all() # Next data load: json_path = os.path.join(PROBLEM_DATA_DIRECTORY, 'problems02.json') u2 = decode(session, json_path) for updates in u2.values(): session.add_all(updates) session.commit() # Make sure they're still the same problems p0 = Problem.query.filter_by(name='Poverty').one() assert p0 is Problem['poverty'] p1 = Problem.query.filter_by(name='Homelessness').one() assert p1 is Problem['homelessness'] p2 = Problem.query.filter_by(name='Domestic Violence').one() assert p2 is Problem['domestic_violence'] c1 = ProblemConnection.query.filter( ProblemConnection.axis == 'scoped', ProblemConnection.broader == p0, ProblemConnection.narrower == p1).one() assert c1.axis == 'scoped' assert c1.broader is p0 assert c1.narrower is p1 c2 = ProblemConnection.query.filter( ProblemConnection.axis == 'causal', ProblemConnection.driver == p2, ProblemConnection.impact == p1).one() assert c2.axis == 'causal' assert c2.driver is p2 assert c2.impact is p1 rs1 = ProblemConnectionRating.query.filter( ProblemConnectionRating.connection == c1) assert len(rs1.all()) > 0 for r in rs1: assert r.connection is c1 geo = Geo['us/tx/austin'] rs2 = ProblemConnectionRating.query.filter( ProblemConnectionRating.problem == p1, ProblemConnectionRating.connection == c2, ProblemConnectionRating.org.is_(None), ProblemConnectionRating.geo == geo) assert len(rs2.all()) > 0 for r in rs2: assert r.problem == p1 assert r.connection == c2 assert r.org is None assert r.geo is geo
def teardown(): Trackable.clear_all() transaction.rollback() connection.close() session.remove()
def teardown(): Trackable.clear_all() intertwine_db.drop_all() if os.path.exists(TESTDB_PATH): os.unlink(TESTDB_PATH)
def teardown(): Trackable.clear_all() ctx.pop()
def caching(app, request): """Enable caching of model instances""" with Trackable.caching(): yield