class TestSeriesService: def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) """ @pytest.fixture(scope="class", autouse=True) def build_db(self): """ #Builds an empty sqlite (in-memory) database for testing #:return: None """ # path to the ddl script for building the database ddlpath= abspath(join(dirname(__file__), 'data/empty.sql')) # create and empty sqlite database for testing db = dbconnection.createConnection('sqlite', ':memory:') # read the ddl script and remove the first (BEGIN TRANSACTION) and last (COMMIT) lines ddl = open(ddlpath, 'r').read() ddl = ddl.replace('BEGIN TRANSACTION;','') ddl = ddl.replace('COMMIT;','') # execute each statement to build the odm2 database for line in ddl.split(');')[:-1]: try: db.engine.execute(line + ');') except Exception as e: print e self.write = CreateODM2(db) self.engine= db.engine globals['write'] = self.write globals['engine'] = self.engine globals['db'] = db # return self.write, self.engine def setup(self): self.writer = globals['write'] self.engine = globals['engine'] self.db = globals['db'] """ def test_get_db_version(self): version = test_util.add_version(self.session) db_version = self.series_service.get_db_version() assert version.version_number == db_version def test_get_all_sites_empty(self): sites = self.series_service.get_used_sites() #assert len(sites) == 0 assert sites is None def test_create_qualifier(self): qual = Qualifier() qual.code = "ABC123" qual.description = "This is a test" self.series_service.create_qualifier_by_qual(qual) assert qual.id is not None def test_get_qualifier_by_code(self): assert self.series_service.get_all_qualifiers() == [] qual= self.series_service.create_qualifier("ABC123","This is a test") db_qual = self.series_service.get_qualifier_by_code("ABC123") assert qual.id == db_qual.id def test_get_qualifiers(self): assert self.series_service.get_all_qualifiers() == [] qual= self.series_service.create_qualifier("ABC123","This is a test") db_qual = self.series_service.get_all_qualifiers()[0] assert qual.id == db_qual.id def test_get_all_sites(self): assert self.series_service.get_used_sites() is None site = test_util.add_site(self.session) sites = self.series_service.get_used_sites() assert sites is None if isinstance(sites, list) and len(sites) > 0: assert site.code == sites[0].code series = test_util.add_series(self.session) site = series.site sites = self.series_service.get_used_sites() assert len(sites) == 1 if isinstance(sites, list) and len(sites) > 0: assert site.code == sites[0].code def test_get_site_by_id_fail(self): assert self.series_service.get_site_by_id(0) == None site = test_util.add_site(self.session) db_site = self.series_service.get_site_by_id(10) assert db_site == None def test_get_site_by_id(self): site = test_util.add_site(self.session) db_site = self.series_service.get_site_by_id(site.id) assert db_site != None assert site.code == db_site.code def test_get_all_variables(self): assert self.series_service.get_all_variables() == [] variable = test_util.add_variable(self.session) variables = self.series_service.get_all_variables() assert len(variables) == 1 assert variable.code == variables[0].code def test_get_variable_by_id(self): assert self.series_service.get_variable_by_id(10) == None variable = test_util.add_variable(self.session) db_var = self.series_service.get_variable_by_id(variable.id) assert db_var != None assert db_var.code == variable.code def test_get_variables_by_site_code(self): assert self.series_service.get_variables_by_site_code('ABC123') == [] series = test_util.add_series(self.session) variable = series.variable db_variables = self.series_service.get_variables_by_site_code(series.site_code) assert db_variables != None assert variable.code == db_variables[0].code def test_get_all_units(self): assert self.series_service.get_all_units() == [] unit = test_util.add_unit(self.session) units = self.series_service.get_all_units() assert len(units) == 1 assert unit.name == units[0].name def test_get_unit_by_name(self): assert self.series_service.get_unit_by_name("FAIL") == None unit = test_util.add_unit(self.session) db_unit = self.series_service.get_unit_by_name(unit.name) assert unit.id == db_unit.id def test_get_unit_by_id(self): assert self.series_service.get_unit_by_id(10) == None unit = test_util.add_unit(self.session) db_unit = self.series_service.get_unit_by_id(unit.id) assert unit.name == db_unit.name def test_get_all_series(self): assert self.series_service.get_all_series() == [] series = test_util.add_series(self.session) all_series = self.series_service.get_all_series() assert all_series != [] assert series.id == all_series[0].id def test_get_series_by_id(self): assert self.series_service.get_series_by_id(10) == None series = test_util.add_series(self.session) db_series = self.series_service.get_series_by_id(series.id) assert series.id == db_series.id def test_get_series_by_id_quint(self): assert self.series_service.get_series_by_id_quint(10, 10, 10, 10, 10) == None series = test_util.add_series(self.session) db_series = self.series_service.get_series_by_id_quint(series.site_id, series.variable_id, series.method_id, series.source_id, series.quality_control_level_id) assert series.id == db_series.id def test_series_exists(self): assert self.series_service.series_exists_quint(10, 10, 10, 10, 10) == False series = test_util.add_series(self.session) site_id = series.site_id var_id = series.variable_id method_id = series.method_id source_id = series.source_id qcl_id = series.quality_control_level_id assert self.series_service.series_exists_quint(site_id, var_id, method_id, source_id, qcl_id) == True ''' def test_save_series(self): series = Series() site = test_util.add_site(self.session) variable = test_util.add_variable(self.session) method = test_util.add_method(self.session) source = test_util.add_source(self.session) qcl = test_util.add_qcl(self.session) series.site_id = site.id series.variable_id = variable.id series.method_id = method.id series.source_id = source.id series.quality_control_level_id = qcl.id dvs = [] for val in range(10): dv = DataValue() dv.data_value = val dv.site_id = site.id dv.variable_id = variable.id dv.method_id = method.id dv.source_id = source.id dv.quality_control_level_id = qcl.id dvs.append(dv) print series.variable_code assert self.series_service.save_series(series) assert self.series_service.series_exists(site.id, variable.id, method.id, source.id, qcl.id) assert not self.series_service.save_series(series) ''' def test_get_data_value_by_id(self): assert self.series_service.get_data_value_by_id(10) == None data_values = test_util.add_series(self.session).data_values dv = data_values[0] db_dv = self.series_service.get_data_value_by_id(dv.id) assert dv.data_value == db_dv.data_value def test_get_qcl_by_id(self): assert self.series_service.get_qcl_by_id(10) == None qcl = test_util.add_qcl(self.session) db_qcl = self.series_service.get_qcl_by_id(qcl.id) assert qcl.code == db_qcl.code def test_get_all_qcls(self): assert self.series_service.get_all_qcls() == [] qcl = test_util.add_qcl(self.session) all_qcls = self.series_service.get_all_qcls() assert len(all_qcls) == 1 assert qcl.id == all_qcls[0].id def test_get_all_methods(self): assert self.series_service.get_all_methods() == [] method = test_util.add_method(self.session) all_methods = self.series_service.get_all_methods() assert len(all_methods) == 1 assert method.id == all_methods[0].id def test_get_method_by_id(self): assert self.series_service.get_method_by_id(10) == None method = test_util.add_method(self.session) db_method = self.series_service.get_method_by_id(method.id) assert method.description == db_method.description def test_delete_dvs(self): series = test_util.add_series(self.session) dvs = series.data_values subset = dvs[:5] self.series_service.delete_dvs([x.local_date_time for x in subset]) assert self.series_service.get_data_value_by_id(subset[0].id) == None series = self.series_service.get_series_by_id(series.id) # Reload assert len(series.data_values) == 5 def test_update_dvs(self): series = test_util.add_series(self.session) dvs = series.data_values subset = dvs[:5] for i in range(len(subset)): subset[i].data_value = 100 self.series_service.update_dvs(subset) series = self.series_service.get_series_by_id(series.id) assert series.data_values[0].data_value == 100 def test_create_new_series(self): site = test_util.add_site(self.session) variable = test_util.add_variable(self.session) method = test_util.add_method(self.session) source = test_util.add_source(self.session) qcl = test_util.add_qcl(self.session) dvs = [] for val in range(10): dv = DataValue() dv.data_value = val dv.site_id = site.id dv.variable_id = variable.id dv.method_id = method.id dv.source_id = source.id dv.quality_control_level_id = qcl.id dvs.append(dv) series = self.series_service.create_new_series(dvs, site.id, variable.id, method.id, source.id, qcl.id) assert series != None assert len(series.data_values) == 10 assert series.site_id == site.id assert series.variable_id == variable.id def test_update_series(self): series = test_util.add_series(self.session) series.site_code = "NEW" series.variable_code = "NEW" self.series_service.update_series(series) series = self.series_service.get_series_by_id(series.id) assert series.site_code == "NEW" assert series.variable_code == "NEW" def test_create_method(self): description = "This tests creating a method" link = "http://www.example.com" method = self.series_service.create_method(description, link) assert method.id != None assert method.description == description assert method.link == link def test_create_variable(self): unit = test_util.add_unit(self.session) variable = self.series_service.create_variable("Code", "Name", "Speciation", unit.id, "SampleMedium", "ValueType", True, # is_regular 5.0, # time_support unit.id, # time_unit_id "DataType", "GeneralCategory", -999.0) # no_data_value assert variable.id != None assert variable.code == "Code" assert variable.variable_unit_id == unit.id def test_create_qcl(self): qcl = self.series_service.create_qcl("Code", "Definition", "Explanation") assert qcl.id != None assert qcl.code == "Code" def test_delete_series(self): series = test_util.add_series(self.session) assert self.series_service.get_series_by_id(series.id) != None self.series_service.delete_series(series) assert self.series_service.get_series_by_id(series.id) == None def test_delete_values(self): series = test_util.add_series(self.session) assert self.series_service.get_series_by_id(series.id) != None self.series_service.delete_values_by_series(series) val = self.series_service.get_series_by_id(series.id) print val assert val != None def test_qcl_exists(self): qcl = test_util.add_qcl(self.session) assert self.series_service.qcl_exists(qcl) == True qcl.code = "00000" qcl.definition = "A new definition" assert self.series_service.qcl_exists(qcl) == False def test_method_exists(self): method = test_util.add_method(self.session) assert self.series_service.method_exists(method) == True method.description = "A new description" assert self.series_service.method_exists(method) == False def test_variable_exists(self): variable = test_util.add_variable(self.session) assert self.series_service.variable_exists(variable) variable.code = "00000" variable.name = "A new name" assert not self.series_service.variable_exists(variable)
class TestSeriesService: def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) def test_get_db_version(self): version = test_util.add_version(self.session) db_version = self.series_service.get_db_version() assert version.version_number == db_version def test_get_all_sites_empty(self): sites = self.series_service.get_used_sites() #assert len(sites) == 0 assert sites is None def test_create_qualifier(self): qual = Qualifier() qual.code = "ABC123" qual.description = "This is a test" self.series_service.create_qualifier_by_qual(qual) assert qual.id is not None def test_get_qualifiers(self): assert self.series_service.get_all_qualifiers() == [] qual= self.series_service.create_qualifier("ABC123","This is a test") db_qual = self.series_service.get_all_qualifiers()[0] assert qual.id == db_qual.id def test_get_all_sites(self): assert self.series_service.get_used_sites() is None site = test_util.add_site(self.session) sites = self.series_service.get_used_sites() assert sites is None if isinstance(sites, list) and len(sites) > 0: assert site.code == sites[0].code series = test_util.add_series(self.session) site = series.site sites = self.series_service.get_used_sites() assert len(sites) == 1 if isinstance(sites, list) and len(sites) > 0: assert site.code == sites[0].code def test_get_site_by_id_fail(self): assert self.series_service.get_site_by_id(0) == None site = test_util.add_site(self.session) db_site = self.series_service.get_site_by_id(10) assert db_site == None def test_get_site_by_id(self): site = test_util.add_site(self.session) db_site = self.series_service.get_site_by_id(site.id) assert db_site != None assert site.code == db_site.code def test_get_all_variables(self): assert self.series_service.get_all_variables() == [] variable = test_util.add_variable(self.session) variables = self.series_service.get_all_variables() assert len(variables) == 1 assert variable.code == variables[0].code def test_get_variable_by_id(self): assert self.series_service.get_variable_by_id(10) == None variable = test_util.add_variable(self.session) db_var = self.series_service.get_variable_by_id(variable.id) assert db_var != None assert db_var.code == variable.code def test_get_variables_by_site_code(self): assert self.series_service.get_variables_by_site_code('ABC123') == [] series = test_util.add_series(self.session) variable = series.variable db_variables = self.series_service.get_variables_by_site_code(series.site_code) assert db_variables != None assert variable.code == db_variables[0].code def test_get_all_units(self): assert self.series_service.get_all_units() == [] unit = test_util.add_unit(self.session) units = self.series_service.get_all_units() assert len(units) == 1 assert unit.name == units[0].name def test_get_unit_by_name(self): assert self.series_service.get_unit_by_name("FAIL") == None unit = test_util.add_unit(self.session) db_unit = self.series_service.get_unit_by_name(unit.name) assert unit.id == db_unit.id def test_get_unit_by_id(self): assert self.series_service.get_unit_by_id(10) == None unit = test_util.add_unit(self.session) db_unit = self.series_service.get_unit_by_id(unit.id) assert unit.name == db_unit.name def test_get_all_series(self): assert self.series_service.get_all_series() == [] series = test_util.add_series(self.session) all_series = self.series_service.get_all_series() assert all_series != [] assert series.id == all_series[0].id def test_get_series_by_id(self): assert self.series_service.get_series_by_id(10) == None series = test_util.add_series(self.session) db_series = self.series_service.get_series_by_id(series.id) assert series.id == db_series.id def test_get_series_by_id_quint(self): assert self.series_service.get_series_by_id_quint(10, 10, 10, 10, 10) == None series = test_util.add_series(self.session) db_series = self.series_service.get_series_by_id_quint(series.site_id, series.variable_id, series.method_id, series.source_id, series.quality_control_level_id) assert series.id == db_series.id def test_series_exists(self): assert self.series_service.series_exists_quint(10, 10, 10, 10, 10) == False series = test_util.add_series(self.session) site_id = series.site_id var_id = series.variable_id method_id = series.method_id source_id = series.source_id qcl_id = series.quality_control_level_id assert self.series_service.series_exists_quint(site_id, var_id, method_id, source_id, qcl_id) == True ## TODO Unittest save_series, save_as, save_as_existing ''' def test_save_series(self): series = Series() site = test_util.add_site(self.session) variable = test_util.add_variable(self.session) method = test_util.add_method(self.session) source = test_util.add_source(self.session) qcl = test_util.add_qcl(self.session) series.site_id = site.id series.variable_id = variable.id series.method_id = method.id series.source_id = source.id series.quality_control_level_id = qcl.id dvs = [] for val in range(10): dv = DataValue() dv.data_value = val dv.site_id = site.id dv.variable_id = variable.id dv.method_id = method.id dv.source_id = source.id dv.quality_control_level_id = qcl.id dvs.append(dv) print series.variable_code assert self.series_service.save_series(series) assert self.series_service.series_exists(site.id, variable.id, method.id, source.id, qcl.id) assert not self.series_service.save_series(series) ''' def test_get_data_value_by_id(self): assert self.series_service.get_data_value_by_id(10) == None data_values = test_util.add_series(self.session).data_values dv = data_values[0] db_dv = self.series_service.get_data_value_by_id(dv.id) assert dv.data_value == db_dv.data_value def test_get_qcl_by_id(self): assert self.series_service.get_qcl_by_id(10) == None qcl = test_util.add_qcl(self.session) db_qcl = self.series_service.get_qcl_by_id(qcl.id) assert qcl.code == db_qcl.code def test_get_all_qcls(self): assert self.series_service.get_all_qcls() == [] qcl = test_util.add_qcl(self.session) all_qcls = self.series_service.get_all_qcls() assert len(all_qcls) == 1 assert qcl.id == all_qcls[0].id def test_get_all_methods(self): assert self.series_service.get_all_methods() == [] method = test_util.add_method(self.session) all_methods = self.series_service.get_all_methods() assert len(all_methods) == 1 assert method.id == all_methods[0].id def test_get_method_by_id(self): assert self.series_service.get_method_by_id(10) == None method = test_util.add_method(self.session) db_method = self.series_service.get_method_by_id(method.id) assert method.description == db_method.description def test_delete_dvs(self): series = test_util.add_series(self.session) dvs = series.data_values subset = dvs[:5] self.series_service.delete_dvs([x.id for x in subset]) assert self.series_service.get_data_value_by_id(subset[0].id) == None series = self.series_service.get_series_by_id(series.id) # Reload assert len(series.data_values) == 5 def test_update_dvs(self): series = test_util.add_series(self.session) dvs = series.data_values subset = dvs[:5] for i in range(len(subset)): subset[i].data_value = 100 self.series_service.update_dvs(subset) series = self.series_service.get_series_by_id(series.id) assert series.data_values[0].data_value == 100 def test_create_new_series(self): site = test_util.add_site(self.session) variable = test_util.add_variable(self.session) method = test_util.add_method(self.session) source = test_util.add_source(self.session) qcl = test_util.add_qcl(self.session) dvs = [] for val in range(10): dv = DataValue() dv.data_value = val dv.site_id = site.id dv.variable_id = variable.id dv.method_id = method.id dv.source_id = source.id dv.quality_control_level_id = qcl.id dvs.append(dv) series = self.series_service.create_new_series(dvs, site.id, variable.id, method.id, source.id, qcl.id) assert series != None assert len(series.data_values) == 10 assert series.site_id == site.id assert series.variable_id == variable.id def test_update_series(self): series = test_util.add_series(self.session) series.site_code = "NEW" series.variable_code = "NEW" self.series_service.update_series(series) series = self.series_service.get_series_by_id(series.id) assert series.site_code == "NEW" assert series.variable_code == "NEW" def test_create_method(self): description = "This tests creating a method" link = "http://www.example.com" method = self.series_service.create_method(description, link) assert method.id != None assert method.description == description assert method.link == link def test_create_variable(self): unit = test_util.add_unit(self.session) variable = self.series_service.create_variable("Code", "Name", "Speciation", unit.id, "SampleMedium", "ValueType", True, # is_regular 5.0, # time_support unit.id, # time_unit_id "DataType", "GeneralCategory", -999.0) # no_data_value assert variable.id != None assert variable.code == "Code" assert variable.variable_unit_id == unit.id def test_create_qcl(self): qcl = self.series_service.create_qcl("Code", "Definition", "Explanation") assert qcl.id != None assert qcl.code == "Code" def test_delete_series(self): series = test_util.add_series(self.session) assert self.series_service.get_series_by_id(series.id) != None self.series_service.delete_series(series) assert self.series_service.get_series_by_id(series.id) == None def test_qcl_exists(self): qcl = test_util.add_qcl(self.session) assert self.series_service.qcl_exists(qcl) == True qcl.code = "00000" qcl.definition = "A new definition" assert self.series_service.qcl_exists(qcl) == False def test_method_exists(self): method = test_util.add_method(self.session) assert self.series_service.method_exists(method) == True method.description = "A new description" assert self.series_service.method_exists(method) == False def test_variable_exists(self): variable = test_util.add_variable(self.session) assert self.series_service.variable_exists(variable) variable.code = "00000" variable.name = "A new name" assert not self.series_service.variable_exists(variable)
class MemoryDatabase(object): ### this code should be changed to work with the database abstract layer so that sql queries are not in the code # series_service is a SeriesService def __init__(self, taskserver=None): self.editLoaded = False self.df = None # Series_Service handles remote database self.series_service = None # Memory_service handles in memory database self.mem_service = SeriesService("sqlite:///:memory:") # TODO clean up closing of program # if taskserver is None: #numproc = cpu_count() #self.taskserver = TaskServerMP(numproc=numproc) #else: self.taskserver = taskserver def reset_edit(self): self.mem_service = SeriesService("sqlite:///:memory:") def set_series_service(self, service): self.series_service = service ############## # DB Queries ############## def getDataValuesDF(self): logging.debug("update in memory dataframe") ''' if self.taskserver: results = self.taskserver.getCompletedTasks() df=results['UpdateEditDF'] #else: # self.updateDF() ''' self.updateDF() # pick up thread here before it is needed logging.debug("done updating memory dataframe") return self.df def getDataValues(self): return self.mem_service.get_all_values() def getEditRowCount(self): return len(self.df) def getEditColumns(self): columns = [] tmp_columns = self.df.columns.tolist() tmp_columns.remove('DataValue') tmp_columns.remove('LocalDateTime') tmp_columns.remove('QualifierID') columns.append('DataValue') columns.append('LocalDateTime') columns.append('QualifierID') columns.extend(tmp_columns) return [(x, i) for (i, x) in enumerate(columns)] # return [(x, i) for (i, x) in enumerate(self.df.columns)] def getDataValuesforGraph(self, seriesID, noDataValue, startDate=None, endDate=None): return self.series_service.get_plot_values(seriesID, noDataValue, startDate, endDate) def getEditDataValuesforGraph(self): return self.mem_service.get_all_plot_values() def commit(self): self.mem_service._edit_session.commit() def rollback(self): self.mem_service._edit_session.rollback() # self.mem_service._session_factory.engine.connect().connection.rollback() #self.updateDF() def update(self, updates): ''' updates : list of dictionary that contains 2 items, id and value ''' stmt = (DataValue.__table__.update(). where(DataValue.local_date_time == bindparam('id')). values(DataValue=bindparam('value')) ) self.mem_service._edit_session.execute(stmt, updates) # self.updateDF() def updateValue(self, ids, operator, value): # query = DataValue.data_value+value if operator == '+': query = DataValue.data_value + value elif operator == '-': query = DataValue.data_value - value elif operator == '*': query = DataValue.data_value * value elif operator == '=': query = value #break into chunks to get around sqlites restriction. allowing user to send in only 999 arguments at once chunks=self.chunking(ids) for c in chunks: q=self.mem_service._edit_session.query(DataValue).filter(DataValue.local_date_time.in_(c)) q.update({DataValue.data_value: query}, False) #self.updateDF() def chunking(self, data): return [data[x:x+998] for x in xrange(0, len(data), 998)] #break into chunks to get around sqlite's restriction. allowing user to send in only 999 arguments at once def updateFlag(self, ids, value): chunks=self.chunking(ids) for c in chunks: self.mem_service._edit_session.query(DataValue).filter(DataValue.local_date_time.in_(c))\ .update({DataValue.qualifier_id: value}, False) def delete(self, ids): chunks=self.chunking(ids) for c in chunks: self.mem_service.delete_dvs(c) #self.updateDF() def addPoints(self, points): """ Takes in a list of points and loads each point into the database """ stmt = DataValue.__table__.insert() if not isinstance(points, list): points = [points] for point in points: vals = {"DataValue": point[0], "ValueAccuracy": point[1], "LocalDateTime": point[2], "UTCOffset": point[3], "DateTimeUTC": point[4], "OffsetValue": point[5], "OffsetTypeID": point[6], "CensorCode": point[7], "QualifierID": point[8], "SampleID": point[9], "SiteID": point[10], "VariableID": point[11], "MethodID": point[12], "SourceID": point[13], "QualityControlLevelID": point[14]} self.mem_service._edit_session.execute(stmt, vals) def stopEdit(self): self.editLoaded = False self.df = None def setConnection(self, service): self.mem_service = service # TODO multiprocess this function def updateDF(self): ''' if self.taskserver: # Give tasks to the taskserver to run parallelly logger.debug("Sending tasks to taskserver") self.taskserver.setTasks(("UpdateEditDF", self.mem_service)) self.taskserver.processTasks() else: ''' self.df = self.mem_service.get_all_values_df() def initEditValues(self, seriesID): """ :param df: dataframe :return: nothing """ if not self.editLoaded: logger.debug("Load series from db") self.df = self.series_service.get_values_by_series(seriesID) self.editLoaded = True ''' if self.taskserver: self.taskserver.setTasks([("InitEditValues", (self.mem_service._session_factory.engine, self.df))]) self.taskserver.processTasks() # results = self.taskserver.getCompletedTasks() # self.conn = results["InitEditValues"] else: ''' #TODO: Thread this call if len(self.df)>0: self.df.to_sql(name="DataValues", if_exists='replace', con=self.mem_service._session_factory.engine, index=False)#,flavor='sqlite', chunksize=10000) logger.debug("done loading database") else: logger.debug("no data in series") def changeSeriesIDs(self, var=None, qcl=None, method=None): """ :param var: :param qcl: :param method: :return: """ query = self.mem_service._edit_session.query(DataValue) if var is not None: logger.debug(var) query.update({DataValue.variable_id: var}) if method is not None: logger.debug(method) query.update({DataValue.method_id: method}) # check that the code is not zero # if qcl is not None and qcl.code != 0: if qcl is not None: logger.debug(qcl) query.update({DataValue.quality_control_level_id: qcl})
class TestSeriesService: def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) """ @pytest.fixture(scope="class", autouse=True) def build_db(self): """ #Builds an empty sqlite (in-memory) database for testing #:return: None """ # path to the ddl script for building the database ddlpath= abspath(join(dirname(__file__), 'data/empty.sql')) # create and empty sqlite database for testing db = dbconnection.createConnection('sqlite', ':memory:') # read the ddl script and remove the first (BEGIN TRANSACTION) and last (COMMIT) lines ddl = open(ddlpath, 'r').read() ddl = ddl.replace('BEGIN TRANSACTION;','') ddl = ddl.replace('COMMIT;','') # execute each statement to build the odm2 database for line in ddl.split(');')[:-1]: try: db.engine.execute(line + ');') except Exception as e: print e self.write = CreateODM2(db) self.engine= db.engine globals['write'] = self.write globals['engine'] = self.engine globals['db'] = db # return self.write, self.engine def setup(self): self.writer = globals['write'] self.engine = globals['engine'] self.db = globals['db'] """ def test_get_db_version(self): version = test_util.add_version(self.session) db_version = self.series_service.get_db_version() assert version.version_number == db_version def test_get_all_sites_empty(self): sites = self.series_service.get_used_sites() #assert len(sites) == 0 assert sites is None def test_create_qualifier(self): qual = Qualifier() qual.code = "ABC123" qual.description = "This is a test" self.series_service.create_qualifier_by_qual(qual) assert qual.id is not None def test_get_qualifier_by_code(self): assert self.series_service.get_all_qualifiers() == [] qual = self.series_service.create_qualifier("ABC123", "This is a test") db_qual = self.series_service.get_qualifier_by_code("ABC123") assert qual.id == db_qual.id def test_get_qualifiers(self): assert self.series_service.get_all_qualifiers() == [] qual = self.series_service.create_qualifier("ABC123", "This is a test") db_qual = self.series_service.get_all_qualifiers()[0] assert qual.id == db_qual.id def test_get_all_sites(self): assert self.series_service.get_used_sites() is None site = test_util.add_site(self.session) sites = self.series_service.get_used_sites() assert sites is None if isinstance(sites, list) and len(sites) > 0: assert site.code == sites[0].code series = test_util.add_series(self.session) site = series.site sites = self.series_service.get_used_sites() assert len(sites) == 1 if isinstance(sites, list) and len(sites) > 0: assert site.code == sites[0].code def test_get_site_by_id_fail(self): assert self.series_service.get_site_by_id(0) == None site = test_util.add_site(self.session) db_site = self.series_service.get_site_by_id(10) assert db_site == None def test_get_site_by_id(self): site = test_util.add_site(self.session) db_site = self.series_service.get_site_by_id(site.id) assert db_site != None assert site.code == db_site.code def test_get_all_variables(self): assert self.series_service.get_all_variables() == [] variable = test_util.add_variable(self.session) variables = self.series_service.get_all_variables() assert len(variables) == 1 assert variable.code == variables[0].code def test_get_variable_by_id(self): assert self.series_service.get_variable_by_id(10) == None variable = test_util.add_variable(self.session) db_var = self.series_service.get_variable_by_id(variable.id) assert db_var != None assert db_var.code == variable.code def test_get_variables_by_site_code(self): assert self.series_service.get_variables_by_site_code('ABC123') == [] series = test_util.add_series(self.session) variable = series.variable db_variables = self.series_service.get_variables_by_site_code( series.site_code) assert db_variables != None assert variable.code == db_variables[0].code def test_get_all_units(self): assert self.series_service.get_all_units() == [] unit = test_util.add_unit(self.session) units = self.series_service.get_all_units() assert len(units) == 1 assert unit.name == units[0].name def test_get_unit_by_name(self): assert self.series_service.get_unit_by_name("FAIL") == None unit = test_util.add_unit(self.session) db_unit = self.series_service.get_unit_by_name(unit.name) assert unit.id == db_unit.id def test_get_unit_by_id(self): assert self.series_service.get_unit_by_id(10) == None unit = test_util.add_unit(self.session) db_unit = self.series_service.get_unit_by_id(unit.id) assert unit.name == db_unit.name def test_get_all_series(self): assert self.series_service.get_all_series() == [] series = test_util.add_series(self.session) all_series = self.series_service.get_all_series() assert all_series != [] assert series.id == all_series[0].id def test_get_series_by_id(self): assert self.series_service.get_series_by_id(10) == None series = test_util.add_series(self.session) db_series = self.series_service.get_series_by_id(series.id) assert series.id == db_series.id def test_get_series_by_id_quint(self): assert self.series_service.get_series_by_id_quint(10, 10, 10, 10, 10) == None series = test_util.add_series(self.session) db_series = self.series_service.get_series_by_id_quint( series.site_id, series.variable_id, series.method_id, series.source_id, series.quality_control_level_id) assert series.id == db_series.id def test_series_exists(self): assert self.series_service.series_exists_quint(10, 10, 10, 10, 10) == False series = test_util.add_series(self.session) site_id = series.site_id var_id = series.variable_id method_id = series.method_id source_id = series.source_id qcl_id = series.quality_control_level_id assert self.series_service.series_exists_quint(site_id, var_id, method_id, source_id, qcl_id) == True ''' def test_save_series(self): series = Series() site = test_util.add_site(self.session) variable = test_util.add_variable(self.session) method = test_util.add_method(self.session) source = test_util.add_source(self.session) qcl = test_util.add_qcl(self.session) series.site_id = site.id series.variable_id = variable.id series.method_id = method.id series.source_id = source.id series.quality_control_level_id = qcl.id dvs = [] for val in range(10): dv = DataValue() dv.data_value = val dv.site_id = site.id dv.variable_id = variable.id dv.method_id = method.id dv.source_id = source.id dv.quality_control_level_id = qcl.id dvs.append(dv) print series.variable_code assert self.series_service.save_series(series) assert self.series_service.series_exists(site.id, variable.id, method.id, source.id, qcl.id) assert not self.series_service.save_series(series) ''' def test_get_data_value_by_id(self): assert self.series_service.get_data_value_by_id(10) == None data_values = test_util.add_series(self.session).data_values dv = data_values[0] db_dv = self.series_service.get_data_value_by_id(dv.id) assert dv.data_value == db_dv.data_value def test_get_qcl_by_id(self): assert self.series_service.get_qcl_by_id(10) == None qcl = test_util.add_qcl(self.session) db_qcl = self.series_service.get_qcl_by_id(qcl.id) assert qcl.code == db_qcl.code def test_get_all_qcls(self): assert self.series_service.get_all_qcls() == [] qcl = test_util.add_qcl(self.session) all_qcls = self.series_service.get_all_qcls() assert len(all_qcls) == 1 assert qcl.id == all_qcls[0].id def test_get_all_methods(self): assert self.series_service.get_all_methods() == [] method = test_util.add_method(self.session) all_methods = self.series_service.get_all_methods() assert len(all_methods) == 1 assert method.id == all_methods[0].id def test_get_method_by_id(self): assert self.series_service.get_method_by_id(10) == None method = test_util.add_method(self.session) db_method = self.series_service.get_method_by_id(method.id) assert method.description == db_method.description def test_delete_dvs(self): series = test_util.add_series(self.session) dvs = series.data_values subset = dvs[:5] self.series_service.delete_dvs([x.local_date_time for x in subset]) assert self.series_service.get_data_value_by_id(subset[0].id) == None series = self.series_service.get_series_by_id(series.id) # Reload assert len(series.data_values) == 5 def test_update_dvs(self): series = test_util.add_series(self.session) dvs = series.data_values subset = dvs[:5] for i in range(len(subset)): subset[i].data_value = 100 self.series_service.update_dvs(subset) series = self.series_service.get_series_by_id(series.id) assert series.data_values[0].data_value == 100 def test_create_new_series(self): site = test_util.add_site(self.session) variable = test_util.add_variable(self.session) method = test_util.add_method(self.session) source = test_util.add_source(self.session) qcl = test_util.add_qcl(self.session) dvs = [] for val in range(10): dv = DataValue() dv.data_value = val dv.site_id = site.id dv.variable_id = variable.id dv.method_id = method.id dv.source_id = source.id dv.quality_control_level_id = qcl.id dvs.append(dv) series = self.series_service.create_new_series(dvs, site.id, variable.id, method.id, source.id, qcl.id) assert series != None assert len(series.data_values) == 10 assert series.site_id == site.id assert series.variable_id == variable.id def test_update_series(self): series = test_util.add_series(self.session) series.site_code = "NEW" series.variable_code = "NEW" self.series_service.update_series(series) series = self.series_service.get_series_by_id(series.id) assert series.site_code == "NEW" assert series.variable_code == "NEW" def test_create_method(self): description = "This tests creating a method" link = "http://www.example.com" method = self.series_service.create_method(description, link) assert method.id != None assert method.description == description assert method.link == link def test_create_variable(self): unit = test_util.add_unit(self.session) variable = self.series_service.create_variable( "Code", "Name", "Speciation", unit.id, "SampleMedium", "ValueType", True, # is_regular 5.0, # time_support unit.id, # time_unit_id "DataType", "GeneralCategory", -999.0) # no_data_value assert variable.id != None assert variable.code == "Code" assert variable.variable_unit_id == unit.id def test_create_qcl(self): qcl = self.series_service.create_qcl("Code", "Definition", "Explanation") assert qcl.id != None assert qcl.code == "Code" def test_delete_series(self): series = test_util.add_series(self.session) assert self.series_service.get_series_by_id(series.id) != None self.series_service.delete_series(series) assert self.series_service.get_series_by_id(series.id) == None def test_delete_values(self): series = test_util.add_series(self.session) assert self.series_service.get_series_by_id(series.id) != None self.series_service.delete_values_by_series(series) val = self.series_service.get_series_by_id(series.id) print val assert val != None def test_qcl_exists(self): qcl = test_util.add_qcl(self.session) assert self.series_service.qcl_exists(qcl) == True qcl.code = "00000" qcl.definition = "A new definition" assert self.series_service.qcl_exists(qcl) == False def test_method_exists(self): method = test_util.add_method(self.session) assert self.series_service.method_exists(method) == True method.description = "A new description" assert self.series_service.method_exists(method) == False def test_variable_exists(self): variable = test_util.add_variable(self.session) assert self.series_service.variable_exists(variable) variable.code = "00000" variable.name = "A new name" assert not self.series_service.variable_exists(variable)
class TestSeriesService: def setup(self): self.connection_string = "sqlite:///:memory:" self.series_service = SeriesService(self.connection_string, debug=False) self.session = self.series_service._session_factory.get_session() engine = self.series_service._session_factory.engine test_util.build_db(engine) def test_get_db_version(self): version = test_util.add_version(self.session) db_version = self.series_service.get_db_version() assert version.version_number == db_version def test_get_all_sites_empty(self): sites = self.series_service.get_used_sites() #assert len(sites) == 0 assert sites is None def test_create_qualifier(self): qual = Qualifier() qual.code = "ABC123" qual.description = "This is a test" self.series_service.create_qualifier_by_qual(qual) assert qual.id is not None def test_get_qualifiers(self): assert self.series_service.get_all_qualifiers() == [] qual = self.series_service.create_qualifier("ABC123", "This is a test") db_qual = self.series_service.get_all_qualifiers()[0] assert qual.id == db_qual.id def test_get_all_sites(self): assert self.series_service.get_used_sites() is None site = test_util.add_site(self.session) sites = self.series_service.get_used_sites() assert sites is None if isinstance(sites, list) and len(sites) > 0: assert site.code == sites[0].code series = test_util.add_series(self.session) site = series.site sites = self.series_service.get_used_sites() assert len(sites) == 1 if isinstance(sites, list) and len(sites) > 0: assert site.code == sites[0].code def test_get_site_by_id_fail(self): assert self.series_service.get_site_by_id(0) == None site = test_util.add_site(self.session) db_site = self.series_service.get_site_by_id(10) assert db_site == None def test_get_site_by_id(self): site = test_util.add_site(self.session) db_site = self.series_service.get_site_by_id(site.id) assert db_site != None assert site.code == db_site.code def test_get_all_variables(self): assert self.series_service.get_all_variables() == [] variable = test_util.add_variable(self.session) variables = self.series_service.get_all_variables() assert len(variables) == 1 assert variable.code == variables[0].code def test_get_variable_by_id(self): assert self.series_service.get_variable_by_id(10) == None variable = test_util.add_variable(self.session) db_var = self.series_service.get_variable_by_id(variable.id) assert db_var != None assert db_var.code == variable.code def test_get_variables_by_site_code(self): assert self.series_service.get_variables_by_site_code('ABC123') == [] series = test_util.add_series(self.session) variable = series.variable db_variables = self.series_service.get_variables_by_site_code( series.site_code) assert db_variables != None assert variable.code == db_variables[0].code def test_get_all_units(self): assert self.series_service.get_all_units() == [] unit = test_util.add_unit(self.session) units = self.series_service.get_all_units() assert len(units) == 1 assert unit.name == units[0].name def test_get_unit_by_name(self): assert self.series_service.get_unit_by_name("FAIL") == None unit = test_util.add_unit(self.session) db_unit = self.series_service.get_unit_by_name(unit.name) assert unit.id == db_unit.id def test_get_unit_by_id(self): assert self.series_service.get_unit_by_id(10) == None unit = test_util.add_unit(self.session) db_unit = self.series_service.get_unit_by_id(unit.id) assert unit.name == db_unit.name def test_get_all_series(self): assert self.series_service.get_all_series() == [] series = test_util.add_series(self.session) all_series = self.series_service.get_all_series() assert all_series != [] assert series.id == all_series[0].id def test_get_series_by_id(self): assert self.series_service.get_series_by_id(10) == None series = test_util.add_series(self.session) db_series = self.series_service.get_series_by_id(series.id) assert series.id == db_series.id def test_get_series_by_id_quint(self): assert self.series_service.get_series_by_id_quint(10, 10, 10, 10, 10) == None series = test_util.add_series(self.session) db_series = self.series_service.get_series_by_id_quint( series.site_id, series.variable_id, series.method_id, series.source_id, series.quality_control_level_id) assert series.id == db_series.id def test_series_exists(self): assert self.series_service.series_exists_quint(10, 10, 10, 10, 10) == False series = test_util.add_series(self.session) site_id = series.site_id var_id = series.variable_id method_id = series.method_id source_id = series.source_id qcl_id = series.quality_control_level_id assert self.series_service.series_exists_quint(site_id, var_id, method_id, source_id, qcl_id) == True ''' def test_save_series(self): series = Series() site = test_util.add_site(self.session) variable = test_util.add_variable(self.session) method = test_util.add_method(self.session) source = test_util.add_source(self.session) qcl = test_util.add_qcl(self.session) series.site_id = site.id series.variable_id = variable.id series.method_id = method.id series.source_id = source.id series.quality_control_level_id = qcl.id dvs = [] for val in range(10): dv = DataValue() dv.data_value = val dv.site_id = site.id dv.variable_id = variable.id dv.method_id = method.id dv.source_id = source.id dv.quality_control_level_id = qcl.id dvs.append(dv) print series.variable_code assert self.series_service.save_series(series) assert self.series_service.series_exists(site.id, variable.id, method.id, source.id, qcl.id) assert not self.series_service.save_series(series) ''' def test_get_data_value_by_id(self): assert self.series_service.get_data_value_by_id(10) == None data_values = test_util.add_series(self.session).data_values dv = data_values[0] db_dv = self.series_service.get_data_value_by_id(dv.id) assert dv.data_value == db_dv.data_value def test_get_qcl_by_id(self): assert self.series_service.get_qcl_by_id(10) == None qcl = test_util.add_qcl(self.session) db_qcl = self.series_service.get_qcl_by_id(qcl.id) assert qcl.code == db_qcl.code def test_get_all_qcls(self): assert self.series_service.get_all_qcls() == [] qcl = test_util.add_qcl(self.session) all_qcls = self.series_service.get_all_qcls() assert len(all_qcls) == 1 assert qcl.id == all_qcls[0].id def test_get_all_methods(self): assert self.series_service.get_all_methods() == [] method = test_util.add_method(self.session) all_methods = self.series_service.get_all_methods() assert len(all_methods) == 1 assert method.id == all_methods[0].id def test_get_method_by_id(self): assert self.series_service.get_method_by_id(10) == None method = test_util.add_method(self.session) db_method = self.series_service.get_method_by_id(method.id) assert method.description == db_method.description def test_delete_dvs(self): series = test_util.add_series(self.session) dvs = series.data_values subset = dvs[:5] self.series_service.delete_dvs([x.id for x in subset]) assert self.series_service.get_data_value_by_id(subset[0].id) == None series = self.series_service.get_series_by_id(series.id) # Reload assert len(series.data_values) == 5 def test_update_dvs(self): series = test_util.add_series(self.session) dvs = series.data_values subset = dvs[:5] for i in range(len(subset)): subset[i].data_value = 100 self.series_service.update_dvs(subset) series = self.series_service.get_series_by_id(series.id) assert series.data_values[0].data_value == 100 def test_create_new_series(self): site = test_util.add_site(self.session) variable = test_util.add_variable(self.session) method = test_util.add_method(self.session) source = test_util.add_source(self.session) qcl = test_util.add_qcl(self.session) dvs = [] for val in range(10): dv = DataValue() dv.data_value = val dv.site_id = site.id dv.variable_id = variable.id dv.method_id = method.id dv.source_id = source.id dv.quality_control_level_id = qcl.id dvs.append(dv) series = self.series_service.create_new_series(dvs, site.id, variable.id, method.id, source.id, qcl.id) assert series != None assert len(series.data_values) == 10 assert series.site_id == site.id assert series.variable_id == variable.id def test_update_series(self): series = test_util.add_series(self.session) series.site_code = "NEW" series.variable_code = "NEW" self.series_service.update_series(series) series = self.series_service.get_series_by_id(series.id) assert series.site_code == "NEW" assert series.variable_code == "NEW" def test_create_method(self): description = "This tests creating a method" link = "http://www.example.com" method = self.series_service.create_method(description, link) assert method.id != None assert method.description == description assert method.link == link def test_create_variable(self): unit = test_util.add_unit(self.session) variable = self.series_service.create_variable( "Code", "Name", "Speciation", unit.id, "SampleMedium", "ValueType", True, # is_regular 5.0, # time_support unit.id, # time_unit_id "DataType", "GeneralCategory", -999.0) # no_data_value assert variable.id != None assert variable.code == "Code" assert variable.variable_unit_id == unit.id def test_create_qcl(self): qcl = self.series_service.create_qcl("Code", "Definition", "Explanation") assert qcl.id != None assert qcl.code == "Code" def test_delete_series(self): series = test_util.add_series(self.session) assert self.series_service.get_series_by_id(series.id) != None self.series_service.delete_series(series) assert self.series_service.get_series_by_id(series.id) == None def test_qcl_exists(self): qcl = test_util.add_qcl(self.session) assert self.series_service.qcl_exists(qcl) == True qcl.code = "00000" qcl.definition = "A new definition" assert self.series_service.qcl_exists(qcl) == False def test_method_exists(self): method = test_util.add_method(self.session) assert self.series_service.method_exists(method) == True method.description = "A new description" assert self.series_service.method_exists(method) == False def test_variable_exists(self): variable = test_util.add_variable(self.session) assert self.series_service.variable_exists(variable) variable.code = "00000" variable.name = "A new name" assert not self.series_service.variable_exists(variable)