class TestSeriesService:
def setup(self):
self.connection_string = "sqlite:///:memory:"
self.series_service = SeriesService(self.connection_string, debug=False)
self.session = self.series_service._session_factory.get_session()
engine = self.series_service._session_factory.engine
test_util.build_db(engine)
"""
@pytest.fixture(scope="class", autouse=True)
def build_db(self):
"""
#Builds an empty sqlite (in-memory) database for testing
#:return: None
"""
# path to the ddl script for building the database
ddlpath= abspath(join(dirname(__file__), 'data/empty.sql'))
# create and empty sqlite database for testing
db = dbconnection.createConnection('sqlite', ':memory:')
# read the ddl script and remove the first (BEGIN TRANSACTION) and last (COMMIT) lines
ddl = open(ddlpath, 'r').read()
ddl = ddl.replace('BEGIN TRANSACTION;','')
ddl = ddl.replace('COMMIT;','')
# execute each statement to build the odm2 database
for line in ddl.split(');')[:-1]:
try:
db.engine.execute(line + ');')
except Exception as e:
print e
self.write = CreateODM2(db)
self.engine= db.engine
globals['write'] = self.write
globals['engine'] = self.engine
globals['db'] = db
# return self.write, self.engine
def setup(self):
self.writer = globals['write']
self.engine = globals['engine']
self.db = globals['db']
"""
def test_get_db_version(self):
version = test_util.add_version(self.session)
db_version = self.series_service.get_db_version()
assert version.version_number == db_version
def test_get_all_sites_empty(self):
sites = self.series_service.get_used_sites()
#assert len(sites) == 0
assert sites is None
def test_create_qualifier(self):
qual = Qualifier()
qual.code = "ABC123"
qual.description = "This is a test"
self.series_service.create_qualifier_by_qual(qual)
assert qual.id is not None
def test_get_qualifier_by_code(self):
assert self.series_service.get_all_qualifiers() == []
qual= self.series_service.create_qualifier("ABC123","This is a test")
db_qual = self.series_service.get_qualifier_by_code("ABC123")
assert qual.id == db_qual.id
def test_get_qualifiers(self):
assert self.series_service.get_all_qualifiers() == []
qual= self.series_service.create_qualifier("ABC123","This is a test")
db_qual = self.series_service.get_all_qualifiers()[0]
assert qual.id == db_qual.id
def test_get_all_sites(self):
assert self.series_service.get_used_sites() is None
site = test_util.add_site(self.session)
sites = self.series_service.get_used_sites()
assert sites is None
if isinstance(sites, list) and len(sites) > 0:
assert site.code == sites[0].code
series = test_util.add_series(self.session)
site = series.site
sites = self.series_service.get_used_sites()
assert len(sites) == 1
if isinstance(sites, list) and len(sites) > 0:
assert site.code == sites[0].code
def test_get_site_by_id_fail(self):
assert self.series_service.get_site_by_id(0) == None
site = test_util.add_site(self.session)
db_site = self.series_service.get_site_by_id(10)
assert db_site == None
def test_get_site_by_id(self):
site = test_util.add_site(self.session)
db_site = self.series_service.get_site_by_id(site.id)
assert db_site != None
assert site.code == db_site.code
def test_get_all_variables(self):
assert self.series_service.get_all_variables() == []
variable = test_util.add_variable(self.session)
variables = self.series_service.get_all_variables()
assert len(variables) == 1
assert variable.code == variables[0].code
def test_get_variable_by_id(self):
assert self.series_service.get_variable_by_id(10) == None
variable = test_util.add_variable(self.session)
db_var = self.series_service.get_variable_by_id(variable.id)
assert db_var != None
assert db_var.code == variable.code
def test_get_variables_by_site_code(self):
assert self.series_service.get_variables_by_site_code('ABC123') == []
series = test_util.add_series(self.session)
variable = series.variable
db_variables = self.series_service.get_variables_by_site_code(series.site_code)
assert db_variables != None
assert variable.code == db_variables[0].code
def test_get_all_units(self):
assert self.series_service.get_all_units() == []
unit = test_util.add_unit(self.session)
units = self.series_service.get_all_units()
assert len(units) == 1
assert unit.name == units[0].name
def test_get_unit_by_name(self):
assert self.series_service.get_unit_by_name("FAIL") == None
unit = test_util.add_unit(self.session)
db_unit = self.series_service.get_unit_by_name(unit.name)
assert unit.id == db_unit.id
def test_get_unit_by_id(self):
assert self.series_service.get_unit_by_id(10) == None
unit = test_util.add_unit(self.session)
db_unit = self.series_service.get_unit_by_id(unit.id)
assert unit.name == db_unit.name
def test_get_all_series(self):
assert self.series_service.get_all_series() == []
series = test_util.add_series(self.session)
all_series = self.series_service.get_all_series()
assert all_series != []
assert series.id == all_series[0].id
def test_get_series_by_id(self):
assert self.series_service.get_series_by_id(10) == None
series = test_util.add_series(self.session)
db_series = self.series_service.get_series_by_id(series.id)
assert series.id == db_series.id
def test_get_series_by_id_quint(self):
assert self.series_service.get_series_by_id_quint(10, 10, 10, 10, 10) == None
series = test_util.add_series(self.session)
db_series = self.series_service.get_series_by_id_quint(series.site_id, series.variable_id, series.method_id,
series.source_id, series.quality_control_level_id)
assert series.id == db_series.id
def test_series_exists(self):
assert self.series_service.series_exists_quint(10, 10, 10, 10, 10) == False
series = test_util.add_series(self.session)
site_id = series.site_id
var_id = series.variable_id
method_id = series.method_id
source_id = series.source_id
qcl_id = series.quality_control_level_id
assert self.series_service.series_exists_quint(site_id, var_id, method_id, source_id, qcl_id) == True
'''
def test_save_series(self):
series = Series()
site = test_util.add_site(self.session)
variable = test_util.add_variable(self.session)
method = test_util.add_method(self.session)
source = test_util.add_source(self.session)
qcl = test_util.add_qcl(self.session)
series.site_id = site.id
series.variable_id = variable.id
series.method_id = method.id
series.source_id = source.id
series.quality_control_level_id = qcl.id
dvs = []
for val in range(10):
dv = DataValue()
dv.data_value = val
dv.site_id = site.id
dv.variable_id = variable.id
dv.method_id = method.id
dv.source_id = source.id
dv.quality_control_level_id = qcl.id
dvs.append(dv)
print series.variable_code
assert self.series_service.save_series(series)
assert self.series_service.series_exists(site.id, variable.id, method.id, source.id, qcl.id)
assert not self.series_service.save_series(series)
'''
def test_get_data_value_by_id(self):
assert self.series_service.get_data_value_by_id(10) == None
data_values = test_util.add_series(self.session).data_values
dv = data_values[0]
db_dv = self.series_service.get_data_value_by_id(dv.id)
assert dv.data_value == db_dv.data_value
def test_get_qcl_by_id(self):
assert self.series_service.get_qcl_by_id(10) == None
qcl = test_util.add_qcl(self.session)
db_qcl = self.series_service.get_qcl_by_id(qcl.id)
assert qcl.code == db_qcl.code
def test_get_all_qcls(self):
assert self.series_service.get_all_qcls() == []
qcl = test_util.add_qcl(self.session)
all_qcls = self.series_service.get_all_qcls()
assert len(all_qcls) == 1
assert qcl.id == all_qcls[0].id
def test_get_all_methods(self):
assert self.series_service.get_all_methods() == []
method = test_util.add_method(self.session)
all_methods = self.series_service.get_all_methods()
assert len(all_methods) == 1
assert method.id == all_methods[0].id
def test_get_method_by_id(self):
assert self.series_service.get_method_by_id(10) == None
method = test_util.add_method(self.session)
db_method = self.series_service.get_method_by_id(method.id)
assert method.description == db_method.description
def test_delete_dvs(self):
series = test_util.add_series(self.session)
dvs = series.data_values
subset = dvs[:5]
self.series_service.delete_dvs([x.local_date_time for x in subset])
assert self.series_service.get_data_value_by_id(subset[0].id) == None
series = self.series_service.get_series_by_id(series.id) # Reload
assert len(series.data_values) == 5
def test_update_dvs(self):
series = test_util.add_series(self.session)
dvs = series.data_values
subset = dvs[:5]
for i in range(len(subset)):
subset[i].data_value = 100
self.series_service.update_dvs(subset)
series = self.series_service.get_series_by_id(series.id)
assert series.data_values[0].data_value == 100
def test_create_new_series(self):
site = test_util.add_site(self.session)
variable = test_util.add_variable(self.session)
method = test_util.add_method(self.session)
source = test_util.add_source(self.session)
qcl = test_util.add_qcl(self.session)
dvs = []
for val in range(10):
dv = DataValue()
dv.data_value = val
dv.site_id = site.id
dv.variable_id = variable.id
dv.method_id = method.id
dv.source_id = source.id
dv.quality_control_level_id = qcl.id
dvs.append(dv)
series = self.series_service.create_new_series(dvs, site.id, variable.id, method.id, source.id, qcl.id)
assert series != None
assert len(series.data_values) == 10
assert series.site_id == site.id
assert series.variable_id == variable.id
def test_update_series(self):
series = test_util.add_series(self.session)
series.site_code = "NEW"
series.variable_code = "NEW"
self.series_service.update_series(series)
series = self.series_service.get_series_by_id(series.id)
assert series.site_code == "NEW"
assert series.variable_code == "NEW"
def test_create_method(self):
description = "This tests creating a method"
link = "http://www.example.com"
method = self.series_service.create_method(description, link)
assert method.id != None
assert method.description == description
assert method.link == link
def test_create_variable(self):
unit = test_util.add_unit(self.session)
variable = self.series_service.create_variable("Code", "Name", "Speciation", unit.id, "SampleMedium",
"ValueType", True, # is_regular
5.0, # time_support
unit.id, # time_unit_id
"DataType", "GeneralCategory", -999.0) # no_data_value
assert variable.id != None
assert variable.code == "Code"
assert variable.variable_unit_id == unit.id
def test_create_qcl(self):
qcl = self.series_service.create_qcl("Code", "Definition", "Explanation")
assert qcl.id != None
assert qcl.code == "Code"
def test_delete_series(self):
series = test_util.add_series(self.session)
assert self.series_service.get_series_by_id(series.id) != None
self.series_service.delete_series(series)
assert self.series_service.get_series_by_id(series.id) == None
def test_delete_values(self):
series = test_util.add_series(self.session)
assert self.series_service.get_series_by_id(series.id) != None
self.series_service.delete_values_by_series(series)
val = self.series_service.get_series_by_id(series.id)
print val
assert val != None
def test_qcl_exists(self):
qcl = test_util.add_qcl(self.session)
assert self.series_service.qcl_exists(qcl) == True
qcl.code = "00000"
qcl.definition = "A new definition"
assert self.series_service.qcl_exists(qcl) == False
def test_method_exists(self):
method = test_util.add_method(self.session)
assert self.series_service.method_exists(method) == True
method.description = "A new description"
assert self.series_service.method_exists(method) == False
def test_variable_exists(self):
variable = test_util.add_variable(self.session)
assert self.series_service.variable_exists(variable)
variable.code = "00000"
variable.name = "A new name"
assert not self.series_service.variable_exists(variable)
class TestSeriesService:
def setup(self):
self.connection_string = "sqlite:///:memory:"
self.series_service = SeriesService(self.connection_string, debug=False)
self.session = self.series_service._session_factory.get_session()
engine = self.series_service._session_factory.engine
test_util.build_db(engine)
def test_get_db_version(self):
version = test_util.add_version(self.session)
db_version = self.series_service.get_db_version()
assert version.version_number == db_version
def test_get_all_sites_empty(self):
sites = self.series_service.get_used_sites()
#assert len(sites) == 0
assert sites is None
def test_create_qualifier(self):
qual = Qualifier()
qual.code = "ABC123"
qual.description = "This is a test"
self.series_service.create_qualifier_by_qual(qual)
assert qual.id is not None
def test_get_qualifiers(self):
assert self.series_service.get_all_qualifiers() == []
qual= self.series_service.create_qualifier("ABC123","This is a test")
db_qual = self.series_service.get_all_qualifiers()[0]
assert qual.id == db_qual.id
def test_get_all_sites(self):
assert self.series_service.get_used_sites() is None
site = test_util.add_site(self.session)
sites = self.series_service.get_used_sites()
assert sites is None
if isinstance(sites, list) and len(sites) > 0:
assert site.code == sites[0].code
series = test_util.add_series(self.session)
site = series.site
sites = self.series_service.get_used_sites()
assert len(sites) == 1
if isinstance(sites, list) and len(sites) > 0:
assert site.code == sites[0].code
def test_get_site_by_id_fail(self):
assert self.series_service.get_site_by_id(0) == None
site = test_util.add_site(self.session)
db_site = self.series_service.get_site_by_id(10)
assert db_site == None
def test_get_site_by_id(self):
site = test_util.add_site(self.session)
db_site = self.series_service.get_site_by_id(site.id)
assert db_site != None
assert site.code == db_site.code
def test_get_all_variables(self):
assert self.series_service.get_all_variables() == []
variable = test_util.add_variable(self.session)
variables = self.series_service.get_all_variables()
assert len(variables) == 1
assert variable.code == variables[0].code
def test_get_variable_by_id(self):
assert self.series_service.get_variable_by_id(10) == None
variable = test_util.add_variable(self.session)
db_var = self.series_service.get_variable_by_id(variable.id)
assert db_var != None
assert db_var.code == variable.code
def test_get_variables_by_site_code(self):
assert self.series_service.get_variables_by_site_code('ABC123') == []
series = test_util.add_series(self.session)
variable = series.variable
db_variables = self.series_service.get_variables_by_site_code(series.site_code)
assert db_variables != None
assert variable.code == db_variables[0].code
def test_get_all_units(self):
assert self.series_service.get_all_units() == []
unit = test_util.add_unit(self.session)
units = self.series_service.get_all_units()
assert len(units) == 1
assert unit.name == units[0].name
def test_get_unit_by_name(self):
assert self.series_service.get_unit_by_name("FAIL") == None
unit = test_util.add_unit(self.session)
db_unit = self.series_service.get_unit_by_name(unit.name)
assert unit.id == db_unit.id
def test_get_unit_by_id(self):
assert self.series_service.get_unit_by_id(10) == None
unit = test_util.add_unit(self.session)
db_unit = self.series_service.get_unit_by_id(unit.id)
assert unit.name == db_unit.name
def test_get_all_series(self):
assert self.series_service.get_all_series() == []
series = test_util.add_series(self.session)
all_series = self.series_service.get_all_series()
assert all_series != []
assert series.id == all_series[0].id
def test_get_series_by_id(self):
assert self.series_service.get_series_by_id(10) == None
series = test_util.add_series(self.session)
db_series = self.series_service.get_series_by_id(series.id)
assert series.id == db_series.id
def test_get_series_by_id_quint(self):
assert self.series_service.get_series_by_id_quint(10, 10, 10, 10, 10) == None
series = test_util.add_series(self.session)
db_series = self.series_service.get_series_by_id_quint(series.site_id, series.variable_id, series.method_id,
series.source_id, series.quality_control_level_id)
assert series.id == db_series.id
def test_series_exists(self):
assert self.series_service.series_exists_quint(10, 10, 10, 10, 10) == False
series = test_util.add_series(self.session)
site_id = series.site_id
var_id = series.variable_id
method_id = series.method_id
source_id = series.source_id
qcl_id = series.quality_control_level_id
assert self.series_service.series_exists_quint(site_id, var_id, method_id, source_id, qcl_id) == True
## TODO Unittest save_series, save_as, save_as_existing
'''
def test_save_series(self):
series = Series()
site = test_util.add_site(self.session)
variable = test_util.add_variable(self.session)
method = test_util.add_method(self.session)
source = test_util.add_source(self.session)
qcl = test_util.add_qcl(self.session)
series.site_id = site.id
series.variable_id = variable.id
series.method_id = method.id
series.source_id = source.id
series.quality_control_level_id = qcl.id
dvs = []
for val in range(10):
dv = DataValue()
dv.data_value = val
dv.site_id = site.id
dv.variable_id = variable.id
dv.method_id = method.id
dv.source_id = source.id
dv.quality_control_level_id = qcl.id
dvs.append(dv)
print series.variable_code
assert self.series_service.save_series(series)
assert self.series_service.series_exists(site.id, variable.id, method.id, source.id, qcl.id)
assert not self.series_service.save_series(series)
'''
def test_get_data_value_by_id(self):
assert self.series_service.get_data_value_by_id(10) == None
data_values = test_util.add_series(self.session).data_values
dv = data_values[0]
db_dv = self.series_service.get_data_value_by_id(dv.id)
assert dv.data_value == db_dv.data_value
def test_get_qcl_by_id(self):
assert self.series_service.get_qcl_by_id(10) == None
qcl = test_util.add_qcl(self.session)
db_qcl = self.series_service.get_qcl_by_id(qcl.id)
assert qcl.code == db_qcl.code
def test_get_all_qcls(self):
assert self.series_service.get_all_qcls() == []
qcl = test_util.add_qcl(self.session)
all_qcls = self.series_service.get_all_qcls()
assert len(all_qcls) == 1
assert qcl.id == all_qcls[0].id
def test_get_all_methods(self):
assert self.series_service.get_all_methods() == []
method = test_util.add_method(self.session)
all_methods = self.series_service.get_all_methods()
assert len(all_methods) == 1
assert method.id == all_methods[0].id
def test_get_method_by_id(self):
assert self.series_service.get_method_by_id(10) == None
method = test_util.add_method(self.session)
db_method = self.series_service.get_method_by_id(method.id)
assert method.description == db_method.description
def test_delete_dvs(self):
series = test_util.add_series(self.session)
dvs = series.data_values
subset = dvs[:5]
self.series_service.delete_dvs([x.id for x in subset])
assert self.series_service.get_data_value_by_id(subset[0].id) == None
series = self.series_service.get_series_by_id(series.id) # Reload
assert len(series.data_values) == 5
def test_update_dvs(self):
series = test_util.add_series(self.session)
dvs = series.data_values
subset = dvs[:5]
for i in range(len(subset)):
subset[i].data_value = 100
self.series_service.update_dvs(subset)
series = self.series_service.get_series_by_id(series.id)
assert series.data_values[0].data_value == 100
def test_create_new_series(self):
site = test_util.add_site(self.session)
variable = test_util.add_variable(self.session)
method = test_util.add_method(self.session)
source = test_util.add_source(self.session)
qcl = test_util.add_qcl(self.session)
dvs = []
for val in range(10):
dv = DataValue()
dv.data_value = val
dv.site_id = site.id
dv.variable_id = variable.id
dv.method_id = method.id
dv.source_id = source.id
dv.quality_control_level_id = qcl.id
dvs.append(dv)
series = self.series_service.create_new_series(dvs, site.id, variable.id, method.id, source.id, qcl.id)
assert series != None
assert len(series.data_values) == 10
assert series.site_id == site.id
assert series.variable_id == variable.id
def test_update_series(self):
series = test_util.add_series(self.session)
series.site_code = "NEW"
series.variable_code = "NEW"
self.series_service.update_series(series)
series = self.series_service.get_series_by_id(series.id)
assert series.site_code == "NEW"
assert series.variable_code == "NEW"
def test_create_method(self):
description = "This tests creating a method"
link = "http://www.example.com"
method = self.series_service.create_method(description, link)
assert method.id != None
assert method.description == description
assert method.link == link
def test_create_variable(self):
unit = test_util.add_unit(self.session)
variable = self.series_service.create_variable("Code", "Name", "Speciation", unit.id, "SampleMedium",
"ValueType", True, # is_regular
5.0, # time_support
unit.id, # time_unit_id
"DataType", "GeneralCategory", -999.0) # no_data_value
assert variable.id != None
assert variable.code == "Code"
assert variable.variable_unit_id == unit.id
def test_create_qcl(self):
qcl = self.series_service.create_qcl("Code", "Definition", "Explanation")
assert qcl.id != None
assert qcl.code == "Code"
def test_delete_series(self):
series = test_util.add_series(self.session)
assert self.series_service.get_series_by_id(series.id) != None
self.series_service.delete_series(series)
assert self.series_service.get_series_by_id(series.id) == None
def test_qcl_exists(self):
qcl = test_util.add_qcl(self.session)
assert self.series_service.qcl_exists(qcl) == True
qcl.code = "00000"
qcl.definition = "A new definition"
assert self.series_service.qcl_exists(qcl) == False
def test_method_exists(self):
method = test_util.add_method(self.session)
assert self.series_service.method_exists(method) == True
method.description = "A new description"
assert self.series_service.method_exists(method) == False
def test_variable_exists(self):
variable = test_util.add_variable(self.session)
assert self.series_service.variable_exists(variable)
variable.code = "00000"
variable.name = "A new name"
assert not self.series_service.variable_exists(variable)
class MemoryDatabase(object):
### this code should be changed to work with the database abstract layer so that sql queries are not in the code
# series_service is a SeriesService
def __init__(self, taskserver=None):
self.editLoaded = False
self.df = None
# Series_Service handles remote database
self.series_service = None
# Memory_service handles in memory database
self.mem_service = SeriesService("sqlite:///:memory:")
# TODO clean up closing of program
# if taskserver is None:
#numproc = cpu_count()
#self.taskserver = TaskServerMP(numproc=numproc)
#else:
self.taskserver = taskserver
def reset_edit(self):
self.mem_service = SeriesService("sqlite:///:memory:")
def set_series_service(self, service):
self.series_service = service
##############
# DB Queries
##############
def getDataValuesDF(self):
logging.debug("update in memory dataframe")
'''
if self.taskserver:
results = self.taskserver.getCompletedTasks()
df=results['UpdateEditDF']
#else:
# self.updateDF()
'''
self.updateDF()
# pick up thread here before it is needed
logging.debug("done updating memory dataframe")
return self.df
def getDataValues(self):
return self.mem_service.get_all_values()
def getEditRowCount(self):
return len(self.df)
def getEditColumns(self):
columns = []
tmp_columns = self.df.columns.tolist()
tmp_columns.remove('DataValue')
tmp_columns.remove('LocalDateTime')
tmp_columns.remove('QualifierID')
columns.append('DataValue')
columns.append('LocalDateTime')
columns.append('QualifierID')
columns.extend(tmp_columns)
return [(x, i) for (i, x) in enumerate(columns)]
# return [(x, i) for (i, x) in enumerate(self.df.columns)]
def getDataValuesforGraph(self, seriesID, noDataValue, startDate=None, endDate=None):
return self.series_service.get_plot_values(seriesID, noDataValue, startDate, endDate)
def getEditDataValuesforGraph(self):
return self.mem_service.get_all_plot_values()
def commit(self):
self.mem_service._edit_session.commit()
def rollback(self):
self.mem_service._edit_session.rollback()
# self.mem_service._session_factory.engine.connect().connection.rollback()
#self.updateDF()
def update(self, updates):
'''
updates : list of dictionary that contains 2 items, id and value
'''
stmt = (DataValue.__table__.update().
where(DataValue.local_date_time == bindparam('id')).
values(DataValue=bindparam('value'))
)
self.mem_service._edit_session.execute(stmt, updates)
# self.updateDF()
def updateValue(self, ids, operator, value):
# query = DataValue.data_value+value
if operator == '+':
query = DataValue.data_value + value
elif operator == '-':
query = DataValue.data_value - value
elif operator == '*':
query = DataValue.data_value * value
elif operator == '=':
query = value
#break into chunks to get around sqlites restriction. allowing user to send in only 999 arguments at once
chunks=self.chunking(ids)
for c in chunks:
q=self.mem_service._edit_session.query(DataValue).filter(DataValue.local_date_time.in_(c))
q.update({DataValue.data_value: query}, False)
#self.updateDF()
def chunking(self, data):
return [data[x:x+998] for x in xrange(0, len(data), 998)]
#break into chunks to get around sqlite's restriction. allowing user to send in only 999 arguments at once
def updateFlag(self, ids, value):
chunks=self.chunking(ids)
for c in chunks:
self.mem_service._edit_session.query(DataValue).filter(DataValue.local_date_time.in_(c))\
.update({DataValue.qualifier_id: value}, False)
def delete(self, ids):
chunks=self.chunking(ids)
for c in chunks:
self.mem_service.delete_dvs(c)
#self.updateDF()
def addPoints(self, points):
"""
Takes in a list of points and loads each point into the database
"""
stmt = DataValue.__table__.insert()
if not isinstance(points, list):
points = [points]
for point in points:
vals = {"DataValue": point[0], "ValueAccuracy": point[1],
"LocalDateTime": point[2], "UTCOffset": point[3],
"DateTimeUTC": point[4], "OffsetValue": point[5],
"OffsetTypeID": point[6], "CensorCode": point[7],
"QualifierID": point[8], "SampleID": point[9],
"SiteID": point[10], "VariableID": point[11],
"MethodID": point[12], "SourceID": point[13],
"QualityControlLevelID": point[14]}
self.mem_service._edit_session.execute(stmt, vals)
def stopEdit(self):
self.editLoaded = False
self.df = None
def setConnection(self, service):
self.mem_service = service
# TODO multiprocess this function
def updateDF(self):
'''
if self.taskserver:
# Give tasks to the taskserver to run parallelly
logger.debug("Sending tasks to taskserver")
self.taskserver.setTasks(("UpdateEditDF", self.mem_service))
self.taskserver.processTasks()
else:
'''
self.df = self.mem_service.get_all_values_df()
def initEditValues(self, seriesID):
"""
:param df: dataframe
:return: nothing
"""
if not self.editLoaded:
logger.debug("Load series from db")
self.df = self.series_service.get_values_by_series(seriesID)
self.editLoaded = True
'''
if self.taskserver:
self.taskserver.setTasks([("InitEditValues", (self.mem_service._session_factory.engine, self.df))])
self.taskserver.processTasks()
# results = self.taskserver.getCompletedTasks()
# self.conn = results["InitEditValues"]
else:
''' #TODO: Thread this call
if len(self.df)>0:
self.df.to_sql(name="DataValues", if_exists='replace', con=self.mem_service._session_factory.engine,
index=False)#,flavor='sqlite', chunksize=10000)
logger.debug("done loading database")
else:
logger.debug("no data in series")
def changeSeriesIDs(self, var=None, qcl=None, method=None):
"""
:param var:
:param qcl:
:param method:
:return:
"""
query = self.mem_service._edit_session.query(DataValue)
if var is not None:
logger.debug(var)
query.update({DataValue.variable_id: var})
if method is not None:
logger.debug(method)
query.update({DataValue.method_id: method})
# check that the code is not zero
# if qcl is not None and qcl.code != 0:
if qcl is not None:
logger.debug(qcl)
query.update({DataValue.quality_control_level_id: qcl})
class TestSeriesService:
def setup(self):
self.connection_string = "sqlite:///:memory:"
self.series_service = SeriesService(self.connection_string,
debug=False)
self.session = self.series_service._session_factory.get_session()
engine = self.series_service._session_factory.engine
test_util.build_db(engine)
"""
@pytest.fixture(scope="class", autouse=True)
def build_db(self):
"""
#Builds an empty sqlite (in-memory) database for testing
#:return: None
"""
# path to the ddl script for building the database
ddlpath= abspath(join(dirname(__file__), 'data/empty.sql'))
# create and empty sqlite database for testing
db = dbconnection.createConnection('sqlite', ':memory:')
# read the ddl script and remove the first (BEGIN TRANSACTION) and last (COMMIT) lines
ddl = open(ddlpath, 'r').read()
ddl = ddl.replace('BEGIN TRANSACTION;','')
ddl = ddl.replace('COMMIT;','')
# execute each statement to build the odm2 database
for line in ddl.split(');')[:-1]:
try:
db.engine.execute(line + ');')
except Exception as e:
print e
self.write = CreateODM2(db)
self.engine= db.engine
globals['write'] = self.write
globals['engine'] = self.engine
globals['db'] = db
# return self.write, self.engine
def setup(self):
self.writer = globals['write']
self.engine = globals['engine']
self.db = globals['db']
"""
def test_get_db_version(self):
version = test_util.add_version(self.session)
db_version = self.series_service.get_db_version()
assert version.version_number == db_version
def test_get_all_sites_empty(self):
sites = self.series_service.get_used_sites()
#assert len(sites) == 0
assert sites is None
def test_create_qualifier(self):
qual = Qualifier()
qual.code = "ABC123"
qual.description = "This is a test"
self.series_service.create_qualifier_by_qual(qual)
assert qual.id is not None
def test_get_qualifier_by_code(self):
assert self.series_service.get_all_qualifiers() == []
qual = self.series_service.create_qualifier("ABC123", "This is a test")
db_qual = self.series_service.get_qualifier_by_code("ABC123")
assert qual.id == db_qual.id
def test_get_qualifiers(self):
assert self.series_service.get_all_qualifiers() == []
qual = self.series_service.create_qualifier("ABC123", "This is a test")
db_qual = self.series_service.get_all_qualifiers()[0]
assert qual.id == db_qual.id
def test_get_all_sites(self):
assert self.series_service.get_used_sites() is None
site = test_util.add_site(self.session)
sites = self.series_service.get_used_sites()
assert sites is None
if isinstance(sites, list) and len(sites) > 0:
assert site.code == sites[0].code
series = test_util.add_series(self.session)
site = series.site
sites = self.series_service.get_used_sites()
assert len(sites) == 1
if isinstance(sites, list) and len(sites) > 0:
assert site.code == sites[0].code
def test_get_site_by_id_fail(self):
assert self.series_service.get_site_by_id(0) == None
site = test_util.add_site(self.session)
db_site = self.series_service.get_site_by_id(10)
assert db_site == None
def test_get_site_by_id(self):
site = test_util.add_site(self.session)
db_site = self.series_service.get_site_by_id(site.id)
assert db_site != None
assert site.code == db_site.code
def test_get_all_variables(self):
assert self.series_service.get_all_variables() == []
variable = test_util.add_variable(self.session)
variables = self.series_service.get_all_variables()
assert len(variables) == 1
assert variable.code == variables[0].code
def test_get_variable_by_id(self):
assert self.series_service.get_variable_by_id(10) == None
variable = test_util.add_variable(self.session)
db_var = self.series_service.get_variable_by_id(variable.id)
assert db_var != None
assert db_var.code == variable.code
def test_get_variables_by_site_code(self):
assert self.series_service.get_variables_by_site_code('ABC123') == []
series = test_util.add_series(self.session)
variable = series.variable
db_variables = self.series_service.get_variables_by_site_code(
series.site_code)
assert db_variables != None
assert variable.code == db_variables[0].code
def test_get_all_units(self):
assert self.series_service.get_all_units() == []
unit = test_util.add_unit(self.session)
units = self.series_service.get_all_units()
assert len(units) == 1
assert unit.name == units[0].name
def test_get_unit_by_name(self):
assert self.series_service.get_unit_by_name("FAIL") == None
unit = test_util.add_unit(self.session)
db_unit = self.series_service.get_unit_by_name(unit.name)
assert unit.id == db_unit.id
def test_get_unit_by_id(self):
assert self.series_service.get_unit_by_id(10) == None
unit = test_util.add_unit(self.session)
db_unit = self.series_service.get_unit_by_id(unit.id)
assert unit.name == db_unit.name
def test_get_all_series(self):
assert self.series_service.get_all_series() == []
series = test_util.add_series(self.session)
all_series = self.series_service.get_all_series()
assert all_series != []
assert series.id == all_series[0].id
def test_get_series_by_id(self):
assert self.series_service.get_series_by_id(10) == None
series = test_util.add_series(self.session)
db_series = self.series_service.get_series_by_id(series.id)
assert series.id == db_series.id
def test_get_series_by_id_quint(self):
assert self.series_service.get_series_by_id_quint(10, 10, 10, 10,
10) == None
series = test_util.add_series(self.session)
db_series = self.series_service.get_series_by_id_quint(
series.site_id, series.variable_id, series.method_id,
series.source_id, series.quality_control_level_id)
assert series.id == db_series.id
def test_series_exists(self):
assert self.series_service.series_exists_quint(10, 10, 10, 10,
10) == False
series = test_util.add_series(self.session)
site_id = series.site_id
var_id = series.variable_id
method_id = series.method_id
source_id = series.source_id
qcl_id = series.quality_control_level_id
assert self.series_service.series_exists_quint(site_id, var_id,
method_id, source_id,
qcl_id) == True
'''
def test_save_series(self):
series = Series()
site = test_util.add_site(self.session)
variable = test_util.add_variable(self.session)
method = test_util.add_method(self.session)
source = test_util.add_source(self.session)
qcl = test_util.add_qcl(self.session)
series.site_id = site.id
series.variable_id = variable.id
series.method_id = method.id
series.source_id = source.id
series.quality_control_level_id = qcl.id
dvs = []
for val in range(10):
dv = DataValue()
dv.data_value = val
dv.site_id = site.id
dv.variable_id = variable.id
dv.method_id = method.id
dv.source_id = source.id
dv.quality_control_level_id = qcl.id
dvs.append(dv)
print series.variable_code
assert self.series_service.save_series(series)
assert self.series_service.series_exists(site.id, variable.id, method.id, source.id, qcl.id)
assert not self.series_service.save_series(series)
'''
def test_get_data_value_by_id(self):
assert self.series_service.get_data_value_by_id(10) == None
data_values = test_util.add_series(self.session).data_values
dv = data_values[0]
db_dv = self.series_service.get_data_value_by_id(dv.id)
assert dv.data_value == db_dv.data_value
def test_get_qcl_by_id(self):
assert self.series_service.get_qcl_by_id(10) == None
qcl = test_util.add_qcl(self.session)
db_qcl = self.series_service.get_qcl_by_id(qcl.id)
assert qcl.code == db_qcl.code
def test_get_all_qcls(self):
assert self.series_service.get_all_qcls() == []
qcl = test_util.add_qcl(self.session)
all_qcls = self.series_service.get_all_qcls()
assert len(all_qcls) == 1
assert qcl.id == all_qcls[0].id
def test_get_all_methods(self):
assert self.series_service.get_all_methods() == []
method = test_util.add_method(self.session)
all_methods = self.series_service.get_all_methods()
assert len(all_methods) == 1
assert method.id == all_methods[0].id
def test_get_method_by_id(self):
assert self.series_service.get_method_by_id(10) == None
method = test_util.add_method(self.session)
db_method = self.series_service.get_method_by_id(method.id)
assert method.description == db_method.description
def test_delete_dvs(self):
series = test_util.add_series(self.session)
dvs = series.data_values
subset = dvs[:5]
self.series_service.delete_dvs([x.local_date_time for x in subset])
assert self.series_service.get_data_value_by_id(subset[0].id) == None
series = self.series_service.get_series_by_id(series.id) # Reload
assert len(series.data_values) == 5
def test_update_dvs(self):
series = test_util.add_series(self.session)
dvs = series.data_values
subset = dvs[:5]
for i in range(len(subset)):
subset[i].data_value = 100
self.series_service.update_dvs(subset)
series = self.series_service.get_series_by_id(series.id)
assert series.data_values[0].data_value == 100
def test_create_new_series(self):
site = test_util.add_site(self.session)
variable = test_util.add_variable(self.session)
method = test_util.add_method(self.session)
source = test_util.add_source(self.session)
qcl = test_util.add_qcl(self.session)
dvs = []
for val in range(10):
dv = DataValue()
dv.data_value = val
dv.site_id = site.id
dv.variable_id = variable.id
dv.method_id = method.id
dv.source_id = source.id
dv.quality_control_level_id = qcl.id
dvs.append(dv)
series = self.series_service.create_new_series(dvs, site.id,
variable.id, method.id,
source.id, qcl.id)
assert series != None
assert len(series.data_values) == 10
assert series.site_id == site.id
assert series.variable_id == variable.id
def test_update_series(self):
series = test_util.add_series(self.session)
series.site_code = "NEW"
series.variable_code = "NEW"
self.series_service.update_series(series)
series = self.series_service.get_series_by_id(series.id)
assert series.site_code == "NEW"
assert series.variable_code == "NEW"
def test_create_method(self):
description = "This tests creating a method"
link = "http://www.example.com"
method = self.series_service.create_method(description, link)
assert method.id != None
assert method.description == description
assert method.link == link
def test_create_variable(self):
unit = test_util.add_unit(self.session)
variable = self.series_service.create_variable(
"Code",
"Name",
"Speciation",
unit.id,
"SampleMedium",
"ValueType",
True, # is_regular
5.0, # time_support
unit.id, # time_unit_id
"DataType",
"GeneralCategory",
-999.0) # no_data_value
assert variable.id != None
assert variable.code == "Code"
assert variable.variable_unit_id == unit.id
def test_create_qcl(self):
qcl = self.series_service.create_qcl("Code", "Definition",
"Explanation")
assert qcl.id != None
assert qcl.code == "Code"
def test_delete_series(self):
series = test_util.add_series(self.session)
assert self.series_service.get_series_by_id(series.id) != None
self.series_service.delete_series(series)
assert self.series_service.get_series_by_id(series.id) == None
def test_delete_values(self):
series = test_util.add_series(self.session)
assert self.series_service.get_series_by_id(series.id) != None
self.series_service.delete_values_by_series(series)
val = self.series_service.get_series_by_id(series.id)
print val
assert val != None
def test_qcl_exists(self):
qcl = test_util.add_qcl(self.session)
assert self.series_service.qcl_exists(qcl) == True
qcl.code = "00000"
qcl.definition = "A new definition"
assert self.series_service.qcl_exists(qcl) == False
def test_method_exists(self):
method = test_util.add_method(self.session)
assert self.series_service.method_exists(method) == True
method.description = "A new description"
assert self.series_service.method_exists(method) == False
def test_variable_exists(self):
variable = test_util.add_variable(self.session)
assert self.series_service.variable_exists(variable)
variable.code = "00000"
variable.name = "A new name"
assert not self.series_service.variable_exists(variable)
class TestSeriesService:
def setup(self):
self.connection_string = "sqlite:///:memory:"
self.series_service = SeriesService(self.connection_string,
debug=False)
self.session = self.series_service._session_factory.get_session()
engine = self.series_service._session_factory.engine
test_util.build_db(engine)
def test_get_db_version(self):
version = test_util.add_version(self.session)
db_version = self.series_service.get_db_version()
assert version.version_number == db_version
def test_get_all_sites_empty(self):
sites = self.series_service.get_used_sites()
#assert len(sites) == 0
assert sites is None
def test_create_qualifier(self):
qual = Qualifier()
qual.code = "ABC123"
qual.description = "This is a test"
self.series_service.create_qualifier_by_qual(qual)
assert qual.id is not None
def test_get_qualifiers(self):
assert self.series_service.get_all_qualifiers() == []
qual = self.series_service.create_qualifier("ABC123", "This is a test")
db_qual = self.series_service.get_all_qualifiers()[0]
assert qual.id == db_qual.id
def test_get_all_sites(self):
assert self.series_service.get_used_sites() is None
site = test_util.add_site(self.session)
sites = self.series_service.get_used_sites()
assert sites is None
if isinstance(sites, list) and len(sites) > 0:
assert site.code == sites[0].code
series = test_util.add_series(self.session)
site = series.site
sites = self.series_service.get_used_sites()
assert len(sites) == 1
if isinstance(sites, list) and len(sites) > 0:
assert site.code == sites[0].code
def test_get_site_by_id_fail(self):
assert self.series_service.get_site_by_id(0) == None
site = test_util.add_site(self.session)
db_site = self.series_service.get_site_by_id(10)
assert db_site == None
def test_get_site_by_id(self):
site = test_util.add_site(self.session)
db_site = self.series_service.get_site_by_id(site.id)
assert db_site != None
assert site.code == db_site.code
def test_get_all_variables(self):
assert self.series_service.get_all_variables() == []
variable = test_util.add_variable(self.session)
variables = self.series_service.get_all_variables()
assert len(variables) == 1
assert variable.code == variables[0].code
def test_get_variable_by_id(self):
assert self.series_service.get_variable_by_id(10) == None
variable = test_util.add_variable(self.session)
db_var = self.series_service.get_variable_by_id(variable.id)
assert db_var != None
assert db_var.code == variable.code
def test_get_variables_by_site_code(self):
assert self.series_service.get_variables_by_site_code('ABC123') == []
series = test_util.add_series(self.session)
variable = series.variable
db_variables = self.series_service.get_variables_by_site_code(
series.site_code)
assert db_variables != None
assert variable.code == db_variables[0].code
def test_get_all_units(self):
assert self.series_service.get_all_units() == []
unit = test_util.add_unit(self.session)
units = self.series_service.get_all_units()
assert len(units) == 1
assert unit.name == units[0].name
def test_get_unit_by_name(self):
assert self.series_service.get_unit_by_name("FAIL") == None
unit = test_util.add_unit(self.session)
db_unit = self.series_service.get_unit_by_name(unit.name)
assert unit.id == db_unit.id
def test_get_unit_by_id(self):
assert self.series_service.get_unit_by_id(10) == None
unit = test_util.add_unit(self.session)
db_unit = self.series_service.get_unit_by_id(unit.id)
assert unit.name == db_unit.name
def test_get_all_series(self):
assert self.series_service.get_all_series() == []
series = test_util.add_series(self.session)
all_series = self.series_service.get_all_series()
assert all_series != []
assert series.id == all_series[0].id
def test_get_series_by_id(self):
assert self.series_service.get_series_by_id(10) == None
series = test_util.add_series(self.session)
db_series = self.series_service.get_series_by_id(series.id)
assert series.id == db_series.id
def test_get_series_by_id_quint(self):
assert self.series_service.get_series_by_id_quint(10, 10, 10, 10,
10) == None
series = test_util.add_series(self.session)
db_series = self.series_service.get_series_by_id_quint(
series.site_id, series.variable_id, series.method_id,
series.source_id, series.quality_control_level_id)
assert series.id == db_series.id
def test_series_exists(self):
assert self.series_service.series_exists_quint(10, 10, 10, 10,
10) == False
series = test_util.add_series(self.session)
site_id = series.site_id
var_id = series.variable_id
method_id = series.method_id
source_id = series.source_id
qcl_id = series.quality_control_level_id
assert self.series_service.series_exists_quint(site_id, var_id,
method_id, source_id,
qcl_id) == True
'''
def test_save_series(self):
series = Series()
site = test_util.add_site(self.session)
variable = test_util.add_variable(self.session)
method = test_util.add_method(self.session)
source = test_util.add_source(self.session)
qcl = test_util.add_qcl(self.session)
series.site_id = site.id
series.variable_id = variable.id
series.method_id = method.id
series.source_id = source.id
series.quality_control_level_id = qcl.id
dvs = []
for val in range(10):
dv = DataValue()
dv.data_value = val
dv.site_id = site.id
dv.variable_id = variable.id
dv.method_id = method.id
dv.source_id = source.id
dv.quality_control_level_id = qcl.id
dvs.append(dv)
print series.variable_code
assert self.series_service.save_series(series)
assert self.series_service.series_exists(site.id, variable.id, method.id, source.id, qcl.id)
assert not self.series_service.save_series(series)
'''
def test_get_data_value_by_id(self):
assert self.series_service.get_data_value_by_id(10) == None
data_values = test_util.add_series(self.session).data_values
dv = data_values[0]
db_dv = self.series_service.get_data_value_by_id(dv.id)
assert dv.data_value == db_dv.data_value
def test_get_qcl_by_id(self):
assert self.series_service.get_qcl_by_id(10) == None
qcl = test_util.add_qcl(self.session)
db_qcl = self.series_service.get_qcl_by_id(qcl.id)
assert qcl.code == db_qcl.code
def test_get_all_qcls(self):
assert self.series_service.get_all_qcls() == []
qcl = test_util.add_qcl(self.session)
all_qcls = self.series_service.get_all_qcls()
assert len(all_qcls) == 1
assert qcl.id == all_qcls[0].id
def test_get_all_methods(self):
assert self.series_service.get_all_methods() == []
method = test_util.add_method(self.session)
all_methods = self.series_service.get_all_methods()
assert len(all_methods) == 1
assert method.id == all_methods[0].id
def test_get_method_by_id(self):
assert self.series_service.get_method_by_id(10) == None
method = test_util.add_method(self.session)
db_method = self.series_service.get_method_by_id(method.id)
assert method.description == db_method.description
def test_delete_dvs(self):
series = test_util.add_series(self.session)
dvs = series.data_values
subset = dvs[:5]
self.series_service.delete_dvs([x.id for x in subset])
assert self.series_service.get_data_value_by_id(subset[0].id) == None
series = self.series_service.get_series_by_id(series.id) # Reload
assert len(series.data_values) == 5
def test_update_dvs(self):
series = test_util.add_series(self.session)
dvs = series.data_values
subset = dvs[:5]
for i in range(len(subset)):
subset[i].data_value = 100
self.series_service.update_dvs(subset)
series = self.series_service.get_series_by_id(series.id)
assert series.data_values[0].data_value == 100
def test_create_new_series(self):
site = test_util.add_site(self.session)
variable = test_util.add_variable(self.session)
method = test_util.add_method(self.session)
source = test_util.add_source(self.session)
qcl = test_util.add_qcl(self.session)
dvs = []
for val in range(10):
dv = DataValue()
dv.data_value = val
dv.site_id = site.id
dv.variable_id = variable.id
dv.method_id = method.id
dv.source_id = source.id
dv.quality_control_level_id = qcl.id
dvs.append(dv)
series = self.series_service.create_new_series(dvs, site.id,
variable.id, method.id,
source.id, qcl.id)
assert series != None
assert len(series.data_values) == 10
assert series.site_id == site.id
assert series.variable_id == variable.id
def test_update_series(self):
series = test_util.add_series(self.session)
series.site_code = "NEW"
series.variable_code = "NEW"
self.series_service.update_series(series)
series = self.series_service.get_series_by_id(series.id)
assert series.site_code == "NEW"
assert series.variable_code == "NEW"
def test_create_method(self):
description = "This tests creating a method"
link = "http://www.example.com"
method = self.series_service.create_method(description, link)
assert method.id != None
assert method.description == description
assert method.link == link
def test_create_variable(self):
unit = test_util.add_unit(self.session)
variable = self.series_service.create_variable(
"Code",
"Name",
"Speciation",
unit.id,
"SampleMedium",
"ValueType",
True, # is_regular
5.0, # time_support
unit.id, # time_unit_id
"DataType",
"GeneralCategory",
-999.0) # no_data_value
assert variable.id != None
assert variable.code == "Code"
assert variable.variable_unit_id == unit.id
def test_create_qcl(self):
qcl = self.series_service.create_qcl("Code", "Definition",
"Explanation")
assert qcl.id != None
assert qcl.code == "Code"
def test_delete_series(self):
series = test_util.add_series(self.session)
assert self.series_service.get_series_by_id(series.id) != None
self.series_service.delete_series(series)
assert self.series_service.get_series_by_id(series.id) == None
def test_qcl_exists(self):
qcl = test_util.add_qcl(self.session)
assert self.series_service.qcl_exists(qcl) == True
qcl.code = "00000"
qcl.definition = "A new definition"
assert self.series_service.qcl_exists(qcl) == False
def test_method_exists(self):
method = test_util.add_method(self.session)
assert self.series_service.method_exists(method) == True
method.description = "A new description"
assert self.series_service.method_exists(method) == False
def test_variable_exists(self):
variable = test_util.add_variable(self.session)
assert self.series_service.variable_exists(variable)
variable.code = "00000"
variable.name = "A new name"
assert not self.series_service.variable_exists(variable)