def setUp(self):
staticRunDate = datetime.datetime(
2018, 9, 1, 9, 0, 0).strftime('%Y_%m_%d_%H:%M:%S')
xferDB = TransferDB(dbFileName='copy_Pipeline.db',
dbDirectory='.',
dbKey='enilepiP',
plRunDate=staticRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
algorithm = 'InterVA'
sql = 'UPDATE Pipeline_Conf SET algorithm = ?, algorithmMetadataCode = ?'
par = ('InterVA', 'InterVA4|4.04|Custom|1|2016 WHO Verbal Autopsy Form|v1_4_1')
c.execute(sql, par)
sql = 'UPDATE InterVA_Conf SET version = ?'
par = ('4',)
c.execute(sql, par)
settingsPipeline = xferDB.configPipeline(conn)
settingsAlgorithm = xferDB.configOpenVA(conn,
algorithm,
settingsPipeline.workingDirectory)
self.rOpenVA = OpenVA(vaArgs=settingsAlgorithm,
pipelineArgs=settingsPipeline,
odkID=None,
runDate=staticRunDate)
zeroRecords = self.rOpenVA.copyVA()
self.rOpenVA.rScript()
conn.rollback()
conn.close()
def setUpClass(cls):
if not os.path.isfile('Pipeline.db'):
createTransferDB('Pipeline.db', '.', 'enilepiP')
if os.path.isfile('who_cod.R'):
os.remove('who_cod.R')
# if os.path.isfile('tariff_cod.py'):
# os.remove('tariff_cod.py')
pipelineRunDate = datetime.datetime(2018, 9, 1, 9, 0,
0).strftime('%Y_%m_%d_%H:%M:%S')
xferDB = TransferDB(dbFileName='Pipeline.db',
dbDirectory='.',
dbKey='enilepiP',
plRunDate=pipelineRunDate)
conn = xferDB.connectDB()
cls.cod_who = xferDB.configDHIS(conn, 'InSilicoVA')
cls.cod_tariff = xferDB.configDHIS(conn, 'SmartVA')
with open("who_cod.R", "w", newline="") as f:
f.write("data(causetextV5, package='InterVA5')\n")
f.write(
"write.csv(causetextV5, file='who_cod.csv', row.names=FALSE)\n"
)
rArgs = ["R", "CMD", "BATCH", "--vanilla", "who_cod.R"]
subprocess.run(args=rArgs,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=True)
who = read_csv("who_cod.csv", index_col=0)
index_who_causes = [i for i in who.index if "b_" in i]
cls.who_causes = who.loc[index_who_causes, who.columns[0]].tolist()
def test_cleanPipeline_odkLastRun(self):
"""Test update of ODK_Conf.odkLastRun."""
os.makedirs("DHIS/blobs/", exist_ok = True)
dbFileName = "copy_Pipeline.db"
dbDirectory = "."
dbKey = "enilepiP"
useDHIS = True
pl = Pipeline(dbFileName, dbDirectory,
dbKey, useDHIS)
pl.closePipeline()
nowDate = datetime.datetime.now()
pipelineRunDate = nowDate.strftime("%Y-%m-%d_%H:%M:%S")
xferDB = TransferDB(dbFileName = dbFileName,
dbDirectory = dbDirectory,
dbKey = dbKey,
plRunDate = pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
c.execute("SELECT odkLastRun FROM ODK_Conf;")
sqlQuery = c.fetchone()
results = [i for i in sqlQuery]
self.assertEqual(results[0], pipelineRunDate)
c.execute("UPDATE ODK_Conf SET odkLastRun = '1900-01-01_00:00:01';")
conn.commit()
conn.close()
def test_storeVA(self):
"""Check that depositResults() stores VA records in Transfer DB."""
shutil.copy("OpenVAFiles/sample_newStorage.csv",
"OpenVAFiles/newStorage.csv")
dbFileName = "Pipeline.db"
dbDirectory = "."
dbKey = "enilepiP"
useDHIS = True
nowDate = datetime.datetime.now()
pipelineRunDate = nowDate.strftime("%Y-%m-%d_%H:%M:%S")
xferDB = TransferDB(dbFileName = dbFileName,
dbDirectory = dbDirectory,
dbKey = dbKey,
plRunDate = pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
c.execute("DELETE FROM VA_Storage;")
conn.commit()
conn.close()
pl = Pipeline(dbFileName, dbDirectory, dbKey, useDHIS)
settings = pl.config()
settingsPipeline = settings["pipeline"]
settingsODK = settings["odk"]
settingsOpenVA = settings["openVA"]
settingsDHIS = settings["dhis"]
pl.storeResultsDB()
xferDB = TransferDB(dbFileName = dbFileName,
dbDirectory = dbDirectory,
dbKey = dbKey,
plRunDate = pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
sql = "SELECT id FROM VA_Storage"
c.execute(sql)
vaIDs = c.fetchall()
conn.close()
vaIDsList = [j for i in vaIDs for j in i]
s1 = set(vaIDsList)
dfNewStorage = pd.read_csv("OpenVAFiles/newStorage.csv")
dfNewStorageID = dfNewStorage["odkMetaInstanceID"]
s2 = set(dfNewStorageID)
self.assertTrue(s2.issubset(s1))
def setUpClass(cls):
if not os.path.isfile('Pipeline.db'):
createTransferDB('Pipeline.db', '.', 'enilepiP')
if os.path.isfile('OpenVAFiles/newStorage.csv'):
os.remove('OpenVAFiles/newStorage.csv')
shutil.copy('OpenVAFiles/sample_newStorage.csv',
'OpenVAFiles/newStorage.csv')
nowDate = datetime.datetime.now()
pipelineRunDate = nowDate.strftime('%Y-%m-%d_%H:%M:%S')
xferDB = TransferDB(dbFileName='Pipeline.db',
dbDirectory='.',
dbKey='enilepiP',
plRunDate=pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
c.execute('DELETE FROM VA_Storage;')
conn.commit()
conn.close()
pl = Pipeline('Pipeline.db', '.', 'enilepiP', True)
settings = pl.config()
settingsPipeline = settings['pipeline']
settingsODK = settings['odk']
settingsOpenVA = settings['openVA']
settingsDHIS = settings['dhis']
pl.storeResultsDB()
xferDB = TransferDB(dbFileName='Pipeline.db',
dbDirectory='.',
dbKey='enilepiP',
plRunDate=pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
sql = 'SELECT id FROM VA_Storage'
c.execute(sql)
vaIDs = c.fetchall()
conn.close()
vaIDsList = [j for i in vaIDs for j in i]
cls.s1 = set(vaIDsList)
dfNewStorage = read_csv('OpenVAFiles/newStorage.csv')
dfNewStorageID = dfNewStorage['odkMetaInstanceID']
cls.s2 = set(dfNewStorageID)
def setUpClass(cls):
if os.path.isfile('ODKFiles/odkBCExportNew.csv'):
os.remove('ODKFiles/odkBCExportNew.csv')
if os.path.isfile('ODKFiles/odkBCExportPrev.csv'):
os.remove('ODKFiles/odkBCExportPrev.csv')
shutil.copy('ODKFiles/odkExport_phmrc-1.csv',
'ODKFiles/odkBCExportPrev.csv')
shutil.copy('ODKFiles/odkExport_phmrc-2.csv',
'ODKFiles/odkBCExportNew.csv')
if not os.path.isfile('smartva'):
downloadSmartVA()
if not os.path.isfile('Pipeline.db'):
createTransferDB('Pipeline.db', '.', 'enilepiP')
# pipelineRunDate = datetime.datetime.now()
pipelineRunDate = datetime.datetime(2018, 9, 1, 9, 0, 0). \
strftime('%Y_%m_%d_%H:%M:%S')
xferDB = TransferDB(dbFileName = 'copy_Pipeline.db',
dbDirectory = '.',
dbKey = 'enilepiP',
plRunDate = pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
sql = 'UPDATE Pipeline_Conf SET algorithm = ?, algorithmMetadataCode = ?'
par = ('SmartVA', 'SmartVA|2.0.0_a8|PHMRCShort|1|PHMRCShort|1')
c.execute(sql, par)
settingsPipeline = xferDB.configPipeline(conn)
settingsODK = xferDB.configODK(conn)
settingsSmartVA = xferDB.configOpenVA(conn,
'SmartVA',
settingsPipeline.workingDirectory)
conn.rollback()
conn.close()
cls.staticRunDate = datetime.datetime(2018, 9, 1, 9, 0, 0). \
strftime('%Y_%m_%d_%H:%M:%S')
shutil.rmtree(
os.path.join('OpenVAFiles', cls.staticRunDate),
ignore_errors = True
)
cliSmartVA = OpenVA(vaArgs = settingsSmartVA,
pipelineArgs = settingsPipeline,
odkID = settingsODK.odkID,
runDate = cls.staticRunDate)
zeroRecords = cliSmartVA.copyVA()
cls.completed = cliSmartVA.getCOD()
cls.svaOut = os.path.join(
'OpenVAFiles',
cls.staticRunDate,
'1-individual-cause-of-death/individual-cause-of-death.csv'
)
def setUpClass(cls):
if os.path.isfile('ODKFiles/odkBCExportNew.csv'):
os.remove('ODKFiles/odkBCExportNew.csv')
if os.path.isfile('ODKFiles/odkBCExportPrev.csv'):
os.remove('ODKFiles/odkBCExportPrev.csv')
shutil.copy('ODKFiles/previous_bc_export.csv',
'ODKFiles/odkBCExportPrev.csv')
shutil.copy('ODKFiles/another_bc_export.csv',
'ODKFiles/odkBCExportNew.csv')
if os.path.isfile('Check_InSilicoVA_Pipeline.db'):
os.remove('Check_InSilicoVA_Pipeline.db')
createTransferDB('Check_InSilicoVA_Pipeline.db', '.', 'enilepiP')
# pipelineRunDate = datetime.datetime.now()
pipelineRunDate = datetime.datetime(2018, 9, 1, 9, 0, 0). \
strftime('%Y_%m_%d_%H:%M:%S')
xferDB = TransferDB(dbFileName = 'Check_InSilicoVA_Pipeline.db',
dbDirectory = '.',
dbKey = 'enilepiP',
plRunDate = pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
sql = 'UPDATE Pipeline_Conf SET algorithm = ?, algorithmMetadataCode = ?'
par = ('InSilicoVA', 'InSilicoVA-2016|1.0.0|InterVA|5|2016 WHO Verbal Autopsy Form|v1_4_1')
c.execute(sql, par)
sql = 'UPDATE InSilicoVA_Conf SET data_type = ?'
par = ('WHO2016',)
c.execute(sql, par)
settingsPipeline = xferDB.configPipeline(conn)
settingsODK = xferDB.configODK(conn)
settingsInSilicoVA = xferDB.configOpenVA(conn,
'InSilicoVA',
settingsPipeline.workingDirectory)
# conn.rollback()
conn.close()
cls.staticRunDate = datetime.datetime(2018, 9, 1, 9, 0, 0). \
strftime('%Y_%m_%d_%H:%M:%S')
cls.rScript = os.path.join('OpenVAFiles', cls.staticRunDate,
'Rscript_' + cls.staticRunDate + '.R')
cls.rOutFile = os.path.join('OpenVAFiles', cls.staticRunDate,
'Rscript_' + cls.staticRunDate + '.Rout')
rOpenVA = OpenVA(vaArgs = settingsInSilicoVA,
pipelineArgs = settingsPipeline,
odkID = settingsODK.odkID,
runDate = cls.staticRunDate)
zeroRecords = rOpenVA.copyVA()
rOpenVA.rScript()
cls.completed = rOpenVA.getCOD()
def setUpClass(cls):
nowDate = datetime.datetime.now()
pipelineRunDate = nowDate.strftime('%Y-%m-%d_%H:%M:%S')
xferDB = TransferDB(dbFileName='copy_Pipeline.db',
dbDirectory='.',
dbKey='enilepiP',
plRunDate=pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
sql = 'UPDATE Pipeline_Conf SET algorithm = ?, algorithmMetadataCode = ?'
par = (
'InSilicoVA',
'InSilicoVA|1.1.4|InterVA|5|2016 WHO Verbal Autopsy Form|v1_4_1')
c.execute(sql, par)
sql = 'UPDATE InSilicoVA_Conf SET data_type = ?'
par = ('WHO2016', )
c.execute(sql, par)
conn.commit()
conn.close()
cls.pl = Pipeline('copy_Pipeline.db', '.', 'enilepiP', True)
settings = cls.pl.config()
settingsPipeline = settings['pipeline']
settingsODK = settings['odk']
settingsOpenVA = settings['openVA']
if os.path.isfile('ODKFiles/odkBCExportNew.csv'):
os.remove('ODKFiles/odkBCExportNew.csv')
if os.path.isfile('ODKFiles/odkBCExportPrev.csv'):
os.remove('ODKFiles/odkBCExportPrev.csv')
shutil.copy('ODKFiles/previous_bc_export.csv',
'ODKFiles/odkBCExportNew.csv')
shutil.copy('ODKFiles/another_bc_export.csv',
'ODKFiles/odkBCExportNew.csv')
if os.path.isfile('OpenVAFiles/recordStorage.csv'):
os.remove('OpenVAFiles/recordStorage.csv')
if os.path.isfile('OpenVAFiles/entityAttributeValue.csv'):
os.remove('OpenVAFiles/entityAttributeValue.csv')
cls.rOut = cls.pl.runOpenVA(settingsOpenVA, settingsPipeline,
settingsODK.odkID, cls.pl.pipelineRunDate)
def setUpClass(cls):
if os.path.isfile('ODKFiles/odkBCExportNew.csv'):
os.remove('ODKFiles/odkBCExportNew.csv')
if os.path.isfile('ODKFiles/odkBCExportPrev.csv'):
os.remove('ODKFiles/odkBCExportPrev.csv')
shutil.copy('ODKFiles/zeroRecords_bc_export.csv',
'ODKFiles/odkBCExportPrev.csv')
shutil.copy('ODKFiles/zeroRecords_bc_export.csv',
'ODKFiles/odkBCExportNew.csv')
if os.path.isfile('OpenVAFiles/openVA_input.csv'):
os.remove('OpenVAFiles/openVA_input.csv')
if not os.path.isfile('Pipeline.db'):
createTransferDB('Pipeline.db', '.', 'enilepiP')
# pipelineRunDate = datetime.datetime.now()
pipelineRunDate = datetime.datetime(2018, 9, 1, 9, 0, 0). \
strftime('%Y_%m_%d_%H:%M:%S')
xferDB = TransferDB(dbFileName = 'Pipeline.db',
dbDirectory = '.',
dbKey = 'enilepiP',
plRunDate = pipelineRunDate)
conn = xferDB.connectDB()
settingsPipeline = xferDB.configPipeline(conn)
settingsODK = xferDB.configODK(conn)
settingsInterVA = xferDB.configOpenVA(conn,
'InterVA',
settingsPipeline.workingDirectory)
cls.staticRunDate = datetime.datetime(2018, 9, 1, 9, 0, 0). \
strftime('%Y_%m_%d_%H:%M:%S')
shutil.rmtree(
os.path.join('OpenVAFiles', cls.staticRunDate),
ignore_errors = True
)
rOpenVA = OpenVA(vaArgs = settingsInterVA,
pipelineArgs = settingsPipeline,
odkID = settingsODK.odkID,
runDate = cls.staticRunDate)
cls.zeroRecords = rOpenVA.copyVA()
def setUp(self):
staticRunDate = datetime.datetime(
2018, 9, 1, 9, 0, 0).strftime('%Y_%m_%d_%H:%M:%S')
xferDB = TransferDB(dbFileName='copy_Pipeline.db',
dbDirectory='.',
dbKey='enilepiP',
plRunDate=staticRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
algorithm = 'SmartVA'
sql = 'UPDATE Pipeline_Conf SET algorithm = ?, algorithmMetadataCode = ?'
par = ('SmartVA', 'SmartVA|2.0.0_a8|PHMRCShort|1|PHMRCShort|1')
c.execute(sql, par)
ntSmartVA = collections.namedtuple("ntSmartVA",
["SmartVA_country",
"SmartVA_hiv",
"SmartVA_malaria",
"SmartVA_hce",
"SmartVA_freetext",
"SmartVA_figures",
"SmartVA_language"])
settingsAlgorithm = ntSmartVA("Unknown",
"Wrong",
"Wrong",
"Wrong",
"Wrong",
"Wrong",
"Wrong")
settingsPipeline = xferDB.configPipeline(conn)
self.rOpenVA = OpenVA(vaArgs=settingsAlgorithm,
pipelineArgs=settingsPipeline,
odkID=None,
runDate=staticRunDate)
zeroRecords = self.rOpenVA.copyVA()
conn.rollback()
conn.close()
def setUpClass(cls):
if not os.path.isfile('ODKFiles/odkBCExportNew.csv'):
shutil.copy('ODKFiles/previous_bc_export.csv',
'ODKFiles/odkBCExportPrev.csv')
if not os.path.isfile('ODKFiles/odkBCExportPrev.csv'):
shutil.copy('ODKFiles/another_bc_export.csv',
'ODKFiles/odkBCExportNew.csv')
if not os.path.isfile('OpenVAFiles/openVA_input.csv'):
shutil.copy('OpenVAFiles/sample_openVA_input.csv',
'OpenVAFiles/openVA_input.csv')
if not os.path.isfile('OpenVAFiles/entityAttributeValue.csv'):
shutil.copy('OpenVAFiles/sampleEAV.csv',
'OpenVAFiles/entityAttributeValue.csv')
if not os.path.isfile('OpenVAFiles/recordStorage.csv'):
shutil.copy('OpenVAFiles/sample_recordStorage.csv',
'OpenVAFiles/recordStorage.csv')
if not os.path.isfile('OpenVAFiles/newStorage.csv'):
shutil.copy('OpenVAFiles/sample_newStorage.csv',
'OpenVAFiles/newStorage.csv')
os.makedirs('DHIS/blobs/', exist_ok=True)
shutil.copy('OpenVAFiles/sample_newStorage.csv',
'DHIS/blobs/001-002-003.db')
nowDate = datetime.datetime.now()
pipelineRunDate = nowDate.strftime('%Y-%m-%d_%H:%M:%S')
cls.pl = Pipeline('copy_Pipeline.db', '.', 'enilepiP', True)
cls.pl.closePipeline()
xferDB = TransferDB(dbFileName='copy_Pipeline.db',
dbDirectory='.',
dbKey='enilepiP',
plRunDate=pipelineRunDate)
cls.conn = xferDB.connectDB()
cls.c = cls.conn.cursor()
def setUpClass(cls):
shutil.rmtree('DHIS/blobs/', ignore_errors = True)
shutil.copy('OpenVAFiles/sampleEAV.csv',
'OpenVAFiles/entityAttributeValue.csv')
shutil.copy('OpenVAFiles/sample_recordStorage.csv',
'OpenVAFiles/recordStorage.csv')
# Define valid parameters for SwissTPH DHIS2 Server.
dirOpenVA = 'OpenVAFiles'
dhisURL = 'https://va30se.swisstph-mis.ch'
# dhisURL = 'https://va25.swisstph-mis.ch'
dhisUser = '******'
dhisPassword = '******'
dhisOrgUnit = 'SCVeBskgiK6'
# parameters for connecting to DB (assuming DB is in tests folder)
dbFileName = 'Pipeline.db'
dbKey = 'enilepiP'
wrong_dbKey = 'wrongKey'
# dbDirectory = os.path.abspath(os.path.dirname(__file__))
dbDirectory = '.'
if not os.path.isfile('Pipeline.db'):
createTransferDB(dbFileName, dbDirectory, dbKey)
pipelineRunDate = datetime.datetime.now()
xferDB = TransferDB(dbFileName = dbFileName,
dbDirectory = dbDirectory,
dbKey = dbKey,
plRunDate = pipelineRunDate)
conn = xferDB.connectDB()
settingsDHIS = xferDB.configDHIS(conn, 'InSilicoVA')
cls.pipelineDHIS = dhis.DHIS(settingsDHIS, '.')
apiDHIS = cls.pipelineDHIS.connect()
cls.postLog = cls.pipelineDHIS.postVA(apiDHIS)
cls.pipelineDHIS.verifyPost(cls.postLog, apiDHIS)
def setUp(self):
if os.path.isfile('ODKFiles/odkBCExportNew.csv'):
os.remove('ODKFiles/odkBCExportNew.csv')
if os.path.isfile('ODKFiles/odkBCExportPrev.csv'):
os.remove('ODKFiles/odkBCExportPrev.csv')
shutil.copy('ODKFiles/odkExport_phmrc-1.csv',
'ODKFiles/odkBCExportPrev.csv')
shutil.copy('ODKFiles/odkExport_phmrc-2.csv',
'ODKFiles/odkBCExportNew.csv')
if not os.path.isfile('smartva'):
downloadSmartVA()
self.staticRunDate = datetime.datetime(2018, 9, 1, 9, 0, 0). \
strftime('%Y_%m_%d_%H:%M:%S')
xferDB = TransferDB(dbFileName = 'copy_Pipeline.db',
dbDirectory = '.',
dbKey = 'enilepiP',
plRunDate = self.staticRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
if self.id() == 'test_openVA.Check_Exceptions.test_insilico_exception':
algorithm = 'InSilicoVA'
sql = 'UPDATE Pipeline_Conf SET algorithm = ?, algorithmMetadataCode = ?'
par = ('InSilicoVA', 'InSilicoVA|1.1.4|Custom|1|2016 WHO Verbal Autopsy Form|v1_4_1')
c.execute(sql, par)
sql = 'UPDATE InSilicoVA_Conf SET data_type = ?'
par = ('WHO2016',)
c.execute(sql, par)
elif self.id() == 'test_openVA.Check_Exceptions.test_interva_exception':
algorithm = 'InterVA'
sql = 'UPDATE Pipeline_Conf SET algorithm = ?, algorithmMetadataCode = ?'
par = ('InterVA', 'InterVA4|4.04|Custom|1|2016 WHO Verbal Autopsy Form|v1_4_1')
c.execute(sql, par)
sql = 'UPDATE InterVA_Conf SET version = ?'
par = ('5',)
c.execute(sql, par)
else:
algorithm = 'SmartVA'
sql = 'UPDATE Pipeline_Conf SET algorithm = ?, algorithmMetadataCode = ?'
par = ('SmartVA', 'SmartVA|2.0.0_a8|PHMRCShort|1|PHMRCShort|1')
c.execute(sql, par)
settingsPipeline = xferDB.configPipeline(conn)
settingsODK = xferDB.configODK(conn)
settingsAlgorithm = xferDB.configOpenVA(conn,
algorithm,
settingsPipeline.workingDirectory)
if self.id() == 'test_openVA.Check_Exceptions.test_smartva_exception':
ntSmartVA = collections.namedtuple("ntSmartVA",
["SmartVA_country",
"SmartVA_hiv",
"SmartVA_malaria",
"SmartVA_hce",
"SmartVA_freetext",
"SmartVA_figures",
"SmartVA_language"]
)
settingsAlgorithm = ntSmartVA("Unknown",
"Wrong",
"Wrong",
"Wrong",
"Wrong",
"Wrong",
"Wrong")
conn.rollback()
conn.close()
self.rOpenVA = OpenVA(vaArgs = settingsAlgorithm,
pipelineArgs = settingsPipeline,
odkID = '',
runDate = self.staticRunDate)
zeroRecords = self.rOpenVA.copyVA()
self.rOpenVA.rScript()
class Check_storeResultsDB(unittest.TestCase):
"""Check storeResultsDB method marks duplicate records:"""
@classmethod
def setUpClass(cls):
if not os.path.isfile('Pipeline.db'):
createTransferDB('Pipeline.db', '.', 'enilepiP')
def setUp(self):
shutil.rmtree('ODKFiles/ODK Briefcase Storage/', ignore_errors=True)
shutil.rmtree('DHIS/blobs/', ignore_errors=True)
if os.path.isfile('ODKFiles/odkBCExportNew.csv'):
os.remove('ODKFiles/odkBCExportNew.csv')
if os.path.isfile('ODKFiles/odkBCExportPrev.csv'):
os.remove('ODKFiles/odkBCExportPrev.csv')
if not os.path.isfile('ODK-Briefcase-v1.18.0.jar'):
downloadBriefcase()
self.pl = Pipeline('Pipeline.db', '.', 'enilepiP', True)
self.settings = self.pl.config()
self.settingsPipeline = self.settings['pipeline']
self.settingsODK = self.settings['odk']
self.settingsOpenVA = self.settings['openVA']
self.settingsDHIS = self.settings['dhis']
self.xferDB = TransferDB(dbFileName='Pipeline.db',
dbDirectory='.',
dbKey='enilepiP',
plRunDate=True)
self.conn = self.xferDB.connectDB()
self.c = self.conn.cursor()
self.c.execute('DELETE FROM EventLog;')
self.conn.commit()
self.c.execute('DELETE FROM VA_Storage;')
self.conn.commit()
self.odkBC = self.pl.runODK(self.settingsODK, self.settingsPipeline)
def test_runODK_checkDuplicates(self):
"""Check checkDuplicates() method:"""
vaRecords = read_csv('ODKFiles/odkBCExportNew.csv')
nVA = vaRecords.shape[0]
rOut = self.pl.runOpenVA(self.settingsOpenVA, self.settingsPipeline,
self.settingsODK.odkID,
self.pl.pipelineRunDate)
pipelineDHIS = self.pl.runDHIS(self.settingsDHIS,
self.settingsPipeline)
self.pl.storeResultsDB()
os.remove('ODKFiles/odkBCExportNew.csv')
os.remove('OpenVAFiles/pycrossva_input.csv')
os.remove('OpenVAFiles/openVA_input.csv')
odkBC2 = self.pl.runODK(self.settingsODK, self.settingsPipeline)
self.c.execute('SELECT eventDesc FROM EventLog;')
query = self.c.fetchall()
nDuplicates = [i[0] for i in query if 'duplicate' in i[0]]
self.assertEqual(len(nDuplicates), nVA)
def tearDown(self):
if os.path.isfile('ODKFiles/odkBCExportNew.csv'):
os.remove('ODKFiles/odkBCExportNew.csv')
if os.path.isfile('ODKFiles/odkBCExportPrev.csv'):
os.remove('ODKFiles/odkBCExportPrev.csv')
shutil.rmtree('DHIS/blobs/', ignore_errors=True)
shutil.rmtree('ODKFiles/ODK Briefcase Storage/', ignore_errors=True)
self.conn.close()
@classmethod
def tearDownClass(cls):
os.remove('Pipeline.db')
def test_cleanPipeline_rmFiles(self):
"""Test file removal."""
if not os.path.isfile("ODKFiles/odkBCExportNew.csv"):
shutil.copy("ODKFiles/previous_bc_export.csv",
"ODKFiles/odkBCExportPrev.csv")
if not os.path.isfile("ODKFiles/odkBCExportPrev.csv"):
shutil.copy("ODKFiles/another_bc_export.csv",
"ODKFiles/odkBCExportNew.csv")
if not os.path.isfile("OpenVAFiles/openVA_input.csv"):
shutil.copy("OpenVAFiles/sample_openVA_input.csv",
"OpenVAFiles/openVA_input.csv")
if not os.path.isfile("OpenVAFiles/entityAttributeValue.csv"):
shutil.copy("OpenVAFiles/sampleEAV.csv",
"OpenVAFiles/entityAttributeValue.csv")
if not os.path.isfile("OpenVAFiles/recordStorage.csv"):
shutil.copy("OpenVAFiles/sample_recordStorage.csv",
"OpenVAFiles/recordStorage.csv")
if not os.path.isfile("OpenVAFiles/newStorage.csv"):
shutil.copy("OpenVAFiles/sample_newStorage.csv",
"OpenVAFiles/newStorage.csv")
os.makedirs("DHIS/blobs/", exist_ok = True)
shutil.copy("OpenVAFiles/sample_newStorage.csv",
"DHIS/blobs/001-002-003.db")
dbFileName = "copy_Pipeline.db"
dbDirectory = "."
dbKey = "enilepiP"
useDHIS = True
nowDate = datetime.datetime.now()
pipelineRunDate = nowDate.strftime("%Y-%m-%d_%H:%M:%S")
pl = Pipeline(dbFileName, dbDirectory, dbKey, useDHIS)
pl.closePipeline()
fileExist = False
if os.path.isfile("ODKFiles/odkBCExportNew.csv"):
fileExist = True
if os.path.isfile("ODKFiles/odkBCExportPrev.csv"):
fileExist = True
if os.path.isfile("ODKFiles/odkBCExportNew.csv"):
fileExist = True
if os.path.isfile("OpenVAFiles/openVA_input.csv"):
fileExist = True
if os.path.isfile("OpenVAFiles/entityAttributeValue.csv"):
fileExist = True
if os.path.isfile("OpenVAFiles/recordStorage.csv"):
fileExist = True
if os.path.isfile("OpenVAFiles/newStorage.csv"):
fileExist = True
if os.path.isfile("DHIS/blobs/001-002-003.db"):
fileExist = True
self.assertFalse(fileExist)
xferDB = TransferDB(dbFileName = dbFileName,
dbDirectory = dbDirectory,
dbKey = dbKey,
plRunDate = pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
c.execute("UPDATE ODK_Conf SET odkLastRun = '1900-01-01_00:00:01';")
conn.commit()
conn.close()
def test_runODK_6(self):
"""Check checkDuplicates() method."""
if os.path.isfile("ODKFiles/odkBCExportNew.csv"):
os.remove("ODKFiles/odkBCExportNew.csv")
if os.path.isfile("ODKFiles/odkBCExportPrev.csv"):
os.remove("ODKFiles/odkBCExportPrev.csv")
if os.path.isfile("OpenVAFiles/openVA_input.csv"):
os.remove("OpenVAFiles/openVA_input.csv")
dbFileName = "copy_Pipeline.db"
dbDirectory = "."
dbKey = "enilepiP"
useDHIS = True
nowDate = datetime.datetime.now()
pipelineRunDate = nowDate.strftime("%Y-%m-%d_%H:%M:%S")
xferDB = TransferDB(dbFileName = dbFileName,
dbDirectory = dbDirectory,
dbKey = dbKey,
plRunDate = pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
c.execute("DELETE FROM EventLog;")
conn.commit()
c.execute("DELETE FROM VA_Storage;")
conn.commit()
conn.close()
pl = Pipeline(dbFileName, dbDirectory, dbKey, useDHIS)
settings = pl.config()
settingsPipeline = settings["pipeline"]
settingsODK = settings["odk"]
settingsOpenVA = settings["openVA"]
settingsDHIS = settings["dhis"]
odkBC = pl.runODK(settingsODK,
settingsPipeline)
vaRecords = pd.read_csv("ODKFiles/odkBCExportNew.csv")
nVA = vaRecords.shape[0]
rOut = pl.runOpenVA(settingsOpenVA,
settingsPipeline,
settingsODK.odkID,
pl.pipelineRunDate)
pipelineDHIS = pl.runDHIS(settingsDHIS,
settingsPipeline)
pl.storeResultsDB()
os.remove("ODKFiles/odkBCExportNew.csv")
os.remove("OpenVAFiles/openVA_input.csv")
odkBC2 = pl.runODK(settingsODK,
settingsPipeline)
xferDB = TransferDB(dbFileName = dbFileName,
dbDirectory = dbDirectory,
dbKey = dbKey,
plRunDate = pl.pipelineRunDate)
conn = xferDB.connectDB()
c = conn.cursor()
c.execute("SELECT eventDesc FROM EventLog;")
query = c.fetchall()
nDuplicates = [i[0] for i in query if "Duplicate" in i[0]]
self.assertEqual(len(nDuplicates), nVA)
shutil.rmtree("OpenVAFiles/" + pl.pipelineRunDate)
shutil.rmtree("DHIS/blobs/")
os.remove("OpenVAFiles/newStorage.csv")
os.remove("OpenVAFiles/recordStorage.csv")
os.remove("OpenVAFiles/entityAttributeValue.csv")