def test_archive2folder(self):
##############################
print "Running unittests for this project: ", project_name
print "Running this unittest: ", self._testMethodName
##############################
from zipper import zipUtils
zip_archive = r"%s%sInvestigation.blah.zip" % (dir_of_the_current_test, delimiter())
instOfzipUtils = zipUtils(delimiter_value=delimiter())
sub_folder_count = instOfzipUtils.archive2folder(zip_archive)
print "sub_folder_count=", sub_folder_count
assert 0 == sub_folder_count
cwd = os.getcwd()
# os.chdir(r'%s%sdoesNotExist.blah' % (dir_of_the_current_test,delimiter())) #caused unittest to fail as expected (tried separately)
os.chdir(r"%s%sInvestigation.blah" % (dir_of_the_current_test, delimiter()))
os.remove("m_1.xls")
# os.remove('does_not_exist.xls') #caused unittest to fail as expected (tried separately)
os.chdir(dir_of_the_current_test)
os.rmdir("Investigation.blah")
os.chdir(cwd)
def createFlatTxtArchive(xls_folder, mustEditAccessionCodes,
mustRemoveComments,
mustMakeNanoPUZZLESspecificChanges):
flat_txt_archive = xls_folder + "-txt.zip"
###########
#Rename the output file if non-default options are used
if mustEditAccessionCodes:
flat_txt_archive = re.sub('(\.zip$)', '_opt-a.zip', flat_txt_archive)
if mustRemoveComments:
flat_txt_archive = re.sub('(\.zip$)', '_opt-c.zip', flat_txt_archive)
if mustMakeNanoPUZZLESspecificChanges:
flat_txt_archive = re.sub('(\.zip$)', '_opt-N.zip', flat_txt_archive)
###########
cwd = os.getcwd()
os.chdir(xls_folder)
for xls_file in glob.glob('*.xls'):
os.remove(xls_file)
os.chdir(cwd)
del cwd, xls_file
instOfzipUtils = zipUtils(delimiter_value=delimiter())
instOfzipUtils.filesIntoFlatArchive(folder_name=xls_folder,
zip_archive=flat_txt_archive)
del instOfzipUtils
def test_convertTOYtest11_removeComments(self):
##############################
print 'Running unittests for this project: ', project_name
print 'Running this unittest: ', self._testMethodName
##############################
##########################
#IMPORTANT: TOY.test_11.zip JUST COPIED FROM test_11 WITHOUT EDITING; EXPECTED OUTPUT [* - Copy.txt] ALSO COPIED, BUT MANUALLY EDITED [WITHOUT CHANGING FILE NAMES!] IN ACCORDANCE WITH EXPECTED CHANGES NOW THAT "-c" FLAG IS BEING USED!
#########################
cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s" -c' % (project_modules_to_test_dir,delimiter(),example_input)
assert 0 == os.system(cmd)
instOfzipUtils = zipUtils(delimiter_value=delimiter())
sub_folder_count = instOfzipUtils.archive2folder(re.sub('(\.zip$)','-txt_opt-c.zip',example_input))
cwd = os.getcwd()
os.chdir(re.sub('(\.zip$)','-txt_opt-c',example_input))
comparisons_count = 0
for orig_file in example_output_to_compare_to:
new_file = r'%s%s%s' % (os.getcwd(),delimiter(),re.sub('( - Copy\.txt)','.txt',orig_file.split(delimiter())[-1]))
self.compareOriginalAndNewFiles(orig_file,new_file)
os.remove(new_file)
comparisons_count += 1
assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (comparisons_count,expected_no_output_files)
os.chdir(cwd)
os.remove(re.sub('(\.zip$)','-txt_opt-c.zip',example_input))
os.rmdir(re.sub('(\.zip$)','-txt_opt-c',example_input))
def createFlatTxtArchive(xls_folder,mustEditAccessionCodes,mustRemoveComments,mustMakeNanoPUZZLESspecificChanges):
flat_txt_archive = xls_folder+"-txt.zip"
###########
#Rename the output file if non-default options are used
if mustEditAccessionCodes:
flat_txt_archive = re.sub('(\.zip$)','_opt-a.zip',flat_txt_archive)
if mustRemoveComments:
flat_txt_archive = re.sub('(\.zip$)','_opt-c.zip',flat_txt_archive)
if mustMakeNanoPUZZLESspecificChanges:
flat_txt_archive = re.sub('(\.zip$)','_opt-N.zip',flat_txt_archive)
###########
cwd = os.getcwd()
os.chdir(xls_folder)
for xls_file in glob.glob('*.xls'):
os.remove(xls_file)
os.chdir(cwd)
del cwd,xls_file
instOfzipUtils = zipUtils(delimiter_value=delimiter())
instOfzipUtils.filesIntoFlatArchive(folder_name=xls_folder,zip_archive=flat_txt_archive)
del instOfzipUtils
def test_archive2folder(self):
##############################
print 'Running unittests for this project: ', project_name
print 'Running this unittest: ', self._testMethodName
##############################
from zipper import zipUtils
zip_archive = r'%s%sInvestigation.blah.zip' % (dir_of_the_current_test,
delimiter())
instOfzipUtils = zipUtils(delimiter_value=delimiter())
sub_folder_count = instOfzipUtils.archive2folder(zip_archive)
print 'sub_folder_count=', sub_folder_count
assert 0 == sub_folder_count
cwd = os.getcwd()
#os.chdir(r'%s%sdoesNotExist.blah' % (dir_of_the_current_test,delimiter())) #caused unittest to fail as expected (tried separately)
os.chdir(r'%s%sInvestigation.blah' %
(dir_of_the_current_test, delimiter()))
os.remove('m_1.xls')
#os.remove('does_not_exist.xls') #caused unittest to fail as expected (tried separately)
os.chdir(dir_of_the_current_test)
os.rmdir('Investigation.blah')
os.chdir(cwd)
def idInputFiles(xls_folder):
instOfzipUtils = zipUtils(delimiter())
input_files = instOfzipUtils.getRelativeFileNames(xls_folder)
input_files = [
r'%s%s%s' % (xls_folder, delimiter(), file) for file in input_files
if fileNameRegexesDict['input_extension'].search(file)
]
del file
del instOfzipUtils
#non_xls_files = [file_name for file_name in input_files if not fileNameRegexesDict['input_extension'].search(file_name)]
#del file_name
#assert 0 == len(non_xls_files),"There are %d non-xls files in the folder %s created from the input archive." % (len(non_xls_files),xls_folder)
#del non_xls_files
fileType2No = {}
for fileType in all_file_types:
fileType2No[fileType] = len([
file_name for file_name in input_files
if fileNameRegexesDict[fileType].match(
file_name.split(delimiter())[-1])
])
assert not 0 == fileType2No[
fileType], "Zero %s input files in the folder created from the input archive!" % fileType
print "%d %s input files in the folder created from the input archive!" % (
fileType2No[fileType], fileType)
del fileType2No
return input_files #,non_xls_files #non_xls_files should just be copied across to the final zip archive without modification -see "def createFlatTxtArchive(xls_folder):"
def extractXlsFolder(xls_archive):
instOfzipUtils = zipUtils(delimiter())
sub_folder_count = instOfzipUtils.archive2folder(xls_archive)
assert 0 == sub_folder_count
return instOfzipUtils.folder_name
def extractXlsFolder(xls_archive): instOfzipUtils = zipUtils(delimiter()) sub_folder_count = instOfzipUtils.archive2folder(xls_archive) assert 0 == sub_folder_count return instOfzipUtils.folder_name
def test_filesIntoFlatArchive(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## from zipper import zipUtils folder = r'%s%sInvestigation.blah-TXT' % (dir_of_the_current_test,delimiter()) instOfzipUtils = zipUtils(delimiter_value=delimiter()) instOfzipUtils.filesIntoFlatArchive(folder_name=folder,zip_archive=folder+".zip") del instOfzipUtils os.remove(r'%s%sInvestigation.blah-TXT.zip' % (dir_of_the_current_test,delimiter())) #when this was commented, archive looked as expected (flat archive containing a single text file)
def test_NanoPUZZLES_inconsistency_fix_3(self):
##############################
print 'Running unittests for this project: ', project_name
print 'Running this unittest: ', self._testMethodName
##############################
##########################
#This unit test is designed to check (an aspect of) the "-N" option for this program i.e. the conversion of potentially inconsistent fields in datasets arising from older versions of the NanoPUZZLES ISA-TAB-Nano templates
#N.B. The "....TOY.test_11... - Copy.txt" files (which were checked to make sure they were not inconsistent with the expected changes to be made via the "-N" option) were just renamed and the "TOY.test_11.zip" Excel files were updated to make sure they contained (one of) the inconsistencies that the "-N" option is designed to fix, then all such files were renamed to give "TOY.test_17.zip".
#[*] In addition, given the changes to the file names, the internal contents of the files (both Excel and expected text output) needed to be edited such that "TOY.test_11" was replaced with "TOY.test_17" for consistency.
#########################
##############################
#test_17: inconsistency(ies) fix which is (are) being tested: replace "Characteristics[shape]" fields with "Characteristics [shape {NPO:http://purl.bioontology.org/ontology/npo#NPO_274}]"
##############################
cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s" -N' % (
project_modules_to_test_dir, delimiter(), example_input)
assert 0 == os.system(cmd)
instOfzipUtils = zipUtils(delimiter_value=delimiter())
sub_folder_count = instOfzipUtils.archive2folder(
re.sub('(\.zip$)', '-txt_opt-N.zip', example_input))
cwd = os.getcwd()
os.chdir(re.sub('(\.zip$)', '-txt_opt-N', example_input))
comparisons_count = 0 #This line would actually be useful in earlier unit tests!
for orig_file in example_output_to_compare_to:
new_file = r'%s%s%s' % (os.getcwd(), delimiter(),
re.sub('( - Copy\.txt)', '.txt',
orig_file.split(delimiter())[-1]))
self.compareOriginalAndNewFiles(orig_file, new_file)
os.remove(new_file)
comparisons_count += 1 #This line would actually be useful in earlier unit tests!
assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (
comparisons_count, expected_no_output_files
) #This line would actually be useful in earlier unit tests!
os.chdir(cwd)
os.remove(re.sub('(\.zip$)', '-txt_opt-N.zip', example_input))
os.rmdir(re.sub('(\.zip$)', '-txt_opt-N', example_input))
def test_convertTOYtest11_defaultOptions(self):
##############################
print 'Running unittests for this project: ', project_name
print 'Running this unittest: ', self._testMethodName
##############################
##########################
#Observations regarding TOY.test_11.zip files which are relevant to testing capabilities of the code to fix xls errors:
#s_TOY.test_11_InVitro.CB.xls => "Factor Value[culture medium]" : should be converted to "Factor Value [culture medium]" in txt version
#a_TOY.test_11_PC_size_TEM.xls => "Parameter Value[Instrument Version]" : ditto "Parameter Value [Instrument Version]"
#ditto => "Measurement Value[maximum(diameter)]" ditto "Measurement Value [maximum(diameter)]"
#m_TiO2_TOY.test_11.xls -> Excel 2010 saved as tab delimited text => "Characteristics [Product impurities found {MEDDRA:\nhttp://purl.bioontology.org/ontology/MDR/10069178}]" i.e. this field contained a line ending inside a field - which should be removed in code created version!
#ditto => Characteristics[purity {NPO: http://purl.bioontology.org/ontology/npo#NPO_1345}] : should be converted to "Characteristics [purity {NPO:http://purl.bioontology.org/ontology/npo#NPO_1345}]"
#########################
cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s"' % (
project_modules_to_test_dir, delimiter(), example_input)
assert 0 == os.system(cmd)
instOfzipUtils = zipUtils(delimiter_value=delimiter())
sub_folder_count = instOfzipUtils.archive2folder(
re.sub('(\.zip$)', '-txt.zip', example_input))
cwd = os.getcwd()
os.chdir(re.sub('(\.zip$)', '-txt', example_input))
comparisons_count = 0
for orig_file in example_output_to_compare_to:
new_file = r'%s%s%s' % (os.getcwd(), delimiter(),
re.sub('( - Copy\.txt)', '.txt',
orig_file.split(delimiter())[-1]))
self.compareOriginalAndNewFiles(orig_file, new_file)
os.remove(new_file)
comparisons_count += 1
assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (
comparisons_count, expected_no_output_files)
os.chdir(cwd)
os.remove(re.sub('(\.zip$)', '-txt.zip', example_input))
os.rmdir(re.sub('(\.zip$)', '-txt', example_input))
def test_NanoPUZZLES_inconsistency_fix_1(self):
##############################
print 'Running unittests for this project: ', project_name
print 'Running this unittest: ', self._testMethodName
##############################
##########################
#This unit test is designed to check (an aspect of) the "-N" option for this program i.e. the conversion of potentially inconsistent fields in datasets arising from older versions of the NanoPUZZLES ISA-TAB-Nano templates
#N.B. The "....TOY.test_11... - Copy.txt" files (which were checked to make sure they were not inconsistent with the expected changes to be made via the "-N" option) were just renamed and the "TOY.test_11.zip" Excel files were updated to make sure they contained (one of) the inconsistencies that the "-N" option is designed to fix, then all such files were renamed to give "TOY.test_15.zip".
#[*] In addition, given the changes to the file names, the internal contents of the files (both Excel and expected text output) needed to be edited such that "TOY.test_11" was replaced with "TOY.test_15" for consistency.
#########################
##############################
#test_15: inconsistency(ies) fix which is (are) being tested: removal of "Manufacturer supplied " prefixes for Material file "Characteristics [...]" and corresponding "Comment [...:from graph]" fields
##############################
#cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s"' % (project_modules_to_test_dir,delimiter(),example_input) #[*] To check no problems with updating names from test_11 files, this test was initially run without making any changes except replacing "TOY.test_11" with "TOY.test_15" in file names and internal contents => OK [test output scrolled through to see that expected comparisons were being made]. Also, a minor error was introduced to check that self.compareOriginalAndNewFiles(orig_file,new_file) would fail as expected for one file => OK [this error was then fixed, this command was ran again with no errors and the files were further updated for the purpose of this test].
cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s" -N' % (project_modules_to_test_dir,delimiter(),example_input)
assert 0 == os.system(cmd)
instOfzipUtils = zipUtils(delimiter_value=delimiter())
sub_folder_count = instOfzipUtils.archive2folder(re.sub('(\.zip$)','-txt_opt-N.zip',example_input))
cwd = os.getcwd()
os.chdir(re.sub('(\.zip$)','-txt_opt-N',example_input))
comparisons_count = 0 #This line would actually be useful in earlier unit tests!
for orig_file in example_output_to_compare_to:
new_file = r'%s%s%s' % (os.getcwd(),delimiter(),re.sub('( - Copy\.txt)','.txt',orig_file.split(delimiter())[-1]))
self.compareOriginalAndNewFiles(orig_file,new_file)
os.remove(new_file)
comparisons_count += 1 #This line would actually be useful in earlier unit tests!
assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (comparisons_count,expected_no_output_files) #This line would actually be useful in earlier unit tests!
os.chdir(cwd)
os.remove(re.sub('(\.zip$)','-txt_opt-N.zip',example_input))
os.rmdir(re.sub('(\.zip$)','-txt_opt-N',example_input))
def test_filesIntoFlatArchive(self):
##############################
print 'Running unittests for this project: ', project_name
print 'Running this unittest: ', self._testMethodName
##############################
from zipper import zipUtils
folder = r'%s%sInvestigation.blah-TXT' % (dir_of_the_current_test,
delimiter())
instOfzipUtils = zipUtils(delimiter_value=delimiter())
instOfzipUtils.filesIntoFlatArchive(folder_name=folder,
zip_archive=folder + ".zip")
del instOfzipUtils
os.remove(
r'%s%sInvestigation.blah-TXT.zip' %
(dir_of_the_current_test, delimiter())
) #when this was commented, archive looked as expected (flat archive containing a single text file)
def test_convertTOYtest11_defaultOptions(self):
##############################
print 'Running unittests for this project: ', project_name
print 'Running this unittest: ', self._testMethodName
##############################
##########################
#Observations regarding TOY.test_11.zip files which are relevant to testing capabilities of the code to fix xls errors:
#s_TOY.test_11_InVitro.CB.xls => "Factor Value[culture medium]" : should be converted to "Factor Value [culture medium]" in txt version
#a_TOY.test_11_PC_size_TEM.xls => "Parameter Value[Instrument Version]" : ditto "Parameter Value [Instrument Version]"
#ditto => "Measurement Value[maximum(diameter)]" ditto "Measurement Value [maximum(diameter)]"
#m_TiO2_TOY.test_11.xls -> Excel 2010 saved as tab delimited text => "Characteristics [Product impurities found {MEDDRA:\nhttp://purl.bioontology.org/ontology/MDR/10069178}]" i.e. this field contained a line ending inside a field - which should be removed in code created version!
#ditto => Characteristics[purity {NPO: http://purl.bioontology.org/ontology/npo#NPO_1345}] : should be converted to "Characteristics [purity {NPO:http://purl.bioontology.org/ontology/npo#NPO_1345}]"
#########################
cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s"' % (project_modules_to_test_dir,delimiter(),example_input)
assert 0 == os.system(cmd)
instOfzipUtils = zipUtils(delimiter_value=delimiter())
sub_folder_count = instOfzipUtils.archive2folder(re.sub('(\.zip$)','-txt.zip',example_input))
cwd = os.getcwd()
os.chdir(re.sub('(\.zip$)','-txt',example_input))
comparisons_count = 0
for orig_file in example_output_to_compare_to:
new_file = r'%s%s%s' % (os.getcwd(),delimiter(),re.sub('( - Copy\.txt)','.txt',orig_file.split(delimiter())[-1]))
self.compareOriginalAndNewFiles(orig_file,new_file)
os.remove(new_file)
comparisons_count += 1
assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (comparisons_count,expected_no_output_files)
os.chdir(cwd)
os.remove(re.sub('(\.zip$)','-txt.zip',example_input))
os.rmdir(re.sub('(\.zip$)','-txt',example_input))
def test_NanoPUZZLES_inconsistency_fix_5(self):
##############################
print 'Running unittests for this project: ', project_name
print 'Running this unittest: ', self._testMethodName
##############################
##########################
#This unit test is designed to check (an aspect of) the "-N" option for this program i.e. the conversion of potentially inconsistent fields in datasets arising from older versions of the NanoPUZZLES ISA-TAB-Nano templates
#N.B. The "....TOY.test_11... - Copy.txt" files (which were checked to make sure they were not inconsistent with the expected changes to be made via the "-N" option) were just renamed and the "TOY.test_11.zip" Excel files were updated to make sure they contained (one of) the inconsistencies that the "-N" option is designed to fix, then all such files were renamed to give "TOY.test_19.zip".
#[*] In addition, given the changes to the file names, the internal contents of the files (both Excel and expected text output) needed to be edited such that "TOY.test_11" was replaced with "TOY.test_19" for consistency.
#########################
##############################
#test_19: inconsistency(ies) fix which is (are) being tested: replace "nanoparticle sample" with "nanomaterial" and (correspondingly) "http://purl.bioontology.org/ontology/npo#NPO_1404" with "http://purl.bioontology.org/ontology/npo#NPO_199". In practice, this should only affect old biological Study files (name replacements), Material files (name replacements and ontology code replacements) and Investigation files (name replacements and ontology code replacements).
##############################
cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s" -N' % (project_modules_to_test_dir,delimiter(),example_input)
assert 0 == os.system(cmd)
instOfzipUtils = zipUtils(delimiter_value=delimiter())
sub_folder_count = instOfzipUtils.archive2folder(re.sub('(\.zip$)','-txt_opt-N.zip',example_input))
cwd = os.getcwd()
os.chdir(re.sub('(\.zip$)','-txt_opt-N',example_input))
comparisons_count = 0 #This line would actually be useful in earlier unit tests!
for orig_file in example_output_to_compare_to:
new_file = r'%s%s%s' % (os.getcwd(),delimiter(),re.sub('( - Copy\.txt)','.txt',orig_file.split(delimiter())[-1]))
self.compareOriginalAndNewFiles(orig_file,new_file)
os.remove(new_file)
comparisons_count += 1 #This line would actually be useful in earlier unit tests!
assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (comparisons_count,expected_no_output_files) #This line would actually be useful in earlier unit tests!
os.chdir(cwd)
os.remove(re.sub('(\.zip$)','-txt_opt-N.zip',example_input))
os.rmdir(re.sub('(\.zip$)','-txt_opt-N',example_input))
def test_convertTOYtest11_removeComments(self):
##############################
print 'Running unittests for this project: ', project_name
print 'Running this unittest: ', self._testMethodName
##############################
##########################
#IMPORTANT: TOY.test_11.zip JUST COPIED FROM test_11 WITHOUT EDITING; EXPECTED OUTPUT [* - Copy.txt] ALSO COPIED, BUT MANUALLY EDITED [WITHOUT CHANGING FILE NAMES!] IN ACCORDANCE WITH EXPECTED CHANGES NOW THAT "-c" FLAG IS BEING USED!
#########################
cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s" -c' % (
project_modules_to_test_dir, delimiter(), example_input)
assert 0 == os.system(cmd)
instOfzipUtils = zipUtils(delimiter_value=delimiter())
sub_folder_count = instOfzipUtils.archive2folder(
re.sub('(\.zip$)', '-txt_opt-c.zip', example_input))
cwd = os.getcwd()
os.chdir(re.sub('(\.zip$)', '-txt_opt-c', example_input))
comparisons_count = 0
for orig_file in example_output_to_compare_to:
new_file = r'%s%s%s' % (os.getcwd(), delimiter(),
re.sub('( - Copy\.txt)', '.txt',
orig_file.split(delimiter())[-1]))
self.compareOriginalAndNewFiles(orig_file, new_file)
os.remove(new_file)
comparisons_count += 1
assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (
comparisons_count, expected_no_output_files)
os.chdir(cwd)
os.remove(re.sub('(\.zip$)', '-txt_opt-c.zip', example_input))
os.rmdir(re.sub('(\.zip$)', '-txt_opt-c', example_input))
def idInputFiles(xls_folder):
instOfzipUtils = zipUtils(delimiter())
input_files = instOfzipUtils.getRelativeFileNames(xls_folder)
input_files = [r'%s%s%s' % (xls_folder,delimiter(),file) for file in input_files if fileNameRegexesDict['input_extension'].search(file)]
del file
del instOfzipUtils
#non_xls_files = [file_name for file_name in input_files if not fileNameRegexesDict['input_extension'].search(file_name)]
#del file_name
#assert 0 == len(non_xls_files),"There are %d non-xls files in the folder %s created from the input archive." % (len(non_xls_files),xls_folder)
#del non_xls_files
fileType2No = {}
for fileType in all_file_types:
fileType2No[fileType] = len([file_name for file_name in input_files if fileNameRegexesDict[fileType].match(file_name.split(delimiter())[-1])])
assert not 0 == fileType2No[fileType], "Zero %s input files in the folder created from the input archive!" % fileType
print "%d %s input files in the folder created from the input archive!" % (fileType2No[fileType],fileType)
del fileType2No
return input_files#,non_xls_files #non_xls_files should just be copied across to the final zip archive without modification -see "def createFlatTxtArchive(xls_folder):"
