def test_archive2folder(self): ############################## print "Running unittests for this project: ", project_name print "Running this unittest: ", self._testMethodName ############################## from zipper import zipUtils zip_archive = r"%s%sInvestigation.blah.zip" % (dir_of_the_current_test, delimiter()) instOfzipUtils = zipUtils(delimiter_value=delimiter()) sub_folder_count = instOfzipUtils.archive2folder(zip_archive) print "sub_folder_count=", sub_folder_count assert 0 == sub_folder_count cwd = os.getcwd() # os.chdir(r'%s%sdoesNotExist.blah' % (dir_of_the_current_test,delimiter())) #caused unittest to fail as expected (tried separately) os.chdir(r"%s%sInvestigation.blah" % (dir_of_the_current_test, delimiter())) os.remove("m_1.xls") # os.remove('does_not_exist.xls') #caused unittest to fail as expected (tried separately) os.chdir(dir_of_the_current_test) os.rmdir("Investigation.blah") os.chdir(cwd)
def createFlatTxtArchive(xls_folder, mustEditAccessionCodes, mustRemoveComments, mustMakeNanoPUZZLESspecificChanges): flat_txt_archive = xls_folder + "-txt.zip" ########### #Rename the output file if non-default options are used if mustEditAccessionCodes: flat_txt_archive = re.sub('(\.zip$)', '_opt-a.zip', flat_txt_archive) if mustRemoveComments: flat_txt_archive = re.sub('(\.zip$)', '_opt-c.zip', flat_txt_archive) if mustMakeNanoPUZZLESspecificChanges: flat_txt_archive = re.sub('(\.zip$)', '_opt-N.zip', flat_txt_archive) ########### cwd = os.getcwd() os.chdir(xls_folder) for xls_file in glob.glob('*.xls'): os.remove(xls_file) os.chdir(cwd) del cwd, xls_file instOfzipUtils = zipUtils(delimiter_value=delimiter()) instOfzipUtils.filesIntoFlatArchive(folder_name=xls_folder, zip_archive=flat_txt_archive) del instOfzipUtils
def test_convertTOYtest11_removeComments(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## ########################## #IMPORTANT: TOY.test_11.zip JUST COPIED FROM test_11 WITHOUT EDITING; EXPECTED OUTPUT [* - Copy.txt] ALSO COPIED, BUT MANUALLY EDITED [WITHOUT CHANGING FILE NAMES!] IN ACCORDANCE WITH EXPECTED CHANGES NOW THAT "-c" FLAG IS BEING USED! ######################### cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s" -c' % (project_modules_to_test_dir,delimiter(),example_input) assert 0 == os.system(cmd) instOfzipUtils = zipUtils(delimiter_value=delimiter()) sub_folder_count = instOfzipUtils.archive2folder(re.sub('(\.zip$)','-txt_opt-c.zip',example_input)) cwd = os.getcwd() os.chdir(re.sub('(\.zip$)','-txt_opt-c',example_input)) comparisons_count = 0 for orig_file in example_output_to_compare_to: new_file = r'%s%s%s' % (os.getcwd(),delimiter(),re.sub('( - Copy\.txt)','.txt',orig_file.split(delimiter())[-1])) self.compareOriginalAndNewFiles(orig_file,new_file) os.remove(new_file) comparisons_count += 1 assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (comparisons_count,expected_no_output_files) os.chdir(cwd) os.remove(re.sub('(\.zip$)','-txt_opt-c.zip',example_input)) os.rmdir(re.sub('(\.zip$)','-txt_opt-c',example_input))
def createFlatTxtArchive(xls_folder,mustEditAccessionCodes,mustRemoveComments,mustMakeNanoPUZZLESspecificChanges): flat_txt_archive = xls_folder+"-txt.zip" ########### #Rename the output file if non-default options are used if mustEditAccessionCodes: flat_txt_archive = re.sub('(\.zip$)','_opt-a.zip',flat_txt_archive) if mustRemoveComments: flat_txt_archive = re.sub('(\.zip$)','_opt-c.zip',flat_txt_archive) if mustMakeNanoPUZZLESspecificChanges: flat_txt_archive = re.sub('(\.zip$)','_opt-N.zip',flat_txt_archive) ########### cwd = os.getcwd() os.chdir(xls_folder) for xls_file in glob.glob('*.xls'): os.remove(xls_file) os.chdir(cwd) del cwd,xls_file instOfzipUtils = zipUtils(delimiter_value=delimiter()) instOfzipUtils.filesIntoFlatArchive(folder_name=xls_folder,zip_archive=flat_txt_archive) del instOfzipUtils
def test_archive2folder(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## from zipper import zipUtils zip_archive = r'%s%sInvestigation.blah.zip' % (dir_of_the_current_test, delimiter()) instOfzipUtils = zipUtils(delimiter_value=delimiter()) sub_folder_count = instOfzipUtils.archive2folder(zip_archive) print 'sub_folder_count=', sub_folder_count assert 0 == sub_folder_count cwd = os.getcwd() #os.chdir(r'%s%sdoesNotExist.blah' % (dir_of_the_current_test,delimiter())) #caused unittest to fail as expected (tried separately) os.chdir(r'%s%sInvestigation.blah' % (dir_of_the_current_test, delimiter())) os.remove('m_1.xls') #os.remove('does_not_exist.xls') #caused unittest to fail as expected (tried separately) os.chdir(dir_of_the_current_test) os.rmdir('Investigation.blah') os.chdir(cwd)
def idInputFiles(xls_folder): instOfzipUtils = zipUtils(delimiter()) input_files = instOfzipUtils.getRelativeFileNames(xls_folder) input_files = [ r'%s%s%s' % (xls_folder, delimiter(), file) for file in input_files if fileNameRegexesDict['input_extension'].search(file) ] del file del instOfzipUtils #non_xls_files = [file_name for file_name in input_files if not fileNameRegexesDict['input_extension'].search(file_name)] #del file_name #assert 0 == len(non_xls_files),"There are %d non-xls files in the folder %s created from the input archive." % (len(non_xls_files),xls_folder) #del non_xls_files fileType2No = {} for fileType in all_file_types: fileType2No[fileType] = len([ file_name for file_name in input_files if fileNameRegexesDict[fileType].match( file_name.split(delimiter())[-1]) ]) assert not 0 == fileType2No[ fileType], "Zero %s input files in the folder created from the input archive!" % fileType print "%d %s input files in the folder created from the input archive!" % ( fileType2No[fileType], fileType) del fileType2No return input_files #,non_xls_files #non_xls_files should just be copied across to the final zip archive without modification -see "def createFlatTxtArchive(xls_folder):"
def extractXlsFolder(xls_archive): instOfzipUtils = zipUtils(delimiter()) sub_folder_count = instOfzipUtils.archive2folder(xls_archive) assert 0 == sub_folder_count return instOfzipUtils.folder_name
def test_filesIntoFlatArchive(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## from zipper import zipUtils folder = r'%s%sInvestigation.blah-TXT' % (dir_of_the_current_test,delimiter()) instOfzipUtils = zipUtils(delimiter_value=delimiter()) instOfzipUtils.filesIntoFlatArchive(folder_name=folder,zip_archive=folder+".zip") del instOfzipUtils os.remove(r'%s%sInvestigation.blah-TXT.zip' % (dir_of_the_current_test,delimiter())) #when this was commented, archive looked as expected (flat archive containing a single text file)
def test_NanoPUZZLES_inconsistency_fix_3(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## ########################## #This unit test is designed to check (an aspect of) the "-N" option for this program i.e. the conversion of potentially inconsistent fields in datasets arising from older versions of the NanoPUZZLES ISA-TAB-Nano templates #N.B. The "....TOY.test_11... - Copy.txt" files (which were checked to make sure they were not inconsistent with the expected changes to be made via the "-N" option) were just renamed and the "TOY.test_11.zip" Excel files were updated to make sure they contained (one of) the inconsistencies that the "-N" option is designed to fix, then all such files were renamed to give "TOY.test_17.zip". #[*] In addition, given the changes to the file names, the internal contents of the files (both Excel and expected text output) needed to be edited such that "TOY.test_11" was replaced with "TOY.test_17" for consistency. ######################### ############################## #test_17: inconsistency(ies) fix which is (are) being tested: replace "Characteristics[shape]" fields with "Characteristics [shape {NPO:http://purl.bioontology.org/ontology/npo#NPO_274}]" ############################## cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s" -N' % ( project_modules_to_test_dir, delimiter(), example_input) assert 0 == os.system(cmd) instOfzipUtils = zipUtils(delimiter_value=delimiter()) sub_folder_count = instOfzipUtils.archive2folder( re.sub('(\.zip$)', '-txt_opt-N.zip', example_input)) cwd = os.getcwd() os.chdir(re.sub('(\.zip$)', '-txt_opt-N', example_input)) comparisons_count = 0 #This line would actually be useful in earlier unit tests! for orig_file in example_output_to_compare_to: new_file = r'%s%s%s' % (os.getcwd(), delimiter(), re.sub('( - Copy\.txt)', '.txt', orig_file.split(delimiter())[-1])) self.compareOriginalAndNewFiles(orig_file, new_file) os.remove(new_file) comparisons_count += 1 #This line would actually be useful in earlier unit tests! assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % ( comparisons_count, expected_no_output_files ) #This line would actually be useful in earlier unit tests! os.chdir(cwd) os.remove(re.sub('(\.zip$)', '-txt_opt-N.zip', example_input)) os.rmdir(re.sub('(\.zip$)', '-txt_opt-N', example_input))
def test_convertTOYtest11_defaultOptions(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## ########################## #Observations regarding TOY.test_11.zip files which are relevant to testing capabilities of the code to fix xls errors: #s_TOY.test_11_InVitro.CB.xls => "Factor Value[culture medium]" : should be converted to "Factor Value [culture medium]" in txt version #a_TOY.test_11_PC_size_TEM.xls => "Parameter Value[Instrument Version]" : ditto "Parameter Value [Instrument Version]" #ditto => "Measurement Value[maximum(diameter)]" ditto "Measurement Value [maximum(diameter)]" #m_TiO2_TOY.test_11.xls -> Excel 2010 saved as tab delimited text => "Characteristics [Product impurities found {MEDDRA:\nhttp://purl.bioontology.org/ontology/MDR/10069178}]" i.e. this field contained a line ending inside a field - which should be removed in code created version! #ditto => Characteristics[purity {NPO: http://purl.bioontology.org/ontology/npo#NPO_1345}] : should be converted to "Characteristics [purity {NPO:http://purl.bioontology.org/ontology/npo#NPO_1345}]" ######################### cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s"' % ( project_modules_to_test_dir, delimiter(), example_input) assert 0 == os.system(cmd) instOfzipUtils = zipUtils(delimiter_value=delimiter()) sub_folder_count = instOfzipUtils.archive2folder( re.sub('(\.zip$)', '-txt.zip', example_input)) cwd = os.getcwd() os.chdir(re.sub('(\.zip$)', '-txt', example_input)) comparisons_count = 0 for orig_file in example_output_to_compare_to: new_file = r'%s%s%s' % (os.getcwd(), delimiter(), re.sub('( - Copy\.txt)', '.txt', orig_file.split(delimiter())[-1])) self.compareOriginalAndNewFiles(orig_file, new_file) os.remove(new_file) comparisons_count += 1 assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % ( comparisons_count, expected_no_output_files) os.chdir(cwd) os.remove(re.sub('(\.zip$)', '-txt.zip', example_input)) os.rmdir(re.sub('(\.zip$)', '-txt', example_input))
def test_NanoPUZZLES_inconsistency_fix_1(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## ########################## #This unit test is designed to check (an aspect of) the "-N" option for this program i.e. the conversion of potentially inconsistent fields in datasets arising from older versions of the NanoPUZZLES ISA-TAB-Nano templates #N.B. The "....TOY.test_11... - Copy.txt" files (which were checked to make sure they were not inconsistent with the expected changes to be made via the "-N" option) were just renamed and the "TOY.test_11.zip" Excel files were updated to make sure they contained (one of) the inconsistencies that the "-N" option is designed to fix, then all such files were renamed to give "TOY.test_15.zip". #[*] In addition, given the changes to the file names, the internal contents of the files (both Excel and expected text output) needed to be edited such that "TOY.test_11" was replaced with "TOY.test_15" for consistency. ######################### ############################## #test_15: inconsistency(ies) fix which is (are) being tested: removal of "Manufacturer supplied " prefixes for Material file "Characteristics [...]" and corresponding "Comment [...:from graph]" fields ############################## #cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s"' % (project_modules_to_test_dir,delimiter(),example_input) #[*] To check no problems with updating names from test_11 files, this test was initially run without making any changes except replacing "TOY.test_11" with "TOY.test_15" in file names and internal contents => OK [test output scrolled through to see that expected comparisons were being made]. Also, a minor error was introduced to check that self.compareOriginalAndNewFiles(orig_file,new_file) would fail as expected for one file => OK [this error was then fixed, this command was ran again with no errors and the files were further updated for the purpose of this test]. cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s" -N' % (project_modules_to_test_dir,delimiter(),example_input) assert 0 == os.system(cmd) instOfzipUtils = zipUtils(delimiter_value=delimiter()) sub_folder_count = instOfzipUtils.archive2folder(re.sub('(\.zip$)','-txt_opt-N.zip',example_input)) cwd = os.getcwd() os.chdir(re.sub('(\.zip$)','-txt_opt-N',example_input)) comparisons_count = 0 #This line would actually be useful in earlier unit tests! for orig_file in example_output_to_compare_to: new_file = r'%s%s%s' % (os.getcwd(),delimiter(),re.sub('( - Copy\.txt)','.txt',orig_file.split(delimiter())[-1])) self.compareOriginalAndNewFiles(orig_file,new_file) os.remove(new_file) comparisons_count += 1 #This line would actually be useful in earlier unit tests! assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (comparisons_count,expected_no_output_files) #This line would actually be useful in earlier unit tests! os.chdir(cwd) os.remove(re.sub('(\.zip$)','-txt_opt-N.zip',example_input)) os.rmdir(re.sub('(\.zip$)','-txt_opt-N',example_input))
def test_filesIntoFlatArchive(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## from zipper import zipUtils folder = r'%s%sInvestigation.blah-TXT' % (dir_of_the_current_test, delimiter()) instOfzipUtils = zipUtils(delimiter_value=delimiter()) instOfzipUtils.filesIntoFlatArchive(folder_name=folder, zip_archive=folder + ".zip") del instOfzipUtils os.remove( r'%s%sInvestigation.blah-TXT.zip' % (dir_of_the_current_test, delimiter()) ) #when this was commented, archive looked as expected (flat archive containing a single text file)
def test_convertTOYtest11_defaultOptions(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## ########################## #Observations regarding TOY.test_11.zip files which are relevant to testing capabilities of the code to fix xls errors: #s_TOY.test_11_InVitro.CB.xls => "Factor Value[culture medium]" : should be converted to "Factor Value [culture medium]" in txt version #a_TOY.test_11_PC_size_TEM.xls => "Parameter Value[Instrument Version]" : ditto "Parameter Value [Instrument Version]" #ditto => "Measurement Value[maximum(diameter)]" ditto "Measurement Value [maximum(diameter)]" #m_TiO2_TOY.test_11.xls -> Excel 2010 saved as tab delimited text => "Characteristics [Product impurities found {MEDDRA:\nhttp://purl.bioontology.org/ontology/MDR/10069178}]" i.e. this field contained a line ending inside a field - which should be removed in code created version! #ditto => Characteristics[purity {NPO: http://purl.bioontology.org/ontology/npo#NPO_1345}] : should be converted to "Characteristics [purity {NPO:http://purl.bioontology.org/ontology/npo#NPO_1345}]" ######################### cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s"' % (project_modules_to_test_dir,delimiter(),example_input) assert 0 == os.system(cmd) instOfzipUtils = zipUtils(delimiter_value=delimiter()) sub_folder_count = instOfzipUtils.archive2folder(re.sub('(\.zip$)','-txt.zip',example_input)) cwd = os.getcwd() os.chdir(re.sub('(\.zip$)','-txt',example_input)) comparisons_count = 0 for orig_file in example_output_to_compare_to: new_file = r'%s%s%s' % (os.getcwd(),delimiter(),re.sub('( - Copy\.txt)','.txt',orig_file.split(delimiter())[-1])) self.compareOriginalAndNewFiles(orig_file,new_file) os.remove(new_file) comparisons_count += 1 assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (comparisons_count,expected_no_output_files) os.chdir(cwd) os.remove(re.sub('(\.zip$)','-txt.zip',example_input)) os.rmdir(re.sub('(\.zip$)','-txt',example_input))
def test_NanoPUZZLES_inconsistency_fix_5(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## ########################## #This unit test is designed to check (an aspect of) the "-N" option for this program i.e. the conversion of potentially inconsistent fields in datasets arising from older versions of the NanoPUZZLES ISA-TAB-Nano templates #N.B. The "....TOY.test_11... - Copy.txt" files (which were checked to make sure they were not inconsistent with the expected changes to be made via the "-N" option) were just renamed and the "TOY.test_11.zip" Excel files were updated to make sure they contained (one of) the inconsistencies that the "-N" option is designed to fix, then all such files were renamed to give "TOY.test_19.zip". #[*] In addition, given the changes to the file names, the internal contents of the files (both Excel and expected text output) needed to be edited such that "TOY.test_11" was replaced with "TOY.test_19" for consistency. ######################### ############################## #test_19: inconsistency(ies) fix which is (are) being tested: replace "nanoparticle sample" with "nanomaterial" and (correspondingly) "http://purl.bioontology.org/ontology/npo#NPO_1404" with "http://purl.bioontology.org/ontology/npo#NPO_199". In practice, this should only affect old biological Study files (name replacements), Material files (name replacements and ontology code replacements) and Investigation files (name replacements and ontology code replacements). ############################## cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s" -N' % (project_modules_to_test_dir,delimiter(),example_input) assert 0 == os.system(cmd) instOfzipUtils = zipUtils(delimiter_value=delimiter()) sub_folder_count = instOfzipUtils.archive2folder(re.sub('(\.zip$)','-txt_opt-N.zip',example_input)) cwd = os.getcwd() os.chdir(re.sub('(\.zip$)','-txt_opt-N',example_input)) comparisons_count = 0 #This line would actually be useful in earlier unit tests! for orig_file in example_output_to_compare_to: new_file = r'%s%s%s' % (os.getcwd(),delimiter(),re.sub('( - Copy\.txt)','.txt',orig_file.split(delimiter())[-1])) self.compareOriginalAndNewFiles(orig_file,new_file) os.remove(new_file) comparisons_count += 1 #This line would actually be useful in earlier unit tests! assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % (comparisons_count,expected_no_output_files) #This line would actually be useful in earlier unit tests! os.chdir(cwd) os.remove(re.sub('(\.zip$)','-txt_opt-N.zip',example_input)) os.rmdir(re.sub('(\.zip$)','-txt_opt-N',example_input))
def test_convertTOYtest11_removeComments(self): ############################## print 'Running unittests for this project: ', project_name print 'Running this unittest: ', self._testMethodName ############################## ########################## #IMPORTANT: TOY.test_11.zip JUST COPIED FROM test_11 WITHOUT EDITING; EXPECTED OUTPUT [* - Copy.txt] ALSO COPIED, BUT MANUALLY EDITED [WITHOUT CHANGING FILE NAMES!] IN ACCORDANCE WITH EXPECTED CHANGES NOW THAT "-c" FLAG IS BEING USED! ######################### cmd = r'python "%s%sxls2txtISA.NANO.archive.py" -i "%s" -c' % ( project_modules_to_test_dir, delimiter(), example_input) assert 0 == os.system(cmd) instOfzipUtils = zipUtils(delimiter_value=delimiter()) sub_folder_count = instOfzipUtils.archive2folder( re.sub('(\.zip$)', '-txt_opt-c.zip', example_input)) cwd = os.getcwd() os.chdir(re.sub('(\.zip$)', '-txt_opt-c', example_input)) comparisons_count = 0 for orig_file in example_output_to_compare_to: new_file = r'%s%s%s' % (os.getcwd(), delimiter(), re.sub('( - Copy\.txt)', '.txt', orig_file.split(delimiter())[-1])) self.compareOriginalAndNewFiles(orig_file, new_file) os.remove(new_file) comparisons_count += 1 assert expected_no_output_files == comparisons_count, "comparisons_count=%d;expected_no_output_files=%d" % ( comparisons_count, expected_no_output_files) os.chdir(cwd) os.remove(re.sub('(\.zip$)', '-txt_opt-c.zip', example_input)) os.rmdir(re.sub('(\.zip$)', '-txt_opt-c', example_input))
def idInputFiles(xls_folder): instOfzipUtils = zipUtils(delimiter()) input_files = instOfzipUtils.getRelativeFileNames(xls_folder) input_files = [r'%s%s%s' % (xls_folder,delimiter(),file) for file in input_files if fileNameRegexesDict['input_extension'].search(file)] del file del instOfzipUtils #non_xls_files = [file_name for file_name in input_files if not fileNameRegexesDict['input_extension'].search(file_name)] #del file_name #assert 0 == len(non_xls_files),"There are %d non-xls files in the folder %s created from the input archive." % (len(non_xls_files),xls_folder) #del non_xls_files fileType2No = {} for fileType in all_file_types: fileType2No[fileType] = len([file_name for file_name in input_files if fileNameRegexesDict[fileType].match(file_name.split(delimiter())[-1])]) assert not 0 == fileType2No[fileType], "Zero %s input files in the folder created from the input archive!" % fileType print "%d %s input files in the folder created from the input archive!" % (fileType2No[fileType],fileType) del fileType2No return input_files#,non_xls_files #non_xls_files should just be copied across to the final zip archive without modification -see "def createFlatTxtArchive(xls_folder):"