def test_validate_testdata_data_transformation_isatab(self): test_case = 'TEST-ISA-data-transformation' log_msg_stream = isatab.validate2(fp=open(os.path.join(utils.TAB_DATA_DIR, test_case, 'i_investigation.txt')), config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level) if '(E)' in log_msg_stream.getvalue() or '(F)' in log_msg_stream.getvalue(): self.fail("Error found when validating ISA tab: {}".format(log_msg_stream.getvalue()))
def test_validate_isatab_bii_s_7(self): log_msg_stream = isatab.validate2(open(os.path.join(self._tab_data_dir, 'BII-S-7', 'i_matteo.txt'))) log = log_msg_stream.getvalue() if "Finished validation..." not in log: self.fail("Validation did not complete successfully when it should have!") elif '(W)' not in log: self.fail("Validation error and warnings are missing when should report some with BII-S-7")
def test_validate_testdata_bii_s_3_isatab(self): test_case = 'BII-S-3' log_msg_stream = isatab.validate2(fp=open(os.path.join(utils.TAB_DATA_DIR, test_case, 'i_gilbert.txt')), config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level) if '(E)' in log_msg_stream.getvalue() or '(F)' in log_msg_stream.getvalue(): self.fail("Error found when validating ISA tab: {}".format(log_msg_stream.getvalue()))
def test_validate_testdata_bii_i_1_isatab(self): # FIXME: Fails because of #135 test_case = 'BII-I-1' log_msg_stream = isatab.validate2(fp=open(os.path.join(utils.TAB_DATA_DIR, test_case, 'i_investigation.txt')), config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level) if '(E)' in log_msg_stream.getvalue() or '(F)' in log_msg_stream.getvalue(): self.fail("Error found when validating ISA tab: {}".format(log_msg_stream.getvalue()))
def test_validate_testdata_sra_transcriptome_seq_isatab(self): test_case = 'TEST-ISA-SRA-transcriptome-seq' log_msg_stream = isatab.validate2(fp=open(os.path.join(utils.TAB_DATA_DIR, test_case, 'i_TEST_SRA_tx_seq.txt')), config_dir=utils.SRA2016_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level) if '(E)' in log_msg_stream.getvalue() or '(F)' in log_msg_stream.getvalue(): self.fail("Error found when validating ISA tab: {}".format(log_msg_stream.getvalue()))
def test_validate_testdata_sra_env_gene_survey_isatab(self): test_case = 'TEST-ISA-SRA-env-gene-survey-seq' report = isatab.validate2( fp=open(os.path.join(utils.TAB_DATA_DIR, test_case, 'i_matteo.txt')), config_dir=utils.SRA2016_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level) if len(report['errors']) > 0: self.fail("Error found when validating ISA tab: {}".format( len(report['errors'])))
def test_validate_testdata_bii_s_7_isatab(self): test_case = 'BII-S-7' report = isatab.validate2( fp=open(os.path.join(utils.TAB_DATA_DIR, test_case, 'i_matteo.txt')), config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level) if len(report['errors']) > 0: self.fail("Error found when validating ISA tab: {}".format( report['errors']))
def test_validate_isatab_bii_s_7(self): report = isatab.validate2( open(os.path.join(self._tab_data_dir, 'BII-S-7', 'i_matteo.txt'))) if not report['validation_finished']: self.fail( "Validation did not complete successfully when it should have!" ) elif len(report['errors'] + report['warnings']) == 0: self.fail( "Validation error and warnings are missing when should report some with BII-S-7" )
def test_validate_testdata_data_transformation_isatab(self): test_case = 'TEST-ISA-data-transformation' report = isatab.validate2( fp=open( os.path.join(utils.TAB_DATA_DIR, test_case, 'i_investigation.txt')), config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level) if len(report['errors']) > 0: self.fail("Error found when validating ISA tab: {}".format( report['errors']))
def test_validate_testdata_sra_transcriptome_seq_isatab(self): test_case = 'TEST-ISA-SRA-transcriptome-seq' report = isatab.validate2( fp=open( os.path.join(utils.TAB_DATA_DIR, test_case, 'i_TEST_SRA_tx_seq.txt')), config_dir=utils.SRA2016_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level) if len(report['errors']) > 0: self.fail("Error found when validating ISA tab: {}".format( report['errors']))
def test_validate_testdata_bii_i_1_isatab( self): # FIXME: Fails because of #135 test_case = 'BII-I-1' report = isatab.validate2( fp=open( os.path.join(utils.TAB_DATA_DIR, test_case, 'i_investigation.txt')), config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR, log_level=self._reporting_level) if len(report['errors']) > 0: self.fail("Error found when validating ISA tab: {}".format( report['errors']))
def convert(work_dir, identifier_type=IdentifierType.name, validate_first=True): if validate_first: logger.info("Validating input ISA tab before conversion") i_files = [f for f in os.listdir(work_dir) if f.startswith('i_') and f.endswith('.txt')] if len(i_files) != 1: logging.fatal("Could not resolves input investigation file, please check input ISA tab directory.") return report = isatab.validate2(fp=open(os.path.join(work_dir, i_files[0])), log_level=logging.ERROR) if len(report['errors']) > 0: logging.fatal("Could not proceed with conversion as there are some fatal validation errors. Check log.") return converter = ISATab2ISAjson_v1(identifier_type) logger.info("Converting ISA-Tab to ISA JSON...") return converter.convert(work_dir)
def convert(work_dir, identifier_type=IdentifierType.name, validate_first=True): if validate_first: logger.info("Validating input ISA tab before conversion") i_files = [f for f in os.listdir(work_dir) if f.startswith("i_") and f.endswith(".txt")] if len(i_files) != 1: logging.fatal("Could not resolves input investigation file, please check input ISA tab directory.") return log_msgs = isatab.validate2(fp=open(os.path.join(work_dir, i_files[0])), log_level=logging.ERROR) if "(F)" in log_msgs.getvalue(): logging.fatal("Could not proceed with conversion as there are some fatal validation errors. Check log.") return converter = ISATab2ISAjson_v1(identifier_type) logger.info("Converting ISA-Tab to ISA JSON...") return converter.convert(work_dir)
def post(self): response = Response(status=415) if request.mimetype == "application/zip": tmp_dir = _create_temp_dir() try: if tmp_dir is None: raise IOError("Could not create temporary directory " + tmp_dir) if not request.mimetype == "application/zip": raise TypeError("Incorrect media type received. Got " + request.mimetype + ", expected application/zip") else: # Write request data to file file_path = _write_request_data(request, tmp_dir, 'isatab.zip') if file_path is None: raise IOError("Could not create temporary file " + file_path) # Setup path to configuration with zipfile.ZipFile(file_path, 'r') as z: # extract ISArchive files z.extractall(tmp_dir) i_file_list = [i.filename for i in z.filelist if 'i_' in i.filename and i.filename.endswith('.txt')] if len(i_file_list) == 1: src_file_path = os.path.normpath(os.path.join(tmp_dir, i_file_list[0])) fp = open(src_file_path) # find just the combined JSON log_msg_stream = isatab.validate2(fp) result = { "result:": log_msg_stream.getvalue() } response = jsonify(result) else: raise Exception("Could not resolve investigation file entry point") except Exception: response = Response(status=500) finally: # cleanup generated directories shutil.rmtree(tmp_dir, ignore_errors=True) return response
#!/usr/bin/env python import sys import os from isatools import isatab if __name__ == "__main__": study_dir = sys.argv[-2] config_dir = sys.argv[-1] with open(os.path.join(study_dir, 'i_Investigation.txt')) as s: status = isatab.validate2(s, config_dir) if status['errors']: sys.exit("Validation failed !\nErrors:\n{}".format('\n'.join( str(x) for x in status['errors'])))
def create_sra(source_path, dest_path, config_path=default_config_dir): """ This function converts a set of ISA-Tab files into SRA XML format. The SRA conversion uses the Java compiled validator and converter, packaged under the isa_line_commands package. Take note that this requires bash and at least a Java Runtime Environment for Java 6. Args: source_path (str): Path to the source ISA-Tab files directory. dest_path (str): Path to the destination directory where SRA XML files will be written to. Note that the converter will automatically create sub-directory /sra under where you specify the dest_path. config_path (str): Path to the ISA Configuration XML files to validate the input ISA-Tab. log_file (str): the absolute or realtive path to the log file where Raises: TypeErrpr: If something goes wrong calling the shell commands to run the Java conversion, this will raise a TypeError (NOTE: should this error be customised?) """ source_path = os.path.abspath(source_path) dest_path = os.path.abspath(dest_path) config_path = os.path.abspath(config_path) if not os.path.exists(source_path): raise IOError("source_path " + source_path + " does not exist") if not os.path.exists(dest_path): raise IOError("dest_path " + dest_path + " does not exist") if not os.path.exists(config_path): raise IOError("config_path " + config_path + " does not exist") print("Using source ISA Tab folder: " + source_path) print("Writing to destination SRA folder: " + dest_path) print("ISA configuration XML folder: " + config_path) i_files = [f for f in os.listdir(source_path) if f.startswith('i_') and f.endswith('.txt')] if len(i_files) != 1: logging.fatal("Could not resolves input investigation file, please check input ISA tab directory.") return log_msgs = isatab.validate2(fp=open(os.path.join(source_path, i_files[0])), log_level=logging.ERROR) if '(F)' in log_msgs.getvalue(): logging.fatal("Could not proceed with conversion as there are some fatal validation errors. Check log.") return convert_command = os.path.join(os.path.dirname(os.path.abspath(__file__)), "isa_line_commands/bin/convert.sh -t sra " + source_path + " " + dest_path + " " + config_path) print(convert_command) # subprocess.call(['java', '-version']) # return_code = subprocess.call([convert_command], shell=True) try: res = subprocess.check_output([convert_command], shell=True, stderr=subprocess.STDOUT) # with open(log_file, 'w') as logf: # logf.write(str(res, encoding='utf-8')) except subprocess.CalledProcessError as err: print("Execution failed: ", err.output) error_message = str(err.output, encoding='utf-8') raise TypeError(error_message) # returns the buffer containing the SRA element(s) as an archive buffer = BytesIO() sra_dir = os.path.join(dest_path, 'sra') if os.path.isdir(sra_dir): with ZipFile(buffer, 'w') as zip_file: # use relative dir_name to avoid absolute path on file names zipdir(sra_dir, zip_file) print(zip_file.namelist()) # clean up the target directory after the ZIP file has been closed # rmtree(sra_dir) buffer.seek(0) return buffer else: raise TypeError("The provided ISA tab could not be converted to SRA")
def create_sra(source_path, dest_path, config_path=default_config_dir): """ This function converts a set of ISA-Tab files into SRA XML format. The SRA conversion uses the Java compiled validator and converter, packaged under the isa_line_commands package. Take note that this requires bash and at least a Java Runtime Environment for Java 6. Args: source_path (str): Path to the source ISA-Tab files directory. dest_path (str): Path to the destination directory where SRA XML files will be written to. Note that the converter will automatically create sub-directory /sra under where you specify the dest_path. config_path (str): Path to the ISA Configuration XML files to validate the input ISA-Tab. log_file (str): the absolute or realtive path to the log file where Raises: TypeErrpr: If something goes wrong calling the shell commands to run the Java conversion, this will raise a TypeError (NOTE: should this error be customised?) """ source_path = os.path.abspath(source_path) dest_path = os.path.abspath(dest_path) config_path = os.path.abspath(config_path) if not os.path.exists(source_path): raise IOError("source_path " + source_path + " does not exist") if not os.path.exists(dest_path): raise IOError("dest_path " + dest_path + " does not exist") if not os.path.exists(config_path): raise IOError("config_path " + config_path + " does not exist") print("Using source ISA Tab folder: " + source_path) print("Writing to destination SRA folder: " + dest_path) print("ISA configuration XML folder: " + config_path) i_files = [ f for f in os.listdir(source_path) if f.startswith('i_') and f.endswith('.txt') ] if len(i_files) != 1: logging.fatal( "Could not resolves input investigation file, please check input ISA tab directory." ) return log_msgs = isatab.validate2(fp=open(os.path.join(source_path, i_files[0])), log_level=logging.ERROR) if '(F)' in log_msgs.getvalue(): logging.fatal( "Could not proceed with conversion as there are some fatal validation errors. Check log." ) return convert_command = os.path.join( os.path.dirname(os.path.abspath(__file__)), "isa_line_commands/bin/convert.sh -t sra " + source_path + " " + dest_path + " " + config_path) print(convert_command) # subprocess.call(['java', '-version']) # return_code = subprocess.call([convert_command], shell=True) try: res = subprocess.check_output([convert_command], shell=True, stderr=subprocess.STDOUT) # with open(log_file, 'w') as logf: # logf.write(str(res, encoding='utf-8')) except subprocess.CalledProcessError as err: print("Execution failed: ", err.output) error_message = str(err.output, encoding='utf-8') raise TypeError(error_message) # returns the buffer containing the SRA element(s) as an archive buffer = BytesIO() sra_dir = os.path.join(dest_path, 'sra') if os.path.isdir(sra_dir): with ZipFile(buffer, 'w') as zip_file: # use relative dir_name to avoid absolute path on file names zipdir(sra_dir, zip_file) print(zip_file.namelist()) # clean up the target directory after the ZIP file has been closed # rmtree(sra_dir) buffer.seek(0) return buffer else: raise TypeError("The provided ISA tab could not be converted to SRA")