def test_validate_testdata_data_transformation_isatab(self):
test_case = 'TEST-ISA-data-transformation'
log_msg_stream = isatab.validate2(fp=open(os.path.join(utils.TAB_DATA_DIR, test_case, 'i_investigation.txt')),
config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR,
log_level=self._reporting_level)
if '(E)' in log_msg_stream.getvalue() or '(F)' in log_msg_stream.getvalue():
self.fail("Error found when validating ISA tab: {}".format(log_msg_stream.getvalue()))
def test_validate_isatab_bii_s_7(self):
log_msg_stream = isatab.validate2(open(os.path.join(self._tab_data_dir, 'BII-S-7', 'i_matteo.txt')))
log = log_msg_stream.getvalue()
if "Finished validation..." not in log:
self.fail("Validation did not complete successfully when it should have!")
elif '(W)' not in log:
self.fail("Validation error and warnings are missing when should report some with BII-S-7")
def test_validate_testdata_bii_s_3_isatab(self):
test_case = 'BII-S-3'
log_msg_stream = isatab.validate2(fp=open(os.path.join(utils.TAB_DATA_DIR, test_case,
'i_gilbert.txt')),
config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR,
log_level=self._reporting_level)
if '(E)' in log_msg_stream.getvalue() or '(F)' in log_msg_stream.getvalue():
self.fail("Error found when validating ISA tab: {}".format(log_msg_stream.getvalue()))
def test_validate_testdata_bii_i_1_isatab(self): # FIXME: Fails because of #135
test_case = 'BII-I-1'
log_msg_stream = isatab.validate2(fp=open(os.path.join(utils.TAB_DATA_DIR, test_case,
'i_investigation.txt')),
config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR,
log_level=self._reporting_level)
if '(E)' in log_msg_stream.getvalue() or '(F)' in log_msg_stream.getvalue():
self.fail("Error found when validating ISA tab: {}".format(log_msg_stream.getvalue()))
def test_validate_testdata_sra_transcriptome_seq_isatab(self):
test_case = 'TEST-ISA-SRA-transcriptome-seq'
log_msg_stream = isatab.validate2(fp=open(os.path.join(utils.TAB_DATA_DIR, test_case,
'i_TEST_SRA_tx_seq.txt')),
config_dir=utils.SRA2016_XML_CONFIGS_DATA_DIR,
log_level=self._reporting_level)
if '(E)' in log_msg_stream.getvalue() or '(F)' in log_msg_stream.getvalue():
self.fail("Error found when validating ISA tab: {}".format(log_msg_stream.getvalue()))
def test_validate_testdata_sra_env_gene_survey_isatab(self):
test_case = 'TEST-ISA-SRA-env-gene-survey-seq'
report = isatab.validate2(
fp=open(os.path.join(utils.TAB_DATA_DIR, test_case,
'i_matteo.txt')),
config_dir=utils.SRA2016_XML_CONFIGS_DATA_DIR,
log_level=self._reporting_level)
if len(report['errors']) > 0:
self.fail("Error found when validating ISA tab: {}".format(
len(report['errors'])))
def test_validate_testdata_bii_s_7_isatab(self):
test_case = 'BII-S-7'
report = isatab.validate2(
fp=open(os.path.join(utils.TAB_DATA_DIR, test_case,
'i_matteo.txt')),
config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR,
log_level=self._reporting_level)
if len(report['errors']) > 0:
self.fail("Error found when validating ISA tab: {}".format(
report['errors']))
def test_validate_isatab_bii_s_7(self):
report = isatab.validate2(
open(os.path.join(self._tab_data_dir, 'BII-S-7', 'i_matteo.txt')))
if not report['validation_finished']:
self.fail(
"Validation did not complete successfully when it should have!"
)
elif len(report['errors'] + report['warnings']) == 0:
self.fail(
"Validation error and warnings are missing when should report some with BII-S-7"
)
def test_validate_testdata_data_transformation_isatab(self):
test_case = 'TEST-ISA-data-transformation'
report = isatab.validate2(
fp=open(
os.path.join(utils.TAB_DATA_DIR, test_case,
'i_investigation.txt')),
config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR,
log_level=self._reporting_level)
if len(report['errors']) > 0:
self.fail("Error found when validating ISA tab: {}".format(
report['errors']))
def test_validate_testdata_sra_transcriptome_seq_isatab(self):
test_case = 'TEST-ISA-SRA-transcriptome-seq'
report = isatab.validate2(
fp=open(
os.path.join(utils.TAB_DATA_DIR, test_case,
'i_TEST_SRA_tx_seq.txt')),
config_dir=utils.SRA2016_XML_CONFIGS_DATA_DIR,
log_level=self._reporting_level)
if len(report['errors']) > 0:
self.fail("Error found when validating ISA tab: {}".format(
report['errors']))
def test_validate_testdata_bii_i_1_isatab(
self): # FIXME: Fails because of #135
test_case = 'BII-I-1'
report = isatab.validate2(
fp=open(
os.path.join(utils.TAB_DATA_DIR, test_case,
'i_investigation.txt')),
config_dir=utils.DEFAULT2015_XML_CONFIGS_DATA_DIR,
log_level=self._reporting_level)
if len(report['errors']) > 0:
self.fail("Error found when validating ISA tab: {}".format(
report['errors']))
def convert(work_dir, identifier_type=IdentifierType.name, validate_first=True):
if validate_first:
logger.info("Validating input ISA tab before conversion")
i_files = [f for f in os.listdir(work_dir) if f.startswith('i_') and f.endswith('.txt')]
if len(i_files) != 1:
logging.fatal("Could not resolves input investigation file, please check input ISA tab directory.")
return
report = isatab.validate2(fp=open(os.path.join(work_dir, i_files[0])), log_level=logging.ERROR)
if len(report['errors']) > 0:
logging.fatal("Could not proceed with conversion as there are some fatal validation errors. Check log.")
return
converter = ISATab2ISAjson_v1(identifier_type)
logger.info("Converting ISA-Tab to ISA JSON...")
return converter.convert(work_dir)
def convert(work_dir, identifier_type=IdentifierType.name, validate_first=True):
if validate_first:
logger.info("Validating input ISA tab before conversion")
i_files = [f for f in os.listdir(work_dir) if f.startswith("i_") and f.endswith(".txt")]
if len(i_files) != 1:
logging.fatal("Could not resolves input investigation file, please check input ISA tab directory.")
return
log_msgs = isatab.validate2(fp=open(os.path.join(work_dir, i_files[0])), log_level=logging.ERROR)
if "(F)" in log_msgs.getvalue():
logging.fatal("Could not proceed with conversion as there are some fatal validation errors. Check log.")
return
converter = ISATab2ISAjson_v1(identifier_type)
logger.info("Converting ISA-Tab to ISA JSON...")
return converter.convert(work_dir)
def post(self):
response = Response(status=415)
if request.mimetype == "application/zip":
tmp_dir = _create_temp_dir()
try:
if tmp_dir is None:
raise IOError("Could not create temporary directory " + tmp_dir)
if not request.mimetype == "application/zip":
raise TypeError("Incorrect media type received. Got " + request.mimetype +
", expected application/zip")
else:
# Write request data to file
file_path = _write_request_data(request, tmp_dir, 'isatab.zip')
if file_path is None:
raise IOError("Could not create temporary file " + file_path)
# Setup path to configuration
with zipfile.ZipFile(file_path, 'r') as z:
# extract ISArchive files
z.extractall(tmp_dir)
i_file_list = [i.filename for i in z.filelist if 'i_' in i.filename and i.filename.endswith('.txt')]
if len(i_file_list) == 1:
src_file_path = os.path.normpath(os.path.join(tmp_dir, i_file_list[0]))
fp = open(src_file_path)
# find just the combined JSON
log_msg_stream = isatab.validate2(fp)
result = {
"result:": log_msg_stream.getvalue()
}
response = jsonify(result)
else:
raise Exception("Could not resolve investigation file entry point")
except Exception:
response = Response(status=500)
finally:
# cleanup generated directories
shutil.rmtree(tmp_dir, ignore_errors=True)
return response
#!/usr/bin/env python
import sys
import os
from isatools import isatab
if __name__ == "__main__":
study_dir = sys.argv[-2]
config_dir = sys.argv[-1]
with open(os.path.join(study_dir, 'i_Investigation.txt')) as s:
status = isatab.validate2(s, config_dir)
if status['errors']:
sys.exit("Validation failed !\nErrors:\n{}".format('\n'.join(
str(x) for x in status['errors'])))
def create_sra(source_path, dest_path, config_path=default_config_dir):
""" This function converts a set of ISA-Tab files into SRA XML format.
The SRA conversion uses the Java compiled validator and converter, packaged
under the isa_line_commands package. Take note that this requires bash and
at least a Java Runtime Environment for Java 6.
Args:
source_path (str): Path to the source ISA-Tab files directory.
dest_path (str): Path to the destination directory where SRA XML files
will be written to. Note that the converter will automatically create
sub-directory /sra under where you specify the dest_path.
config_path (str): Path to the ISA Configuration XML files to validate
the input ISA-Tab.
log_file (str): the absolute or realtive path to the log file where
Raises:
TypeErrpr: If something goes wrong calling the shell commands to run the
Java conversion, this will raise a TypeError (NOTE: should this error be customised?)
"""
source_path = os.path.abspath(source_path)
dest_path = os.path.abspath(dest_path)
config_path = os.path.abspath(config_path)
if not os.path.exists(source_path):
raise IOError("source_path " + source_path + " does not exist")
if not os.path.exists(dest_path):
raise IOError("dest_path " + dest_path + " does not exist")
if not os.path.exists(config_path):
raise IOError("config_path " + config_path + " does not exist")
print("Using source ISA Tab folder: " + source_path)
print("Writing to destination SRA folder: " + dest_path)
print("ISA configuration XML folder: " + config_path)
i_files = [f for f in os.listdir(source_path) if f.startswith('i_') and f.endswith('.txt')]
if len(i_files) != 1:
logging.fatal("Could not resolves input investigation file, please check input ISA tab directory.")
return
log_msgs = isatab.validate2(fp=open(os.path.join(source_path, i_files[0])), log_level=logging.ERROR)
if '(F)' in log_msgs.getvalue():
logging.fatal("Could not proceed with conversion as there are some fatal validation errors. Check log.")
return
convert_command = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"isa_line_commands/bin/convert.sh -t sra " +
source_path + " " +
dest_path + " " +
config_path)
print(convert_command)
# subprocess.call(['java', '-version'])
# return_code = subprocess.call([convert_command], shell=True)
try:
res = subprocess.check_output([convert_command], shell=True, stderr=subprocess.STDOUT)
# with open(log_file, 'w') as logf:
# logf.write(str(res, encoding='utf-8'))
except subprocess.CalledProcessError as err:
print("Execution failed: ", err.output)
error_message = str(err.output, encoding='utf-8')
raise TypeError(error_message)
# returns the buffer containing the SRA element(s) as an archive
buffer = BytesIO()
sra_dir = os.path.join(dest_path, 'sra')
if os.path.isdir(sra_dir):
with ZipFile(buffer, 'w') as zip_file:
# use relative dir_name to avoid absolute path on file names
zipdir(sra_dir, zip_file)
print(zip_file.namelist())
# clean up the target directory after the ZIP file has been closed
# rmtree(sra_dir)
buffer.seek(0)
return buffer
else:
raise TypeError("The provided ISA tab could not be converted to SRA")
def create_sra(source_path, dest_path, config_path=default_config_dir):
""" This function converts a set of ISA-Tab files into SRA XML format.
The SRA conversion uses the Java compiled validator and converter, packaged
under the isa_line_commands package. Take note that this requires bash and
at least a Java Runtime Environment for Java 6.
Args:
source_path (str): Path to the source ISA-Tab files directory.
dest_path (str): Path to the destination directory where SRA XML files
will be written to. Note that the converter will automatically create
sub-directory /sra under where you specify the dest_path.
config_path (str): Path to the ISA Configuration XML files to validate
the input ISA-Tab.
log_file (str): the absolute or realtive path to the log file where
Raises:
TypeErrpr: If something goes wrong calling the shell commands to run the
Java conversion, this will raise a TypeError (NOTE: should this error be customised?)
"""
source_path = os.path.abspath(source_path)
dest_path = os.path.abspath(dest_path)
config_path = os.path.abspath(config_path)
if not os.path.exists(source_path):
raise IOError("source_path " + source_path + " does not exist")
if not os.path.exists(dest_path):
raise IOError("dest_path " + dest_path + " does not exist")
if not os.path.exists(config_path):
raise IOError("config_path " + config_path + " does not exist")
print("Using source ISA Tab folder: " + source_path)
print("Writing to destination SRA folder: " + dest_path)
print("ISA configuration XML folder: " + config_path)
i_files = [
f for f in os.listdir(source_path)
if f.startswith('i_') and f.endswith('.txt')
]
if len(i_files) != 1:
logging.fatal(
"Could not resolves input investigation file, please check input ISA tab directory."
)
return
log_msgs = isatab.validate2(fp=open(os.path.join(source_path, i_files[0])),
log_level=logging.ERROR)
if '(F)' in log_msgs.getvalue():
logging.fatal(
"Could not proceed with conversion as there are some fatal validation errors. Check log."
)
return
convert_command = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"isa_line_commands/bin/convert.sh -t sra " + source_path + " " +
dest_path + " " + config_path)
print(convert_command)
# subprocess.call(['java', '-version'])
# return_code = subprocess.call([convert_command], shell=True)
try:
res = subprocess.check_output([convert_command],
shell=True,
stderr=subprocess.STDOUT)
# with open(log_file, 'w') as logf:
# logf.write(str(res, encoding='utf-8'))
except subprocess.CalledProcessError as err:
print("Execution failed: ", err.output)
error_message = str(err.output, encoding='utf-8')
raise TypeError(error_message)
# returns the buffer containing the SRA element(s) as an archive
buffer = BytesIO()
sra_dir = os.path.join(dest_path, 'sra')
if os.path.isdir(sra_dir):
with ZipFile(buffer, 'w') as zip_file:
# use relative dir_name to avoid absolute path on file names
zipdir(sra_dir, zip_file)
print(zip_file.namelist())
# clean up the target directory after the ZIP file has been closed
# rmtree(sra_dir)
buffer.seek(0)
return buffer
else:
raise TypeError("The provided ISA tab could not be converted to SRA")
