def make_stagebox(log, info):
dirname = ""
if 'SPACE' in info:
dirname += info['SPACE'] + "+"
if 'PROJECT' in info:
dirname += info['PROJECT'] + "+"
dirname += get_experiment_code(info)
dirname = os.path.join(info[Keys.WORKDIR], dirname)
log.debug("stagebox is " + dirname)
dirs.makedirs_clean(dirname)
return dirname
def make_stagebox(log, info):
dirname = ""
if 'SPACE' in info:
dirname += info['SPACE'] + "+"
if 'PROJECT' in info:
dirname += info['PROJECT'] + "+"
dirname += get_experiment_code(info)
dirname = os.path.join(info[Keys.WORKDIR], dirname)
log.debug("stagebox is " + dirname)
dirs.makedirs_clean(dirname)
return dirname
def run(self, log, info):
info['WORKFLOW'] = dropbox.extendWorkflowID(info.get('WORKFLOW',"wff"))
dia_umpire_version = subprocess.check_output("dia-umpire-version.sh").strip()
info['WORKFLOW'] += " dia-umpire@" + dia_umpire_version
workdir = info[Keys.WORKDIR]
sub_job = int(info["SUBJOBLIST"][0].split("|")[1])
sample = info['SAMPLE']
job_id = info['JOB_ID']
for i, q_file, md5 in zip(itertools.count(), info["Q_FILES"], info["MD5_SUMS"]):
# info['DROPBOXSTAGE'] = stagebox = dropbox.make_stagebox(log, info)
stagebox = os.path.join(workdir, "dia_umpire_%s_%d_q_%s" % (job_id, sub_job, i + 1))
dirs.makedirs_clean(stagebox)
log.info("made stagebox at %s" % stagebox)
shutil.copy(q_file, stagebox)
param_file = info["USED_PARAMETER_FILE"]
shutil.copy(param_file, stagebox)
ds_attrs = {}
ds_attrs['SPACE'] = 'MS_DATA'
ds_attrs['SAMPLE'] = sample
ds_attrs['DATASET_TYPE'] = "MZXML_DATA"
ds_attrs['FILE_TYPE'] = "MZXML"
ds_attrs['PARENT_DATASETS'] = info['DATASET_CODE']
path = os.path.join(stagebox, 'dataset.attributes')
IniInfoHandler().write(ds_attrs, path)
log.info("wrote %s" % path)
ds_props = {}
ds_props['FILE_SIZE'] = os.stat(q_file).st_size
ds_props['CONVERSION_SOFTWARE'] = "dia umpire %s" % dia_umpire_version
ds_props['CENTROIDED'] = True
ds_props['DATA_SET_TYPE'] = "MZXML_DATA"
ds_props['FILE_NAME'] = os.path.basename(q_file)
ds_props['MD5_CHECKSUM'] = md5
ds_props['PARENT_TYPE'] = "MZXML_DATA"
ds_props['FILE_TYPE'] = "MZXML"
path = os.path.join(stagebox, 'dataset.properties')
IniInfoHandler().write(ds_props, path)
log.info("wrote %s" % path)
dropbox.move_stage_to_dropbox(log, stagebox, info['DROPBOX'], keepCopy=True)
return info
def run(self, log, info):
info['WORKFLOW'] = dropbox.extendWorkflowID(info['WORKFLOW'])
info['WORKFLOW'] += " msproteomicstools@" + subprocess.check_output(
"git --git-dir=/cluster/apps/imsbtools/stable/msproteomicstools/master/.git rev-parse --short HEAD", shell=True).strip()
info['WORKFLOW'] += " openms@" + subprocess.check_output(
"git --git-dir=/cluster/apps/openms/2.0.0-memory/OpenMS/.git rev-parse --short HEAD", shell=True).strip()
info['DROPBOXSTAGE'] = stagebox = dropbox.make_stagebox(log, info)
expcode = dropbox.get_experiment_code(info)
#copy and compress align.csv, but not the matrix REQUEST: expcode in name
tgt = os.path.join(stagebox, expcode + '_' + os.path.basename(info['ALIGNMENT_TSV']))
shutil.copy(info['ALIGNMENT_TSV'], tgt)
subprocess.check_call('gzip ' + stagebox + '/* 2>&1', shell=True)
tgt = os.path.join(stagebox, expcode + '_' + os.path.basename(info['ALIGNMENT_MATRIX']))
shutil.copy(info['ALIGNMENT_MATRIX'], tgt)
dropbox.keys_to_dropbox(log, info, ['ALIGNER_STDOUT'], stagebox)
#compress all mprophet files into one zip
archive = os.path.join(stagebox, 'pyprophet_stats.zip')
if not 'MPROPHET_STATS' in info:
info['MPROPHET_STATS'] = []
if not isinstance(info['MPROPHET_STATS'], list):
info['MPROPHET_STATS'] = [info['MPROPHET_STATS']]
for entry in info['MPROPHET_STATS']:
subprocess.check_call('zip -j ' + archive + ' ' + entry + " 2>&1", shell=True)
#PATCH: reimport old classifier if existing was used
if 'MPR_LDA_PATH' in info and info['MPR_LDA_PATH'] != "":
subprocess.check_call('zip -j ' + archive + ' ' + info['MPR_LDA_PATH'] + " 2>&1", shell=True)
if 'MPR_WT_PATH' in info and info['MPR_WT_PATH'] != "":
subprocess.check_call('zip -j ' + archive + ' ' + info['MPR_WT_PATH'] + " 2>&1", shell=True)
#SPACE PROJECT given
dsinfo = {}
dsinfo['SPACE'] = info['SPACE']
dsinfo['PROJECT'] = info['PROJECT']
dsinfo['PARENT_DATASETS'] = info[Keys.DATASET_CODE]
if info.get("DB_SOURCE", "") == "PersonalDB":
if isinstance(dsinfo['PARENT_DATASETS'], list):
dsinfo['PARENT_DATASETS'].append(info["DBASE"])
else:
dsinfo['PARENT_DATASETS'] = [dsinfo['PARENT_DATASETS'], info['DBASE']]
dsinfo['DATASET_TYPE'] = 'SWATH_RESULT'
dsinfo['EXPERIMENT_TYPE'] = 'SWATH_SEARCH'
dsinfo['EXPERIMENT'] = expcode
path = os.path.join(stagebox, 'dataset.attributes')
IniInfoHandler().write(dsinfo, path)
expinfo = {}
expinfo['PARENT-DATA-SET-CODES'] = dsinfo['PARENT_DATASETS']
expinfo['ALIGNER_DSCORE_CUTOFF'] = info['MPR_DSCORE_CUTOFF']
for key in ['WORKFLOW', 'COMMENT',
'TRAML', 'EXTRACTION_WINDOW', 'WINDOW_UNIT', 'RT_EXTRACTION_WINDOW',
'MIN_UPPER_EDGE_DIST', 'IRTTRAML', 'MIN_RSQ', 'MIN_COVERAGE',
'MPR_NUM_XVAL','MPR_LDA_PATH', 'MPR_WT_PATH', 'MPR_FRACT', 'MPR_SSL_IF', 'MPR_SSL_IL', 'MPR_SSL_TF',
'MPR_SSL_TL', 'MPR_SSL_NI', 'MPR_MAINVAR', 'MPR_VARS',
'ALIGNER_FRACSELECTED', 'ALIGNER_MAX_RT_DIFF', 'ALIGNER_METHOD', 'ALIGNER_FDR',
'ALIGNER_MAX_FDRQUAL', 'ALIGNER_TARGETFDR', 'ALIGNER_REALIGN_METHOD', 'DO_CHROMML_REQUANT',
'ALGNER_MST_USERTCORR','ALIGNER_MST_STDEVMULT',
'ISOTOPIC_GROUPING', 'REQUANT_METHOD',
]:
if info.get(key, "") != "":
expinfo[key] = info[key]
path = os.path.join(stagebox, 'experiment.properties')
IniInfoHandler().write(expinfo, path)
#put a copy of the whole ini into the dropbox. copy() to prevent OUTPUT being removed from main ini
IniInfoHandler().write(info.copy(), os.path.join(stagebox, 'input.ini'))
mailtext = ""
if info.get("RUNSWATH2VIEWER","") == "true":
try:
destdir = "/IMSB/ra/%s/html/tapir/%s" % (getpass.getuser(), dsinfo['EXPERIMENT'])
makedirs_clean(destdir)
shutil.copy(info['ALIGNMENT_TSV'],destdir)
for chrom in info['CHROM_MZML']:
shutil.copy(chrom,destdir)
subprocess.call("gunzip -v %s/*.gz 2>&1" % destdir, shell=True)
mailtext = "\n\nswath2viewer was enabled. To visualize results use e.g.\n" \
"user@crick# /opt/imsb/georger/py26/bin/python " \
"/opt/imsb/georger/msproteomicstools/gui/TAPIR.py --in %s/*" % destdir
except Exception, e:
log.warn("swath2viewer failed! " + e.message)
def run(self, log, info):
info['WORKFLOW'] = dropbox.extendWorkflowID(info['WORKFLOW'])
info['WORKFLOW'] += " msproteomicstools@" + subprocess.check_output(
"git --git-dir=/cluster/apps/imsbtools/stable/msproteomicstools/master/.git rev-parse --short HEAD",
shell=True).strip()
info['WORKFLOW'] += " openms@" + subprocess.check_output(
"git --git-dir=/cluster/apps/openms/2.0.0-memory/OpenMS/.git rev-parse --short HEAD",
shell=True).strip()
info['DROPBOXSTAGE'] = stagebox = dropbox.make_stagebox(log, info)
expcode = dropbox.get_experiment_code(info)
#copy and compress align.csv, but not the matrix REQUEST: expcode in name
tgt = os.path.join(
stagebox, expcode + '_' + os.path.basename(info['ALIGNMENT_TSV']))
shutil.copy(info['ALIGNMENT_TSV'], tgt)
subprocess.check_call('gzip ' + stagebox + '/* 2>&1', shell=True)
tgt = os.path.join(
stagebox,
expcode + '_' + os.path.basename(info['ALIGNMENT_MATRIX']))
shutil.copy(info['ALIGNMENT_MATRIX'], tgt)
dropbox.keys_to_dropbox(log, info, ['ALIGNER_STDOUT'], stagebox)
#compress all mprophet files into one zip
archive = os.path.join(stagebox, 'pyprophet_stats.zip')
if not 'MPROPHET_STATS' in info:
info['MPROPHET_STATS'] = []
if not isinstance(info['MPROPHET_STATS'], list):
info['MPROPHET_STATS'] = [info['MPROPHET_STATS']]
for entry in info['MPROPHET_STATS']:
subprocess.check_call('zip -j ' + archive + ' ' + entry + " 2>&1",
shell=True)
#PATCH: reimport old classifier if existing was used
for key in ("MPR_LDA_PATH", "MPR_WT_PATH"):
path = info.get(key, "")
if path:
if not os.path.exists(path):
raise Exception(
"can not add %s to archive, file does not exist" %
path)
subprocess.check_call('zip -j ' + archive + ' ' + path +
" 2>&1",
shell=True)
#if 'MPR_LDA_PATH' in info and info['MPR_LDA_PATH'] != "":
#subprocess.check_call('zip -j ' + archive + ' ' + info['MPR_LDA_PATH'] + " 2>&1", shell=True)
#if 'MPR_WT_PATH' in info and info['MPR_WT_PATH'] != "":
#if not os.path.exists(info['MPR_WT_PATH']):
#raise Exception("%s does not exist" % info['MPR_WT_PATH'])
#subprocess.check_call('zip -j ' + archive + ' ' + info['MPR_WT_PATH'] + " 2>&1", shell=True)
#SPACE PROJECT given
dsinfo = {}
dsinfo['SPACE'] = info['SPACE']
dsinfo['PROJECT'] = info['PROJECT']
dsinfo['PARENT_DATASETS'] = info[Keys.DATASET_CODE]
if info.get("DB_SOURCE", "") == "PersonalDB":
if isinstance(dsinfo['PARENT_DATASETS'], list):
dsinfo['PARENT_DATASETS'].append(info["DBASE"])
else:
dsinfo['PARENT_DATASETS'] = [
dsinfo['PARENT_DATASETS'], info['DBASE']
]
dsinfo['DATASET_TYPE'] = 'SWATH_RESULT'
dsinfo['EXPERIMENT_TYPE'] = 'SWATH_SEARCH'
dsinfo['EXPERIMENT'] = expcode
path = os.path.join(stagebox, 'dataset.attributes')
IniInfoHandler().write(dsinfo, path)
expinfo = {}
expinfo['PARENT-DATA-SET-CODES'] = dsinfo['PARENT_DATASETS']
expinfo['ALIGNER_DSCORE_CUTOFF'] = info['MPR_DSCORE_CUTOFF']
for key in [
'WORKFLOW',
'COMMENT',
'TRAML',
'EXTRACTION_WINDOW',
'WINDOW_UNIT',
'RT_EXTRACTION_WINDOW',
'MIN_UPPER_EDGE_DIST',
'IRTTRAML',
'MIN_RSQ',
'MIN_COVERAGE',
'MPR_NUM_XVAL',
'MPR_LDA_PATH',
'MPR_WT_PATH',
'MPR_FRACT',
'MPR_SSL_IF',
'MPR_SSL_IL',
'MPR_SSL_TF',
'MPR_SSL_TL',
'MPR_SSL_NI',
'MPR_MAINVAR',
'MPR_VARS',
'ALIGNER_FRACSELECTED',
'ALIGNER_MAX_RT_DIFF',
'ALIGNER_METHOD',
'ALIGNER_FDR',
'ALIGNER_MAX_FDRQUAL',
'ALIGNER_TARGETFDR',
'ALIGNER_REALIGN_METHOD',
'DO_CHROMML_REQUANT',
'ALGNER_MST_USERTCORR',
'ALIGNER_MST_STDEVMULT',
'ISOTOPIC_GROUPING',
'REQUANT_METHOD',
]:
if info.get(key, "") != "":
expinfo[key] = info[key]
path = os.path.join(stagebox, 'experiment.properties')
IniInfoHandler().write(expinfo, path)
#put a copy of the whole ini into the dropbox. copy() to prevent OUTPUT being removed from main ini
IniInfoHandler().write(info.copy(),
os.path.join(stagebox, 'input.ini'))
mailtext = ""
if info.get("RUNSWATH2VIEWER", "") == "true":
try:
#destdir = "/IMSB/ra/%s/html/tapir/%s" % (getpass.getuser(), dsinfo['EXPERIMENT'])
destdir = "/nfs/195.176.109.15/nas/fs2102/biol_ibt_usr_s1/%s/tapir/%s" % (
getpass.getuser(), dsinfo['EXPERIMENT'])
makedirs_clean(destdir)
shutil.copy(info['ALIGNMENT_TSV'], destdir)
for chrom in info['CHROM_MZML']:
shutil.copy(chrom, destdir)
subprocess.call("gunzip -v %s/*.gz 2>&1" % destdir, shell=True)
mailtext = "\n\nswath2viewer was enabled. To visualize results use e.g.\n" \
"user@crick# /opt/imsb/georger/py26/bin/python " \
"/opt/imsb/georger/msproteomicstools/gui/TAPIR.py --in %s/*" % destdir
except Exception, e:
log.warn("swath2viewer failed! " + e.message)