def updateruninfo(request):
'''Replaces crawler.py code
Ensure this function/action is idempotent'''
folder = request.POST.get('datapath')
logger.debug("updateruninfo looking at %s" % folder)
# parse explog.txt.
LOG_BASENAME = "explog.txt"
text = load_log(folder, LOG_BASENAME)
if text is None:
payload = "Cannot read %s" % (os.path.join(folder, LOG_BASENAME))
logger.warn(payload)
return http.HttpResponseServerError(payload)
else:
try:
explog_parsed = parse_log(text)
except:
logger.warn("Error parsing %s, skipping %s" % (LOG_BASENAME, folder))
logger.error(traceback.format_exc())
return http.HttpResponseServerError('%s %s' % (folder, traceback.format_exc()))
# Update experiment, plan, eas objects for dataset in this folder
try:
planObj, expObj, easObj = get_planned_exp_objects(explog_parsed, folder)
update_exp_objects_from_log(explog_parsed, folder, planObj, expObj, easObj)
save_serialized_json(folder, planObj, expObj, easObj)
except:
logger.error(traceback.format_exc())
return http.HttpResponseServerError('%s %s' % (folder, traceback.format_exc()))
return http.HttpResponse("%s database objects have been updated" % folder, mimetype='text/plain')
def update_expobj_ftpcompleted(_expobj, folder):
'''Update Experiment object with completed ftp status'''
# Store final explog in database
exptxt = load_log(folder, LOG_FINAL_BASENAME)
if exptxt is not None:
_expobj.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
return
# Set FTP transfer status to complete
if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
_expobj.ftpStatus = RUN_STATUS_COMPLETE
logger.errors.info("FTP status: Complete")
# Save experiment object
_expobj.save()
# Measure and record disk space usage of this dataset
# This is celery task which will execute in separate worker thread
try:
tasks.setRunDiskspace.delay(_expobj.pk)
except:
logger.errors.error(traceback.format_exc())
raise
def __init__(self,name,config):
self.name = name
self.dat_path = os.path.join(config.get('global','results'), name)
self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
self.status = 'new'
explog_file = os.path.join(self.dat_path, "explog.txt")
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict['oninstranalysis'] #yes, no
self.exp_usesynchdats = 'yes' in self.explogdict['use_synchdats'] #yes, no
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = int(config.get('global','block_to_process_start'))
self.block_to_process_end = int(config.get('global','block_to_process_end'))
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def update_expobj_ftpcompleted(_expobj, folder):
'''Update Experiment object with completed ftp status'''
# Store final explog in database
exptxt = load_log(folder, LOG_FINAL_BASENAME)
if exptxt is not None:
_expobj.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
return
# Set FTP transfer status to complete
if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
_expobj.ftpStatus = RUN_STATUS_COMPLETE
logger.errors.info("FTP status: Complete")
# Save experiment object
_expobj.save()
# Measure and record disk space usage of this dataset
# This is celery task which will execute in separate worker thread
try:
tasks.setRunDiskspace.delay(_expobj.pk)
except:
logger.errors.error(traceback.format_exc())
raise
def generate_project_name(exp):
"""generate project name from parsing explog or input"""
if len(sys.argv) > 3:
project_name = sys.argv[3]
else:
fpath = load_log(exp.expDir, "explog.txt")
explog = parse_log(fpath)
crawler.exp_kwargs(explog, exp.expDir)
project_name = explog.get("project", "")
return project_name
def generate_project_name(exp):
"""generate project name from parsing explog or input"""
if len(sys.argv) > 5:
project_name = sys.argv[5]
else:
fpath = load_log(exp.expDir, 'explog.txt')
explog = parse_log(fpath)
#crawler.exp_kwargs(explog, exp.expDir, logger.errors)
project_name = explog.get('project', '')
return project_name
def generate_project_name(exp):
"""generate project name from parsing explog or input"""
if len(sys.argv) > 3:
project_name = sys.argv[3]
else:
fpath = load_log(exp.expDir, 'explog.txt')
explog = parse_log(fpath)
crawler.exp_kwargs(explog, exp.expDir)
project_name = explog.get('project', '')
return project_name
def isChefInfoAvailable(folder):
# parse chef_param.json
chefLog_parsed = {}
JSON_BASENAME = "chef_params.json"
chefLog = load_log(folder, JSON_BASENAME)
if chefLog is not None:
try:
chefLog_parsed = json.loads(chefLog)
except:
logger.warn("Error parsing %s, skipping %s" % (JSON_BASENAME, folder))
logger.error(traceback.format_exc())
print(traceback.format_exc())
return chefLog_parsed
def isChefInfoAvailable(folder):
# parse chef_param.json
chefLog_parsed = {}
JSON_BASENAME = "chef_params.json"
chefLog = load_log(folder, JSON_BASENAME)
if chefLog is None:
payload = "Chef summary info not available read %s" % (os.path.join(
folder, JSON_BASENAME))
logger.warn(payload)
else:
try:
chefLog_parsed = json.loads(chefLog)
except:
logger.warn("Error parsing %s, skipping %s" %
(JSON_BASENAME, folder))
logger.error(traceback.format_exc())
print(traceback.format_exc())
return chefLog_parsed
def update_expobj_ftpcompleted(_expobj, folder):
'''Update Experiment object with completed ftp status'''
# Store final explog in database
exptxt = load_log(folder, LOG_FINAL_BASENAME)
if exptxt is not None:
_expobj.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
return
# Set FTP transfer status to complete
if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
_expobj.ftpStatus = RUN_STATUS_COMPLETE
logger.errors.info("FTP status: Complete")
# Save experiment object
_expobj.save()
return
def update_expobj_ftpcompleted(_expobj, folder):
'''Update Experiment object with completed ftp status'''
# Store final explog in database
exptxt = load_log(folder, LOG_FINAL_BASENAME)
if exptxt is not None:
_expobj.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
return
# Set FTP transfer status to complete
if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
_expobj.ftpStatus = RUN_STATUS_COMPLETE
logger.errors.info("FTP status: Complete")
# Save experiment object
_expobj.save()
return
def updateruninfo(request):
"""Replaces crawler.py code
Ensure this function/action is idempotent"""
folder = request.POST.get("datapath")
logger.debug("updateruninfo looking at %s" % folder)
# parse explog.txt.
LOG_BASENAME = "explog.txt"
text = load_log(folder, LOG_BASENAME)
if text is None:
payload = "Cannot read %s" % (os.path.join(folder, LOG_BASENAME))
logger.warn(payload)
return http.HttpResponseServerError(payload)
else:
try:
explog_parsed = parse_log(text)
except Exception:
logger.warn("Error parsing %s, skipping %s" %
(LOG_BASENAME, folder))
logger.error(traceback.format_exc())
return http.HttpResponseServerError(
"%s %s" % (folder, traceback.format_exc()))
# Update experiment, plan, eas objects for dataset in this folder
try:
planObj, expObj, easObj = get_planned_exp_objects(
explog_parsed, folder)
update_exp_objects_from_log(explog_parsed, folder, planObj, expObj,
easObj)
save_serialized_json(folder, planObj, expObj, easObj)
except Exception:
logger.error(traceback.format_exc())
return http.HttpResponseServerError(
"%s %s" % (folder, traceback.format_exc()))
return http.HttpResponse("%s database objects have been updated" %
folder,
mimetype="text/plain")
def __init__(self,results,name,config):
self.name = name
self.dat_path = os.path.join(results, name)
self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
self.status = 'new'
explog_file = os.path.join(results, name, "explog.txt")
if os.path.exists(explog_file):
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict['oninstranalysis']#yes, no
self.exp_usesynchdats = 'yes' in self.explogdict['use_synchdats']#yes, no
else:
self.explogdict = None
self.explog_file = None
self.exp_flows = None
self.exp_oninstranalysis = None
self.exp_usesynchdats = None
self.blocks = []
if self.explogdict:
try:
self.discover_blocks(0,95)#TODO block_to_process_start,block_to_process_end)
except:
logger.error(traceback.format_exc())
pass
else:
logger.error(traceback.format_exc())
def __init__(self,name,config):
self.name = name
self.dat_path = os.path.join(config.get('global','results'), name)
self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
self.status = 'new'
explog_file = os.path.join(self.dat_path, "explog.txt")
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
self.exp_planned_run_short_id = self.explogdict.get('planned_run_short_id','')
self.exp_chiptype = self.explogdict["chiptype"]
logger.info('chiptype:-%s-' % self.exp_chiptype)
self.exp_runtype = self.explogdict["runtype"]
logger.info('runtype:-%s-' % self.exp_runtype)
'''
if os.path.exists(os.path.join(self.dat_path, "chips.json")):
chips = json.load(open('os.path.join(self.dat_path, "chips.json")','r'))
else:
logger.error('use default args /usr/share/ion/oia/chips.json')
chips = json.load(open('/usr/share/ion/oia/chips.json','r'))
'''
if os.path.exists('/software/config/DataCollect.config'):
f = open('/software/config/DataCollect.config')
text = f.readlines()
f.close()
for line in text:
[key, value] = line.split(':')
key = key.strip()
value = value.strip()
logger.info('%s %s' % (key, value))
if key == 'TSUrl':
TSUrl = value
elif key == 'TSUserName':
TSUserName = value
elif key == 'TSPasswd':
TSPasswd = value
else:
continue
else:
raise
logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd))
headers = {
"Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
"Content-type": "application/json",
"Accept": "application/json" }
# get chip data
conn = httplib.HTTPConnection(TSUrl)
conn.request("GET", "/rundb/api/v1/chip/?format=json", "", headers)
response = conn.getresponse()
data = response.read()
chips = json.loads(data)
# get plan data
conn = httplib.HTTPConnection(TSUrl)
conn.request("GET", "/rundb/api/v1/plannedexperiment/?format=json", "", headers)
response = conn.getresponse()
logger.info('%s %s' % (response.status, response.reason))
data = response.read()
# Need to limit plans
#logger.debug(data)
plans = json.loads(data)
for chip in chips['objects']:
logger.info(chip)
logger.info('chiptype test: test normal chiptype: [%s]' % self.exp_chiptype)
if chip['name']==self.exp_chiptype:
logger.info('chiptype test: found normal chiptype: [%s]' % chip['name'])
self.exp_beadfindArgs = chip['beadfindargs']
self.exp_analysisArgs = chip['analysisargs']
logger.info('chiptype test: test special chiptype/runtype: [%s%s]' % (self.exp_chiptype,self.exp_runtype))
if chip['name']==self.exp_chiptype+self.exp_runtype:
logger.info('chiptype test: found special chiptype/runtype: [%s]' % chip['name'])
self.exp_beadfindArgs = chip['beadfindargs']
self.exp_analysisArgs = chip['analysisargs']
break
logger.info('beadfindArgs:-%s-' % self.exp_beadfindArgs)
logger.info('analysisArgs:-%s-' % self.exp_analysisArgs)
plan = [plan for plan in plans['objects'] if plan.get('planShortID','not_available') == self.exp_planned_run_short_id]
if plan:
logger.debug("Plan: %s" % plan[0])
self.libraryKey=plan[0].get('libraryKey','TCAG') # TODO
self.tfKey=plan[0].get('tfKey','ATCG') # TODO
else:
logger.debug("no plan")
self.libraryKey="TCAG"
self.tfKey="ATCG"
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint('global','block_to_process_start')
self.block_to_process_end = config.getint('global','block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def __init__(self, name, config):
self.name = name
self.dat_path = os.path.join(config.get('global', 'results'), name)
self.analysis_path = os.path.join(
config.get('global', 'analysisresults'), name)
self.sigproc_results_path = os.path.join(
config.get('global', 'analysisresults'), name, "onboard_results",
"sigproc_results")
self.status = 'new'
self.last_flow = -1
try:
if not os.path.exists(self.sigproc_results_path):
os.makedirs(self.sigproc_results_path)
except:
logger.error(traceback.format_exc())
explog_file = os.path.join(self.analysis_path, "explog.txt")
if not os.path.exists(explog_file):
shutil.copyfile(os.path.join(self.dat_path, "explog.txt"),
os.path.join(self.analysis_path, "explog.txt"))
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
if not os.path.exists(
os.path.join(self.dat_path, 'onboard_results')):
os.symlink(os.path.join(self.analysis_path, 'onboard_results'),
os.path.join(self.dat_path, 'onboard_results'))
except:
logger.error(traceback.format_exc())
try:
self.explog_file = explog_file
# parse explog.txt
(head, tail) = os.path.split(explog_file)
explogtext = load_log(head, tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get(
'oninstranalysis', 'no')
self.exp_usesynchdats = 'yes' in self.explogdict.get(
'use_synchdats', 'no')
self.exp_oia_during_run = 'yes' in self.explogdict.get(
'oia_during_run', 'yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get(
'ecc_enabled', 'no')
self.exp_planned_run_guid = self.explogdict.get(
'planned_run_guid', '')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_seqkitplanname = self.explogdict.get('seqkitplanname', '')
self.exp_chipversion = self.explogdict.get('chipversion', '')
if not self.exp_chipversion or not self.exp_chipversion.startswith(
'P'):
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
try:
logger.info('connect to:%s user:%s password:%s' %
(TSUrl, TSUserName, TSPasswd))
headers = {
"Authorization":
"Basic " + string.strip(
base64.encodestring(TSUserName + ':' + TSPasswd)),
"Content-type":
"application/json",
"Accept":
"application/json"
}
conn = httplib.HTTPConnection(TSUrl)
if self.exp_planned_run_guid:
# TODO, use original plan data from file system
request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&planGUID=%s&sequenceKitName=%s&limit=20" % (
self.exp_chipversion, self.exp_planned_run_guid,
self.exp_seqkitplanname)
else:
logger.debug("use default chip args")
request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&sequenceKitName=%s&limit=20" % (
self.exp_chipversion, self.exp_seqkitplanname)
#http://frenzy.ite/rundb/api/v1/analysisargs/getargs/?format=json&chipType=P1.2.18&planGUID=&sequenceKitName=ProtonI200Kit-v2
logger.info('request: %s http://%s%s' % (self.explogdict.get(
'runname', 'unknown'), TSUrl, request))
conn.request("GET", request, "", headers)
response = conn.getresponse()
logger.info('%s %s' % (response.status, response.reason))
data = response.read()
analysisargs = json.loads(data)
logger.debug("Plan: %s" % analysisargs)
f = open(
os.path.join(self.sigproc_results_path,
"retrieved_args.json"), 'w')
f.write(json.dumps(analysisargs, indent=4))
f.close()
if analysisargs:
self.exp_beadfindArgs_thumbnail = analysisargs[
'thumbnailbeadfindargs']
self.exp_analysisArgs_thumbnail = analysisargs[
'thumbnailanalysisargs']
self.exp_beadfindArgs_block = analysisargs['beadfindargs']
self.exp_analysisArgs_block = analysisargs['analysisargs']
self.libraryKey = 'TCAG'
self.tfKey = 'ATCG'
else:
raise Exception("Analysis args are empty")
except:
raise
logger.info("thumbnail beadfindArgs: '%s'" %
self.exp_beadfindArgs_thumbnail)
logger.info("thumbnail analysisArgs: '%s'" %
self.exp_analysisArgs_thumbnail)
logger.info("block beadfindArgs: '%s'" %
self.exp_beadfindArgs_block)
logger.info("block analysisArgs: '%s'" %
self.exp_analysisArgs_block)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint(
'global', 'block_to_process_start')
self.block_to_process_end = config.getint('global',
'block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def crawl(folders, logger):
"""Crawl over ``folders``, reporting information to the ``CrawlLog``
``logger``."""
if folders:
logger.errors.info("checking %d directories" % len(folders))
for f in folders:
try:
text = load_log(f, LOG_BASENAME)
logger.errors.debug("checking directory %s" % f)
#------------------------------------
# Check for a valid explog.txt file
#------------------------------------
if text is not None:
logger.current_folder = f
# Catch any errors parsing this log file and continue to the next experiment.
try:
d = parse_log(text)
except:
logger.errors.info("Error parsing explog, skipping %s" % f)
logger.errors.exception(traceback.format_exc())
continue
exp = None
exp_set = models.Experiment.objects.filter(unique=f)
if exp_set:
# Experiment already in database
exp = exp_set[0]
if exp.ftpStatus == RUN_STATUS_COMPLETE:
continue
else:
# Need to create experiment for this folder
try:
planObj, expObj, easObj = get_planned_exp_objects(d, f)
exp = update_exp_objects_from_log(
d, f, planObj, expObj, easObj)
except:
logger.errors.info(
"Error updating experiment objects, skipping %s" %
f)
logger.errors.exception(traceback.format_exc())
continue
if exp is not None:
logger.errors.info("%s" % f)
#--------------------------------
# Update FTP Transfer Status
#--------------------------------
if check_for_completion(exp):
exptxt = load_log(f, LOG_FINAL_BASENAME)
if exptxt is not None:
exp.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
continue
# FTP transfer is complete
if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
exp.ftpStatus = RUN_STATUS_COMPLETE
exp.save()
# Measure and record disk space usage of this dataset
# This is celery task which will execute in separate worker thread
try:
tasks.setRunDiskspace.delay(exp.pk)
except:
logger.errors.exception(traceback.format_exc())
else:
if exp.ftpStatus != RUN_STATUS_MISSING:
exp.ftpStatus = get_percent(exp)
exp.save()
logger.errors.info("FTP status: Transferring")
#--------------------------------
# Handle auto-run analysis
#--------------------------------
composite_exists, thumbnail_exists = reports_exist(exp)
logger.errors.debug("Reports Exist? %s %s" %
(composite_exists, thumbnail_exists))
if not composite_exists:
logger.errors.debug(
"No auto-run analysis has been started")
# Check if auto-run for whole chip has been requested
if exp.autoAnalyze:
logger.errors.debug(
" auto-run whole chip analysis has been requested"
)
#if check_analyze_early(exp) or check_for_completion(exp):
if ready_to_process(exp) or check_for_completion(
exp):
logger.errors.info(
" Start a whole chip auto-run analysis job"
)
generate_http_post(exp)
else:
logger.errors.info(
" Do not start a whole chip auto-run job yet"
)
else:
logger.errors.debug(
" auto-run whole chip analysis has not been requested"
)
else:
logger.errors.debug("composite report already exists")
if not thumbnail_exists:
# Check if auto-run for thumbnail has been requested
# But only if its a block dataset
if 'tiled' in exp.rawdatastyle:
if autorun_thumbnail(exp):
logger.errors.debug(
"auto-run thumbnail analysis has been requested"
)
if ready_to_process_thumbnail(
exp) or check_for_completion(exp):
logger.errors.info(
" Start a thumbnail auto-run analysis job"
)
generate_http_post(exp, DO_THUMBNAIL)
else:
logger.errors.info(
" Do not start a thumbnail auto-run job yet"
)
else:
logger.errors.debug(
"auto-run thumbnail analysis has not been requested"
)
else:
logger.errors.debug(
"This is not a block dataset; no thumbnail to process"
)
else:
logger.errors.debug("thumbnail report already exists")
else:
logger.errors.debug("exp is empty")
else:
# No explog.txt exists; probably an engineering test run
logger.errors.debug("no %s was found in %s" %
(LOG_BASENAME, f))
except:
logger.errors.exception(traceback.format_exc())
logger.current_folder = '(none)'
def __init__(self,name,config):
self.name = name
self.dat_path = os.path.join(config.get('global','results'), name)
self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
self.status = 'new'
self.last_flow = -1
explog_file = os.path.join(self.dat_path, "explog.txt")
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
self.exp_planned_run_short_id = self.explogdict.get('planned_run_short_id','')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_chipversion = self.explogdict.get('chipversion','')
if not self.exp_chipversion or not self.exp_chipversion.startswith('P'):
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_short_id: '%s'" % self.exp_planned_run_short_id)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
if os.path.exists('/software/config/DataCollect.config'):
f = open('/software/config/DataCollect.config')
text = f.readlines()
f.close()
for line in text:
[key, value] = line.split(':')
key = key.strip()
value = value.strip()
logger.info('%s %s' % (key, value))
if key == 'TSUrl':
TSUrl = value
elif key == 'TSUserName':
TSUserName = value
elif key == 'TSPasswd':
TSPasswd = value
else:
continue
else:
raise
logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd))
headers = {
"Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
"Content-type": "application/json",
"Accept": "application/json" }
try:
plan_found = False
if self.exp_planned_run_short_id:
# get plan data
conn = httplib.HTTPConnection(TSUrl)
conn.request("GET", "/rundb/api/v1/plannedexperiment/?format=json&planShortID=%s&limit=20" % self.exp_planned_run_short_id, "", headers)
response = conn.getresponse()
logger.info('%s %s' % (response.status, response.reason))
data = response.read()
# Need to limit plans
plans = json.loads(data)
logger.debug("number of plans found on %s: %s" % (TSUrl,len(plans['objects'])))
logger.debug("search for plan with planShortID: '%s'" % self.exp_planned_run_short_id)
plan = [plan for plan in plans['objects'] if plan.get('planShortID','not_available') == self.exp_planned_run_short_id]
logger.debug("number of plans with planShortID '%s' found: %s" % (self.exp_planned_run_short_id,len(plan)))
if plan:
logger.debug("Plan: %s" % plan[0])
self.exp_beadfindArgs = plan[0].get('beadfindargs','justBeadFind')
self.exp_analysisArgs = plan[0].get('analysisargs','Analysis')
self.libraryKey = plan[0].get('libraryKey','TCAG')
self.tfKey = plan[0].get('tfKey','ATCG')
plan_found = True
else:
logger.debug("Problem with args in plan '%s'" % self.exp_planned_run_short_id)
if not plan_found:
logger.debug("plan not available")
# TODO: user might starts run during TS upgrade
conn = httplib.HTTPConnection(TSUrl)
conn.request("GET", "/rundb/api/v1/analysisargs/?format=json", "", headers)
response = conn.getresponse()
data = response.read()
allanalysisargs = json.loads(data)
chip_found = False
logger.info('chipversion test: search for chipversion: [%s]' % self.exp_chipversion)
for analysisargs in allanalysisargs['objects']:
logger.info('chipversion test: compare with chipversion: [%s]' % analysisargs['name'])
if analysisargs['name']=="default_"+self.exp_chipversion:
logger.info('chipversion test: found analysis args for chipversion: [%s]' % analysisargs['name'])
logger.info(analysisargs)
self.exp_beadfindArgs = analysisargs['beadfindargs']
self.exp_analysisArgs = analysisargs['analysisargs']
self.libraryKey = 'TCAG'
self.tfKey = 'ATCG'
chip_found = True
break
if not chip_found:
raise Exception("Chiptype %s not found" % self.exp_chipversion)
except:
logger.error(traceback.format_exc())
self.exp_beadfindArgs = 'justBeadFind --beadfind-minlivesnr 3 --region-size=216x224 --total-timeout 602'
self.exp_analysisArgs = 'Analysis --from-beadfind --clonal-filter-bkgmodel on --region-size=216x224 --bkg-bfmask-update off --gpuWorkLoad 1 --total-timeout 602'
self.libraryKey = "TCAG"
self.tfKey = "ATCG"
logger.info("beadfindArgs: '%s'" % self.exp_beadfindArgs)
logger.info("analysisArgs: '%s'" % self.exp_analysisArgs)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint('global','block_to_process_start')
self.block_to_process_end = config.getint('global','block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def crawl(folders, logger):
"""Crawl over ``folders``, reporting information to the ``CrawlLog``
``logger``."""
if folders:
logger.errors.info("checking %d directories" % len(folders))
for f in folders:
try:
text = load_log(f, LOG_BASENAME)
logger.errors.debug("checking directory %s" % f)
#------------------------------------
# Check for a valid explog.txt file
#------------------------------------
if text is not None:
logger.current_folder = f
# Catch any errors parsing this log file and continue to the next experiment.
d = parse_log(text)
kwargs, status = exp_kwargs(d, f)
if status is True:
exp = exp_from_kwargs(kwargs, logger, False)
projectName = d.get('project', '') # part of explog but not a field in Experiment table
if exp is not None:
logger.errors.info("%s" % f)
#make sure 1-1 exp-plan relationship is enforced when new experiment is created
planObj = exp.plan
if planObj and exp.pk == None and len(models.Experiment.objects.filter(plan__pk = planObj.pk)) > 0:
logger.errors.info('WARNING: Plan %s is already associated to an experiment, a copy will be created' % planObj.pk)
planObj.pk = None
planObj.save()
exp.plan = planObj
#--------------------------------
# Update FTP Transfer Status
#--------------------------------
if check_for_completion(exp):
# FTP transfer is complete
if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
exp.ftpStatus = RUN_STATUS_COMPLETE
exp.log = json.dumps(parse_log(load_log(f, LOG_FINAL_BASENAME)), indent=4)
exp.save()
# Measure and record disk space usage of this dataset
# This is celery task which will execute in separate worker thread
try:
tasks.setRunDiskspace.delay(exp.pk)
except:
logger.errors.exception(traceback.format_exc())
else:
if exp.ftpStatus != RUN_STATUS_MISSING:
exp.ftpStatus = get_percent(exp)
exp.save()
logger.errors.info("FTP status: Transferring")
#--------------------------------
# Handle auto-run analysis
#--------------------------------
composite_exists, thumbnail_exists = reports_exist(exp)
logger.errors.debug("Reports Exist? %s %s" % (composite_exists, thumbnail_exists))
if not composite_exists:
logger.errors.debug("No auto-run analysis has been started")
# Check if auto-run for whole chip has been requested
if exp.autoAnalyze:
logger.errors.debug(" auto-run whole chip analysis has been requested")
#if check_analyze_early(exp) or check_for_completion(exp):
if ready_to_process(exp) or check_for_completion(exp):
logger.errors.info(" Start a whole chip auto-run analysis job")
generate_http_post(exp, projectName)
else:
logger.errors.info(" Do not start a whole chip auto-run job yet")
else:
logger.errors.debug(" auto-run whole chip analysis has not been requested")
else:
logger.errors.debug("composite report already exists")
if not thumbnail_exists:
# Check if auto-run for thumbnail has been requested
# But only if its a block dataset
if 'tiled' in exp.rawdatastyle:
if autorun_thumbnail(exp):
logger.errors.debug("auto-run thumbnail analysis has been requested")
if ready_to_process_thumbnail(exp) or check_for_completion(exp):
logger.errors.info(" Start a thumbnail auto-run analysis job")
generate_http_post(exp, projectName, DO_THUMBNAIL)
else:
logger.errors.info(" Do not start a thumbnail auto-run job yet")
else:
logger.errors.debug("auto-run thumbnail analysis has not been requested")
else:
logger.errors.debug("This is not a block dataset; no thumbnail to process")
else:
logger.errors.debug("thumbnail report already exists")
else:
logger.errors.debug("exp is empty")
else:
logger.errors.warn("folder %s is not added due to invalid experiment status from exp_kwargs. " % f)
else:
# No explog.txt exists; probably an engineering test run
logger.errors.debug("no %s was found in %s" % (LOG_BASENAME, f))
except:
logger.errors.exception(traceback.format_exc())
logger.current_folder = '(none)'
def __init__(self,name,config):
self.name = name
self.dat_path = os.path.join(config.get('global','results'), name)
self.analysis_path = os.path.join(config.get('global','analysisresults'), name)
self.sigproc_results_path = os.path.join(config.get('global','analysisresults'), name, "onboard_results", "sigproc_results")
self.status = 'new'
self.last_flow = -1
try:
if not os.path.exists( self.sigproc_results_path ):
os.makedirs( self.sigproc_results_path )
except:
logger.error(traceback.format_exc())
explog_file = os.path.join(self.analysis_path, "explog.txt")
if not os.path.exists(explog_file):
shutil.copyfile(os.path.join(self.dat_path, "explog.txt"), os.path.join(self.analysis_path, "explog.txt"))
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
if not os.path.exists( os.path.join(self.dat_path, 'onboard_results') ):
os.symlink( os.path.join(self.analysis_path,'onboard_results'), os.path.join(self.dat_path,'onboard_results') )
except:
logger.error(traceback.format_exc())
try:
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
self.exp_planned_run_guid = self.explogdict.get('planned_run_guid','')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_seqkitplanname = self.explogdict.get('seqkitplanname','')
self.exp_chipversion = self.explogdict.get('chipversion','')
if not self.exp_chipversion:
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
try:
f = open(os.path.join(self.dat_path, "planned_run.json"),'r')
analysisargs = json.load(f)['objects'][0]
f.close()
if analysisargs:
self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs']
self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs']
self.exp_beadfindArgs_block = analysisargs['beadfindargs']
self.exp_analysisArgs_block = analysisargs['analysisargs']
self.libraryKey = analysisargs['libraryKey']
self.tfKey = analysisargs['tfKey']
else:
raise Exception("Analysis args are empty")
except:
raise
logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail)
logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail)
logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block)
logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint('global','block_to_process_start')
self.block_to_process_end = config.getint('global','block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def __init__(self, name, config):
self.name = name
self.dat_path = os.path.join(config.get('global', 'results'), name)
self.analysis_path = os.path.join(config.get('global', 'analysisresults'), name)
self.sigproc_results_path = os.path.join(
config.get('global', 'analysisresults'), name, "onboard_results", "sigproc_results")
self.status = 'new'
self.last_flow = -1
try:
if not os.path.exists(self.sigproc_results_path):
os.makedirs(self.sigproc_results_path)
except:
logger.error(traceback.format_exc())
explog_file = os.path.join(self.analysis_path, "explog.txt")
if not os.path.exists(explog_file):
shutil.copyfile(os.path.join(self.dat_path, "explog.txt"),
os.path.join(self.analysis_path, "explog.txt"))
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
if not os.path.exists(os.path.join(self.dat_path, 'onboard_results')):
os.symlink(os.path.join(self.analysis_path, 'onboard_results'),
os.path.join(self.dat_path, 'onboard_results'))
except:
logger.error(traceback.format_exc())
try:
self.explog_file = explog_file
# parse explog.txt
(head, tail) = os.path.split(explog_file)
explogtext = load_log(head, tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis', 'no')
self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats', 'no')
self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run', 'yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled', 'no')
self.exp_planned_run_guid = self.explogdict.get('planned_run_guid', '')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_seqkitplanname = self.explogdict.get('seqkitplanname', '')
self.exp_chipversion = self.explogdict.get('chipversion', '')
if not self.exp_chipversion:
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
try:
analysisargs = {}
if os.path.exists(os.path.join(self.dat_path, "reanalysis_args.json")):
f = open(os.path.join(self.dat_path, "reanalysis_args.json"), 'r')
analysisargs = json.load(f)
f.close()
logger.info("use analysisargs from %s" %
os.path.join(self.dat_path, "reanalysis_args.json"))
elif os.path.exists(os.path.join(self.dat_path, "planned_run.json")):
f = open(os.path.join(self.dat_path, "planned_run.json"), 'r')
analysisargs = json.load(f)['objects'][0]
f.close()
logger.info("use analysisargs from %s" % os.path.join(self.dat_path, "planned_run.json"))
else:
logger.info("trying to fetch planned run for %s" %
os.path.join(self.dat_path, "planned_run.json"))
http_command = "python /software/testing/get_reanalysis_args.py -e %s" % (name)
args = shlex.split(http_command.encode('utf8'))
logger.info("run process: %s" % (args))
outfile = open(os.path.join(self.dat_path, 'get_reanalysis_args_oia.log'), "w")
p = subprocess.Popen(args, stdout=outfile, stderr=subprocess.STDOUT)
# add popen process to block
ret = p.wait()
outfile.close()
if os.path.exists(os.path.join(self.dat_path, "planned_run.json")):
with open(os.path.join(self.dat_path, "planned_run.json"), 'r') as f:
try:
analysisargs = json.load(f)['objects'][0]
logger.info("Using Plan information from %s" %
os.path.join(self.dat_path, "planned_run.json"))
except IndexError:
raise Exception("File %s is empty",
os.path.join(self.dat_path, "planned_run.json"))
if len(analysisargs) > 0:
self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs']
self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs']
self.exp_beadfindArgs_block = analysisargs['beadfindargs']
self.exp_analysisArgs_block = analysisargs['analysisargs']
self.exp_prebasecallerArgs_block = analysisargs['prebasecallerargs']
self.libraryKey = analysisargs['libraryKey']
self.tfKey = analysisargs['tfKey']
else:
raise Exception("There are no Analysis args available")
except:
raise
logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail)
logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail)
logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block)
logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block)
logger.info("block prebasecallerArgs: '%s'" % self.exp_prebasecallerArgs_block)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint('global', 'block_to_process_start')
self.block_to_process_end = config.getint('global', 'block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def crawl(folders, logger):
"""Crawl over ``folders``, reporting information to the ``CrawlLog``
``logger``."""
if folders:
logger.errors.info("checking %d directories" % len(folders))
for f in folders:
try:
text = load_log(f, LOG_BASENAME)
logger.errors.debug("checking directory %s" % f)
#------------------------------------
# Check for a valid explog.txt file
#------------------------------------
if text is not None:
logger.current_folder = f
# Catch any errors parsing this log file and continue to the next experiment.
try:
d = parse_log(text)
except:
logger.errors.info("Error parsing explog, skipping %s" % f)
logger.errors.exception(traceback.format_exc())
continue
exp = None
exp_set = models.Experiment.objects.filter(unique=f)
if exp_set:
# Experiment already in database
exp = exp_set[0]
if exp.ftpStatus == RUN_STATUS_COMPLETE:
continue
else:
# Need to create experiment for this folder
try:
planObj, expObj, easObj = get_planned_exp_objects(d, f, logger.errors)
exp = update_exp_objects_from_log(d, f, planObj, expObj, easObj, logger.errors)
except:
logger.errors.info("Error updating experiment objects, skipping %s" % f)
logger.errors.exception(traceback.format_exc())
continue
if exp is not None:
logger.errors.info("%s" % f)
#--------------------------------
# Update FTP Transfer Status
#--------------------------------
if check_for_completion(exp):
exptxt = load_log(f, LOG_FINAL_BASENAME)
if exptxt is not None:
exp.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
continue
# FTP transfer is complete
if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
exp.ftpStatus = RUN_STATUS_COMPLETE
exp.save()
# Measure and record disk space usage of this dataset
# This is celery task which will execute in separate worker thread
try:
tasks.setRunDiskspace.delay(exp.pk)
except:
logger.errors.exception(traceback.format_exc())
else:
if exp.ftpStatus != RUN_STATUS_MISSING:
exp.ftpStatus = get_filecount(exp)
exp.save()
logger.errors.info("FTP status: Transferring")
#--------------------------------
# Handle auto-run analysis
#--------------------------------
composite_exists, thumbnail_exists = reports_exist(exp)
logger.errors.debug("Reports Exist? %s %s" % (composite_exists, thumbnail_exists))
if not composite_exists:
logger.errors.debug("No auto-run analysis has been started")
# Check if auto-run for whole chip has been requested
if exp.autoAnalyze:
logger.errors.debug(" auto-run whole chip analysis has been requested")
#if check_analyze_early(exp) or check_for_completion(exp):
if ready_to_process(exp) or check_for_completion(exp):
logger.errors.info(" Start a whole chip auto-run analysis job")
generate_http_post(exp, logger)
else:
logger.errors.info(" Do not start a whole chip auto-run job yet")
else:
logger.errors.debug(" auto-run whole chip analysis has not been requested")
else:
logger.errors.debug("composite report already exists")
if not thumbnail_exists:
# Check if auto-run for thumbnail has been requested
# But only if its a block dataset
if 'tiled' in exp.rawdatastyle:
if autorun_thumbnail(exp, logger):
logger.errors.debug("auto-run thumbnail analysis has been requested")
if ready_to_process_thumbnail(exp) or check_for_completion(exp):
logger.errors.info(" Start a thumbnail auto-run analysis job")
generate_http_post(exp, logger, DO_THUMBNAIL)
else:
logger.errors.info(" Do not start a thumbnail auto-run job yet")
else:
logger.errors.debug("auto-run thumbnail analysis has not been requested")
else:
logger.errors.debug("This is not a block dataset; no thumbnail to process")
else:
logger.errors.debug("thumbnail report already exists")
else:
logger.errors.debug("exp is empty")
else:
# No explog.txt exists; probably an engineering test run
logger.errors.debug("no %s was found in %s" % (LOG_BASENAME, f))
except:
logger.errors.exception(traceback.format_exc())
logger.current_folder = '(none)'
def __init__(self,name,config):
self.name = name
self.dat_path = os.path.join(config.get('global','results'), name)
self.analysis_path = os.path.join(config.get('global','analysisresults'), name)
self.sigproc_results_path = os.path.join(config.get('global','analysisresults'), name, "onboard_results", "sigproc_results")
self.status = 'new'
self.last_flow = -1
try:
if not os.path.exists( self.sigproc_results_path ):
os.makedirs( self.sigproc_results_path )
except:
logger.error(traceback.format_exc())
explog_file = os.path.join(self.analysis_path, "explog.txt")
if not os.path.exists(explog_file):
shutil.copyfile(os.path.join(self.dat_path, "explog.txt"), os.path.join(self.analysis_path, "explog.txt"))
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
if not os.path.exists( os.path.join(self.dat_path, 'onboard_results') ):
os.symlink( os.path.join(self.analysis_path,'onboard_results'), os.path.join(self.dat_path,'onboard_results') )
except:
logger.error(traceback.format_exc())
try:
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
self.exp_planned_run_guid = self.explogdict.get('planned_run_guid','')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_seqkitplanname = self.explogdict.get('seqkitplanname','')
self.exp_chipversion = self.explogdict.get('chipversion','')
if not self.exp_chipversion:
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
try:
logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd))
headers = {
"Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
"Content-type": "application/json",
"Accept": "application/json" }
conn = httplib.HTTPConnection(TSUrl)
if self.exp_planned_run_guid:
# TODO, use original plan data from file system
request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&planGUID=%s&sequenceKitName=%s&limit=20" % (self.exp_chipversion, self.exp_planned_run_guid, self.exp_seqkitplanname)
else:
logger.debug("use default chip args")
request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&sequenceKitName=%s&limit=20" % (self.exp_chipversion, self.exp_seqkitplanname)
#http://frenzy.ite/rundb/api/v1/analysisargs/getargs/?format=json&chipType=P1.2.18&planGUID=&sequenceKitName=ProtonI200Kit-v2
logger.info('request: %s http://%s%s' % (self.explogdict.get('runname','unknown'), TSUrl, request))
conn.request("GET", request, "", headers)
response = conn.getresponse()
logger.info('%s %s' % (response.status, response.reason))
data = response.read()
analysisargs = json.loads(data)
logger.debug("Plan: %s" % analysisargs)
f = open(os.path.join(self.sigproc_results_path, "retrieved_args.json"),'w')
f.write(json.dumps(analysisargs, indent=4))
f.close()
if analysisargs:
self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs']
self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs']
self.exp_beadfindArgs_block = analysisargs['beadfindargs']
self.exp_analysisArgs_block = analysisargs['analysisargs']
self.libraryKey = 'TCAG'
self.tfKey = 'ATCG'
else:
raise Exception("Analysis args are empty")
except:
raise
logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail)
logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail)
logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block)
logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint('global','block_to_process_start')
self.block_to_process_end = config.getint('global','block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def __init__(self, name, config):
self.name = name
self.dat_path = os.path.join(config.get('global', 'results'), name)
self.sigproc_results_path = os.path.join(self.dat_path,
"onboard_results",
"sigproc_results")
self.status = 'new'
self.last_flow = -1
explog_file = os.path.join(self.dat_path, "explog.txt")
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
self.explog_file = explog_file
# parse explog.txt
(head, tail) = os.path.split(explog_file)
explogtext = load_log(head, tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get(
'oninstranalysis', 'no')
self.exp_usesynchdats = 'yes' in self.explogdict.get(
'use_synchdats', 'no')
self.exp_oia_during_run = 'yes' in self.explogdict.get(
'oia_during_run', 'yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get(
'ecc_enabled', 'no')
self.exp_planned_run_short_id = self.explogdict.get(
'planned_run_short_id', '')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_chipversion = self.explogdict.get('chipversion', '')
if not self.exp_chipversion or not self.exp_chipversion.startswith(
'P'):
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_short_id: '%s'" %
self.exp_planned_run_short_id)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
if os.path.exists('/software/config/DataCollect.config'):
f = open('/software/config/DataCollect.config')
text = f.readlines()
f.close()
for line in text:
[key, value] = line.split(':')
key = key.strip()
value = value.strip()
logger.info('%s %s' % (key, value))
if key == 'TSUrl':
TSUrl = value
elif key == 'TSUserName':
TSUserName = value
elif key == 'TSPasswd':
TSPasswd = value
else:
continue
else:
raise
logger.info('connect to:%s user:%s password:%s' %
(TSUrl, TSUserName, TSPasswd))
headers = {
"Authorization":
"Basic " +
string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
"Content-type":
"application/json",
"Accept":
"application/json"
}
try:
plan_found = False
if self.exp_planned_run_short_id:
# get plan data
conn = httplib.HTTPConnection(TSUrl)
conn.request(
"GET",
"/rundb/api/v1/plannedexperiment/?format=json&planShortID=%s&limit=20"
% self.exp_planned_run_short_id, "", headers)
response = conn.getresponse()
logger.info('%s %s' % (response.status, response.reason))
data = response.read()
# Need to limit plans
plans = json.loads(data)
logger.debug("number of plans found on %s: %s" %
(TSUrl, len(plans['objects'])))
logger.debug("search for plan with planShortID: '%s'" %
self.exp_planned_run_short_id)
plan = [
plan for plan in plans['objects']
if plan.get('planShortID', 'not_available') ==
self.exp_planned_run_short_id
]
logger.debug(
"number of plans with planShortID '%s' found: %s" %
(self.exp_planned_run_short_id, len(plan)))
if plan:
logger.debug("Plan: %s" % plan[0])
self.exp_beadfindArgs = plan[0].get(
'beadfindargs', 'justBeadFind')
self.exp_analysisArgs = plan[0].get(
'analysisargs', 'Analysis')
self.libraryKey = plan[0].get('libraryKey', 'TCAG')
self.tfKey = plan[0].get('tfKey', 'ATCG')
plan_found = True
else:
logger.debug("Problem with args in plan '%s'" %
self.exp_planned_run_short_id)
if not plan_found:
logger.debug("plan not available")
# TODO: user might starts run during TS upgrade
conn = httplib.HTTPConnection(TSUrl)
conn.request("GET",
"/rundb/api/v1/analysisargs/?format=json", "",
headers)
response = conn.getresponse()
data = response.read()
allanalysisargs = json.loads(data)
chip_found = False
logger.info(
'chipversion test: search for chipversion: [%s]' %
self.exp_chipversion)
for analysisargs in allanalysisargs['objects']:
logger.info(
'chipversion test: compare with chipversion: [%s]'
% analysisargs['name'])
if analysisargs[
'name'] == "default_" + self.exp_chipversion:
logger.info(
'chipversion test: found analysis args for chipversion: [%s]'
% analysisargs['name'])
logger.info(analysisargs)
self.exp_beadfindArgs = analysisargs[
'beadfindargs']
self.exp_analysisArgs = analysisargs[
'analysisargs']
self.libraryKey = 'TCAG'
self.tfKey = 'ATCG'
chip_found = True
break
if not chip_found:
raise Exception("Chiptype %s not found" %
self.exp_chipversion)
except:
logger.error(traceback.format_exc())
self.exp_beadfindArgs = 'justBeadFind --beadfind-minlivesnr 3 --region-size=216x224 --total-timeout 602'
self.exp_analysisArgs = 'Analysis --from-beadfind --clonal-filter-bkgmodel on --region-size=216x224 --bkg-bfmask-update off --gpuWorkLoad 1 --total-timeout 602'
self.libraryKey = "TCAG"
self.tfKey = "ATCG"
logger.info("beadfindArgs: '%s'" % self.exp_beadfindArgs)
logger.info("analysisArgs: '%s'" % self.exp_analysisArgs)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint(
'global', 'block_to_process_start')
self.block_to_process_end = config.getint('global',
'block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise