def newExperiment(explog_path):
'''Create Experiment record'''
# Parse the explog.txt file
text = load_log_path(explog_path)
dict = parse_log(text)
# Create the Experiment Record
folder = os.path.dirname(explog_path)
# Test if Experiment object already exists
try:
newExp = models.Experiment.objects.get(unique=folder)
except:
newExp = None
if newExp is None:
try:
expArgs,st = exp_kwargs(dict,folder)
newExp = models.Experiment(**expArgs)
newExp.save()
except:
newExp = None
print traceback.format_exc()
return newExp
def updateruninfo(request):
'''Replaces crawler.py code
Ensure this function/action is idempotent'''
folder = request.POST.get('datapath')
logger.debug("updateruninfo looking at %s" % folder)
# parse explog.txt.
LOG_BASENAME = "explog.txt"
text = load_log(folder, LOG_BASENAME)
if text is None:
payload = "Cannot read %s" % (os.path.join(folder, LOG_BASENAME))
logger.warn(payload)
return http.HttpResponseServerError(payload)
else:
try:
explog_parsed = parse_log(text)
except:
logger.warn("Error parsing %s, skipping %s" % (LOG_BASENAME, folder))
logger.error(traceback.format_exc())
return http.HttpResponseServerError('%s %s' % (folder, traceback.format_exc()))
# Update experiment, plan, eas objects for dataset in this folder
try:
planObj, expObj, easObj = get_planned_exp_objects(explog_parsed, folder)
update_exp_objects_from_log(explog_parsed, folder, planObj, expObj, easObj)
save_serialized_json(folder, planObj, expObj, easObj)
except:
logger.error(traceback.format_exc())
return http.HttpResponseServerError('%s %s' % (folder, traceback.format_exc()))
return http.HttpResponse("%s database objects have been updated" % folder, mimetype='text/plain')
def newExperiment(explog_path, plan_json=''):
'''Create Experiment record'''
folder = os.path.dirname(explog_path)
# Test if Experiment object already exists
try:
newExp = models.Experiment.objects.get(unique=folder)
except:
newExp = None
if newExp is None:
# Parse the explog.txt file
text = load_log_path(explog_path)
explog = parse_log(text)
explog["planned_run_short_id"] = '' # don't allow getting plan by shortId - other plans may exist with that id
try:
planObj, expObj, easObj = get_planned_exp_objects(explog,folder)
newExp = update_exp_objects_from_log(explog,folder, planObj, expObj, easObj)
if plan_json:
update_plan_info(plan_json, planObj, easObj)
except:
newExp = None
print traceback.format_exc()
return newExp
def update_expobj_ftpcompleted(_expobj, folder):
'''Update Experiment object with completed ftp status'''
# Store final explog in database
exptxt = load_log(folder, LOG_FINAL_BASENAME)
if exptxt is not None:
_expobj.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
return
# Set FTP transfer status to complete
if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
_expobj.ftpStatus = RUN_STATUS_COMPLETE
logger.errors.info("FTP status: Complete")
# Save experiment object
_expobj.save()
# Measure and record disk space usage of this dataset
# This is celery task which will execute in separate worker thread
try:
tasks.setRunDiskspace.delay(_expobj.pk)
except:
logger.errors.error(traceback.format_exc())
raise
def setup_explog(self):
"""This method will find the experiment log and return it's location"""
# First look in raw data directory
original = os.path.join(self.raw_data_dir, EXPLOG_FILENAME)
if not os.path.exists(original):
# Next look in parent of raw data (case:Proton data)
original = os.path.join(os.path.dirname(self.raw_data_dir), EXPLOG_FILENAME)
# Next look in the report directory
if not os.path.exists(original):
original = os.path.join(self.analysis_dir, EXPLOG_FILENAME)
# Next look in the pgm_logs.zip file
if not os.path.exists(original) and os.path.exists(os.path.join(self.results_dir, 'pgm_logs.zip')):
original = os.path.join(self.raw_data_dir, EXPLOG_FILENAME)
with zipfile.ZipFile(os.path.join(self.results_dir, 'pgm_logs.zip'), mode='r') as pgm_zip_hangle:
explog_info = pgm_zip_hangle.getinfo(EXPLOG_FILENAME)
pgm_zip_hangle.extract(explog_info, self.raw_data_dir)
# read in the exp log
with open(original, 'r') as original_handle:
explog = parse_log(original_handle.read())
# HACK ALERT! In order to make sure we don't go over the maximum length of the experiment name (currently 128 characters) by the
# appending of the _foreign string in the from_wells_analysis.py logic, we are going to have to add a check to see if it can fit
# into the data base constraints with the appending of the foreign string
if len(explog['experiment_name'] + "_foreign") > 128:
raise Exception("We cannot transfer this result due to the length of the experiment name.")
return original
def newExperiment(explog_path):
'''Create Experiment record'''
# Parse the explog.txt file
text = load_log_path(explog_path)
dict = parse_log(text)
# Create the Experiment Record
folder = os.path.dirname(explog_path)
# Test if Experiment object already exists
try:
newExp = models.Experiment.objects.get(unique=folder)
except:
newExp = None
if newExp is None:
try:
expArgs, st = exp_kwargs(dict, folder)
newExp = models.Experiment(**expArgs)
newExp.save()
except:
newExp = None
print traceback.format_exc()
return newExp
def update_expobj_ftpcompleted(_expobj, folder):
'''Update Experiment object with completed ftp status'''
# Store final explog in database
exptxt = load_log(folder, LOG_FINAL_BASENAME)
if exptxt is not None:
_expobj.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
return
# Set FTP transfer status to complete
if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
_expobj.ftpStatus = RUN_STATUS_COMPLETE
logger.errors.info("FTP status: Complete")
# Save experiment object
_expobj.save()
# Measure and record disk space usage of this dataset
# This is celery task which will execute in separate worker thread
try:
tasks.setRunDiskspace.delay(_expobj.pk)
except:
logger.errors.error(traceback.format_exc())
raise
def __init__(self,name,config):
self.name = name
self.dat_path = os.path.join(config.get('global','results'), name)
self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
self.status = 'new'
explog_file = os.path.join(self.dat_path, "explog.txt")
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict['oninstranalysis'] #yes, no
self.exp_usesynchdats = 'yes' in self.explogdict['use_synchdats'] #yes, no
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = int(config.get('global','block_to_process_start'))
self.block_to_process_end = int(config.get('global','block_to_process_end'))
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def setup_explog(self):
"""This method will find the experiment log and return it's location"""
# First look in raw data directory
original = os.path.join(self.raw_data_dir, EXPLOG_FILENAME)
if not os.path.exists(original):
# Next look in parent of raw data (case:Proton data)
original = os.path.join(os.path.dirname(self.raw_data_dir), EXPLOG_FILENAME)
# Next look in the report directory
if not os.path.exists(original):
original = os.path.join(self.analysis_dir, EXPLOG_FILENAME)
# Next look in the pgm_logs.zip file
if not os.path.exists(original) and os.path.exists(os.path.join(self.results_dir, 'pgm_logs.zip')):
original = os.path.join(self.raw_data_dir, EXPLOG_FILENAME)
with zipfile.ZipFile(os.path.join(self.results_dir, 'pgm_logs.zip'), mode='r') as pgm_zip_hangle:
explog_info = pgm_zip_hangle.getinfo(EXPLOG_FILENAME)
pgm_zip_hangle.extract(explog_info, self.raw_data_dir)
# read in the exp log
with open(original, 'r') as original_handle:
explog = parse_log(original_handle.read())
# HACK ALERT! In order to make sure we don't go over the maximum length of the experiment name (currently 128 characters) by the
# appending of the _foreign string in the from_wells_analysis.py logic, we are going to have to add a check to see if it can fit
# into the data base constraints with the appending of the foreign string
if len(explog['experiment_name'] + "_foreign") > 128:
raise Exception("We cannot transfer this result due to the length of the experiment name.")
return original
def generate_project_name(exp):
"""generate project name from parsing explog or input"""
if len(sys.argv) > 3:
project_name = sys.argv[3]
else:
fpath = load_log(exp.expDir, 'explog.txt')
explog = parse_log(fpath)
crawler.exp_kwargs(explog, exp.expDir)
project_name = explog.get('project', '')
return project_name
def generate_project_name(exp):
"""generate project name from parsing explog or input"""
if len(sys.argv) > 3:
project_name = sys.argv[3]
else:
fpath = load_log(exp.expDir, "explog.txt")
explog = parse_log(fpath)
crawler.exp_kwargs(explog, exp.expDir)
project_name = explog.get("project", "")
return project_name
def generate_project_name(exp):
"""generate project name from parsing explog or input"""
if len(sys.argv) > 5:
project_name = sys.argv[5]
else:
fpath = load_log(exp.expDir, 'explog.txt')
explog = parse_log(fpath)
#crawler.exp_kwargs(explog, exp.expDir, logger.errors)
project_name = explog.get('project', '')
return project_name
def newExperiment(_explog_path, _plan_json=''):
'''Create Experiment record'''
folder = os.path.dirname(_explog_path)
# Test if Experiment object already exists
try:
_newExp = models.Experiment.objects.get(unique=folder)
print("DEBUG: Experiment exists in database: %s" % folder)
except:
print("DEBUG: Experiment does not exist in database")
_newExp = None
if _newExp is None:
# Parse the explog.txt file
text = load_log_path(_explog_path)
explog = parse_log(text)
explog[
"planned_run_short_id"] = '' # don't allow getting plan by shortId - other plans may exist with that id
try:
ret_val = generate_updateruninfo_post(folder, logger)
if ret_val == "Generated POST":
# Get experiment object
exp_set = models.Experiment.objects.filter(unique=folder)
if exp_set:
# Experiment object exists in database
_newExp = exp_set[0]
else:
print(
"ERROR: Could not update/generate new Experiment record in database"
)
print(ret_val)
return None
# Append to expName to indicate imported dataset.
_newExp.expName += "_foreign"
chefLog_parsed = isChefInfoAvailable(folder)
if chefLog_parsed:
update_chefSummary(_newExp, chefLog_parsed)
_newExp.save()
if _plan_json:
planObj = _newExp.plan
easObj = _newExp.get_EAS()
update_plan_info(_plan_json, planObj, easObj)
except:
print("DEBUG: There was an error adding the experiment")
_newExp = None
print(traceback.format_exc())
return _newExp
def newExperiment(_explog_path, _plan_json=''):
'''Create Experiment record'''
folder = os.path.dirname(_explog_path)
# Test if Experiment object already exists
try:
_newExp = models.Experiment.objects.get(unique=folder)
print("DEBUG: Experiment exists in database: %s" % folder)
except:
print("DEBUG: Experiment does not exist in database")
_newExp = None
if _newExp is None:
# Parse the explog.txt file
text = load_log_path(_explog_path)
explog = parse_log(text)
explog[
"planned_run_short_id"] = '' # don't allow getting plan by shortId - other plans may exist with that id
try:
ret_val = generate_updateruninfo_post(folder, logger)
if ret_val == "Generated POST":
# Get experiment object
exp_set = models.Experiment.objects.filter(unique=folder)
if exp_set:
# Experiment object exists in database
_newExp = exp_set[0]
else:
print("ERROR: Could not update/generate new Experiment record in database")
print(ret_val)
return None
# Append to expName to indicate imported dataset.
_newExp.expName += "_foreign"
chefLog_parsed = isChefInfoAvailable(folder)
if chefLog_parsed:
update_chefSummary(_newExp, chefLog_parsed)
_newExp.save()
except:
print("DEBUG: There was an error adding the experiment")
_newExp = None
print(traceback.format_exc())
if _plan_json:
update_plan_info(_plan_json, _newExp.plan, _newExp.get_EAS())
return _newExp
def newExperiment(_explog_path, _plan_json=""):
"""Create Experiment record"""
folder = os.path.dirname(_explog_path)
# Test if Experiment object already exists
try:
_newExp = models.Experiment.objects.get(unique=folder)
print "DEBUG: Experiment exists in database: %s" % (folder)
except:
print "DEBUG: Experiment does not exist in database"
_newExp = None
if _newExp is None:
# Parse the explog.txt file
text = load_log_path(_explog_path)
explog = parse_log(text)
explog["planned_run_short_id"] = "" # don't allow getting plan by shortId - other plans may exist with that id
try:
ret_val = generate_updateruninfo_post(folder, logger)
if ret_val == "Generated POST":
# Get experiment object
exp_set = models.Experiment.objects.filter(unique=folder)
if exp_set:
# Experiment object exists in database
_newExp = exp_set[0]
else:
print "ERROR: Could not update/generate new Experiment record in database"
print ret_val
return None
# Append to expName to indicate imported dataset.
_newExp.expName += "_foreign"
_newExp.save()
if _plan_json:
planObj = _newExp.plan
easObj = _newExp.get_EAS()
update_plan_info(_plan_json, planObj, easObj)
except:
print "DEBUG: There was an error adding the experiment"
_newExp = None
print traceback.format_exc()
return _newExp
def update_expobj_ftpcompleted(_expobj, folder):
'''Update Experiment object with completed ftp status'''
# Store final explog in database
exptxt = load_log(folder, LOG_FINAL_BASENAME)
if exptxt is not None:
_expobj.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
return
# Set FTP transfer status to complete
if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
_expobj.ftpStatus = RUN_STATUS_COMPLETE
logger.errors.info("FTP status: Complete")
# Save experiment object
_expobj.save()
return
def update_expobj_ftpcompleted(_expobj, folder):
'''Update Experiment object with completed ftp status'''
# Store final explog in database
exptxt = load_log(folder, LOG_FINAL_BASENAME)
if exptxt is not None:
_expobj.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
return
# Set FTP transfer status to complete
if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
_expobj.ftpStatus = RUN_STATUS_COMPLETE
logger.errors.info("FTP status: Complete")
# Save experiment object
_expobj.save()
return
def updateruninfo(request):
"""Replaces crawler.py code
Ensure this function/action is idempotent"""
folder = request.POST.get("datapath")
logger.debug("updateruninfo looking at %s" % folder)
# parse explog.txt.
LOG_BASENAME = "explog.txt"
text = load_log(folder, LOG_BASENAME)
if text is None:
payload = "Cannot read %s" % (os.path.join(folder, LOG_BASENAME))
logger.warn(payload)
return http.HttpResponseServerError(payload)
else:
try:
explog_parsed = parse_log(text)
except Exception:
logger.warn("Error parsing %s, skipping %s" %
(LOG_BASENAME, folder))
logger.error(traceback.format_exc())
return http.HttpResponseServerError(
"%s %s" % (folder, traceback.format_exc()))
# Update experiment, plan, eas objects for dataset in this folder
try:
planObj, expObj, easObj = get_planned_exp_objects(
explog_parsed, folder)
update_exp_objects_from_log(explog_parsed, folder, planObj, expObj,
easObj)
save_serialized_json(folder, planObj, expObj, easObj)
except Exception:
logger.error(traceback.format_exc())
return http.HttpResponseServerError(
"%s %s" % (folder, traceback.format_exc()))
return http.HttpResponse("%s database objects have been updated" %
folder,
mimetype="text/plain")
def __init__(self,results,name,config):
self.name = name
self.dat_path = os.path.join(results, name)
self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
self.status = 'new'
explog_file = os.path.join(results, name, "explog.txt")
if os.path.exists(explog_file):
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict['oninstranalysis']#yes, no
self.exp_usesynchdats = 'yes' in self.explogdict['use_synchdats']#yes, no
else:
self.explogdict = None
self.explog_file = None
self.exp_flows = None
self.exp_oninstranalysis = None
self.exp_usesynchdats = None
self.blocks = []
if self.explogdict:
try:
self.discover_blocks(0,95)#TODO block_to_process_start,block_to_process_end)
except:
logger.error(traceback.format_exc())
pass
else:
logger.error(traceback.format_exc())
def __init__(self,name,config):
self.name = name
self.dat_path = os.path.join(config.get('global','results'), name)
self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
self.status = 'new'
explog_file = os.path.join(self.dat_path, "explog.txt")
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
self.exp_planned_run_short_id = self.explogdict.get('planned_run_short_id','')
self.exp_chiptype = self.explogdict["chiptype"]
logger.info('chiptype:-%s-' % self.exp_chiptype)
self.exp_runtype = self.explogdict["runtype"]
logger.info('runtype:-%s-' % self.exp_runtype)
'''
if os.path.exists(os.path.join(self.dat_path, "chips.json")):
chips = json.load(open('os.path.join(self.dat_path, "chips.json")','r'))
else:
logger.error('use default args /usr/share/ion/oia/chips.json')
chips = json.load(open('/usr/share/ion/oia/chips.json','r'))
'''
if os.path.exists('/software/config/DataCollect.config'):
f = open('/software/config/DataCollect.config')
text = f.readlines()
f.close()
for line in text:
[key, value] = line.split(':')
key = key.strip()
value = value.strip()
logger.info('%s %s' % (key, value))
if key == 'TSUrl':
TSUrl = value
elif key == 'TSUserName':
TSUserName = value
elif key == 'TSPasswd':
TSPasswd = value
else:
continue
else:
raise
logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd))
headers = {
"Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
"Content-type": "application/json",
"Accept": "application/json" }
# get chip data
conn = httplib.HTTPConnection(TSUrl)
conn.request("GET", "/rundb/api/v1/chip/?format=json", "", headers)
response = conn.getresponse()
data = response.read()
chips = json.loads(data)
# get plan data
conn = httplib.HTTPConnection(TSUrl)
conn.request("GET", "/rundb/api/v1/plannedexperiment/?format=json", "", headers)
response = conn.getresponse()
logger.info('%s %s' % (response.status, response.reason))
data = response.read()
# Need to limit plans
#logger.debug(data)
plans = json.loads(data)
for chip in chips['objects']:
logger.info(chip)
logger.info('chiptype test: test normal chiptype: [%s]' % self.exp_chiptype)
if chip['name']==self.exp_chiptype:
logger.info('chiptype test: found normal chiptype: [%s]' % chip['name'])
self.exp_beadfindArgs = chip['beadfindargs']
self.exp_analysisArgs = chip['analysisargs']
logger.info('chiptype test: test special chiptype/runtype: [%s%s]' % (self.exp_chiptype,self.exp_runtype))
if chip['name']==self.exp_chiptype+self.exp_runtype:
logger.info('chiptype test: found special chiptype/runtype: [%s]' % chip['name'])
self.exp_beadfindArgs = chip['beadfindargs']
self.exp_analysisArgs = chip['analysisargs']
break
logger.info('beadfindArgs:-%s-' % self.exp_beadfindArgs)
logger.info('analysisArgs:-%s-' % self.exp_analysisArgs)
plan = [plan for plan in plans['objects'] if plan.get('planShortID','not_available') == self.exp_planned_run_short_id]
if plan:
logger.debug("Plan: %s" % plan[0])
self.libraryKey=plan[0].get('libraryKey','TCAG') # TODO
self.tfKey=plan[0].get('tfKey','ATCG') # TODO
else:
logger.debug("no plan")
self.libraryKey="TCAG"
self.tfKey="ATCG"
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint('global','block_to_process_start')
self.block_to_process_end = config.getint('global','block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def __init__(self,name,config):
self.name = name
self.dat_path = os.path.join(config.get('global','results'), name)
self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
self.status = 'new'
self.last_flow = -1
explog_file = os.path.join(self.dat_path, "explog.txt")
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
self.exp_planned_run_short_id = self.explogdict.get('planned_run_short_id','')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_chipversion = self.explogdict.get('chipversion','')
if not self.exp_chipversion or not self.exp_chipversion.startswith('P'):
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_short_id: '%s'" % self.exp_planned_run_short_id)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
if os.path.exists('/software/config/DataCollect.config'):
f = open('/software/config/DataCollect.config')
text = f.readlines()
f.close()
for line in text:
[key, value] = line.split(':')
key = key.strip()
value = value.strip()
logger.info('%s %s' % (key, value))
if key == 'TSUrl':
TSUrl = value
elif key == 'TSUserName':
TSUserName = value
elif key == 'TSPasswd':
TSPasswd = value
else:
continue
else:
raise
logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd))
headers = {
"Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
"Content-type": "application/json",
"Accept": "application/json" }
try:
plan_found = False
if self.exp_planned_run_short_id:
# get plan data
conn = httplib.HTTPConnection(TSUrl)
conn.request("GET", "/rundb/api/v1/plannedexperiment/?format=json&planShortID=%s&limit=20" % self.exp_planned_run_short_id, "", headers)
response = conn.getresponse()
logger.info('%s %s' % (response.status, response.reason))
data = response.read()
# Need to limit plans
plans = json.loads(data)
logger.debug("number of plans found on %s: %s" % (TSUrl,len(plans['objects'])))
logger.debug("search for plan with planShortID: '%s'" % self.exp_planned_run_short_id)
plan = [plan for plan in plans['objects'] if plan.get('planShortID','not_available') == self.exp_planned_run_short_id]
logger.debug("number of plans with planShortID '%s' found: %s" % (self.exp_planned_run_short_id,len(plan)))
if plan:
logger.debug("Plan: %s" % plan[0])
self.exp_beadfindArgs = plan[0].get('beadfindargs','justBeadFind')
self.exp_analysisArgs = plan[0].get('analysisargs','Analysis')
self.libraryKey = plan[0].get('libraryKey','TCAG')
self.tfKey = plan[0].get('tfKey','ATCG')
plan_found = True
else:
logger.debug("Problem with args in plan '%s'" % self.exp_planned_run_short_id)
if not plan_found:
logger.debug("plan not available")
# TODO: user might starts run during TS upgrade
conn = httplib.HTTPConnection(TSUrl)
conn.request("GET", "/rundb/api/v1/analysisargs/?format=json", "", headers)
response = conn.getresponse()
data = response.read()
allanalysisargs = json.loads(data)
chip_found = False
logger.info('chipversion test: search for chipversion: [%s]' % self.exp_chipversion)
for analysisargs in allanalysisargs['objects']:
logger.info('chipversion test: compare with chipversion: [%s]' % analysisargs['name'])
if analysisargs['name']=="default_"+self.exp_chipversion:
logger.info('chipversion test: found analysis args for chipversion: [%s]' % analysisargs['name'])
logger.info(analysisargs)
self.exp_beadfindArgs = analysisargs['beadfindargs']
self.exp_analysisArgs = analysisargs['analysisargs']
self.libraryKey = 'TCAG'
self.tfKey = 'ATCG'
chip_found = True
break
if not chip_found:
raise Exception("Chiptype %s not found" % self.exp_chipversion)
except:
logger.error(traceback.format_exc())
self.exp_beadfindArgs = 'justBeadFind --beadfind-minlivesnr 3 --region-size=216x224 --total-timeout 602'
self.exp_analysisArgs = 'Analysis --from-beadfind --clonal-filter-bkgmodel on --region-size=216x224 --bkg-bfmask-update off --gpuWorkLoad 1 --total-timeout 602'
self.libraryKey = "TCAG"
self.tfKey = "ATCG"
logger.info("beadfindArgs: '%s'" % self.exp_beadfindArgs)
logger.info("analysisArgs: '%s'" % self.exp_analysisArgs)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint('global','block_to_process_start')
self.block_to_process_end = config.getint('global','block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def crawl(folders, logger):
"""Crawl over ``folders``, reporting information to the ``CrawlLog``
``logger``."""
if folders:
logger.errors.info("checking %d directories" % len(folders))
for f in folders:
try:
text = load_log(f, LOG_BASENAME)
logger.errors.debug("checking directory %s" % f)
#------------------------------------
# Check for a valid explog.txt file
#------------------------------------
if text is not None:
logger.current_folder = f
# Catch any errors parsing this log file and continue to the next experiment.
try:
d = parse_log(text)
except:
logger.errors.info("Error parsing explog, skipping %s" % f)
logger.errors.exception(traceback.format_exc())
continue
exp = None
exp_set = models.Experiment.objects.filter(unique=f)
if exp_set:
# Experiment already in database
exp = exp_set[0]
if exp.ftpStatus == RUN_STATUS_COMPLETE:
continue
else:
# Need to create experiment for this folder
try:
planObj, expObj, easObj = get_planned_exp_objects(d, f)
exp = update_exp_objects_from_log(
d, f, planObj, expObj, easObj)
except:
logger.errors.info(
"Error updating experiment objects, skipping %s" %
f)
logger.errors.exception(traceback.format_exc())
continue
if exp is not None:
logger.errors.info("%s" % f)
#--------------------------------
# Update FTP Transfer Status
#--------------------------------
if check_for_completion(exp):
exptxt = load_log(f, LOG_FINAL_BASENAME)
if exptxt is not None:
exp.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
continue
# FTP transfer is complete
if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
exp.ftpStatus = RUN_STATUS_COMPLETE
exp.save()
# Measure and record disk space usage of this dataset
# This is celery task which will execute in separate worker thread
try:
tasks.setRunDiskspace.delay(exp.pk)
except:
logger.errors.exception(traceback.format_exc())
else:
if exp.ftpStatus != RUN_STATUS_MISSING:
exp.ftpStatus = get_percent(exp)
exp.save()
logger.errors.info("FTP status: Transferring")
#--------------------------------
# Handle auto-run analysis
#--------------------------------
composite_exists, thumbnail_exists = reports_exist(exp)
logger.errors.debug("Reports Exist? %s %s" %
(composite_exists, thumbnail_exists))
if not composite_exists:
logger.errors.debug(
"No auto-run analysis has been started")
# Check if auto-run for whole chip has been requested
if exp.autoAnalyze:
logger.errors.debug(
" auto-run whole chip analysis has been requested"
)
#if check_analyze_early(exp) or check_for_completion(exp):
if ready_to_process(exp) or check_for_completion(
exp):
logger.errors.info(
" Start a whole chip auto-run analysis job"
)
generate_http_post(exp)
else:
logger.errors.info(
" Do not start a whole chip auto-run job yet"
)
else:
logger.errors.debug(
" auto-run whole chip analysis has not been requested"
)
else:
logger.errors.debug("composite report already exists")
if not thumbnail_exists:
# Check if auto-run for thumbnail has been requested
# But only if its a block dataset
if 'tiled' in exp.rawdatastyle:
if autorun_thumbnail(exp):
logger.errors.debug(
"auto-run thumbnail analysis has been requested"
)
if ready_to_process_thumbnail(
exp) or check_for_completion(exp):
logger.errors.info(
" Start a thumbnail auto-run analysis job"
)
generate_http_post(exp, DO_THUMBNAIL)
else:
logger.errors.info(
" Do not start a thumbnail auto-run job yet"
)
else:
logger.errors.debug(
"auto-run thumbnail analysis has not been requested"
)
else:
logger.errors.debug(
"This is not a block dataset; no thumbnail to process"
)
else:
logger.errors.debug("thumbnail report already exists")
else:
logger.errors.debug("exp is empty")
else:
# No explog.txt exists; probably an engineering test run
logger.errors.debug("no %s was found in %s" %
(LOG_BASENAME, f))
except:
logger.errors.exception(traceback.format_exc())
logger.current_folder = '(none)'
def __init__(self,name,config):
self.name = name
self.dat_path = os.path.join(config.get('global','results'), name)
self.analysis_path = os.path.join(config.get('global','analysisresults'), name)
self.sigproc_results_path = os.path.join(config.get('global','analysisresults'), name, "onboard_results", "sigproc_results")
self.status = 'new'
self.last_flow = -1
try:
if not os.path.exists( self.sigproc_results_path ):
os.makedirs( self.sigproc_results_path )
except:
logger.error(traceback.format_exc())
explog_file = os.path.join(self.analysis_path, "explog.txt")
if not os.path.exists(explog_file):
shutil.copyfile(os.path.join(self.dat_path, "explog.txt"), os.path.join(self.analysis_path, "explog.txt"))
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
if not os.path.exists( os.path.join(self.dat_path, 'onboard_results') ):
os.symlink( os.path.join(self.analysis_path,'onboard_results'), os.path.join(self.dat_path,'onboard_results') )
except:
logger.error(traceback.format_exc())
try:
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
self.exp_planned_run_guid = self.explogdict.get('planned_run_guid','')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_seqkitplanname = self.explogdict.get('seqkitplanname','')
self.exp_chipversion = self.explogdict.get('chipversion','')
if not self.exp_chipversion:
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
try:
f = open(os.path.join(self.dat_path, "planned_run.json"),'r')
analysisargs = json.load(f)['objects'][0]
f.close()
if analysisargs:
self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs']
self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs']
self.exp_beadfindArgs_block = analysisargs['beadfindargs']
self.exp_analysisArgs_block = analysisargs['analysisargs']
self.libraryKey = analysisargs['libraryKey']
self.tfKey = analysisargs['tfKey']
else:
raise Exception("Analysis args are empty")
except:
raise
logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail)
logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail)
logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block)
logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint('global','block_to_process_start')
self.block_to_process_end = config.getint('global','block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def crawl(folders, logger):
"""Crawl over ``folders``, reporting information to the ``CrawlLog``
``logger``."""
if folders:
logger.errors.info("checking %d directories" % len(folders))
for f in folders:
try:
text = load_log(f, LOG_BASENAME)
logger.errors.debug("checking directory %s" % f)
#------------------------------------
# Check for a valid explog.txt file
#------------------------------------
if text is not None:
logger.current_folder = f
# Catch any errors parsing this log file and continue to the next experiment.
d = parse_log(text)
kwargs, status = exp_kwargs(d, f)
if status is True:
exp = exp_from_kwargs(kwargs, logger, False)
projectName = d.get('project', '') # part of explog but not a field in Experiment table
if exp is not None:
logger.errors.info("%s" % f)
#make sure 1-1 exp-plan relationship is enforced when new experiment is created
planObj = exp.plan
if planObj and exp.pk == None and len(models.Experiment.objects.filter(plan__pk = planObj.pk)) > 0:
logger.errors.info('WARNING: Plan %s is already associated to an experiment, a copy will be created' % planObj.pk)
planObj.pk = None
planObj.save()
exp.plan = planObj
#--------------------------------
# Update FTP Transfer Status
#--------------------------------
if check_for_completion(exp):
# FTP transfer is complete
if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
exp.ftpStatus = RUN_STATUS_COMPLETE
exp.log = json.dumps(parse_log(load_log(f, LOG_FINAL_BASENAME)), indent=4)
exp.save()
# Measure and record disk space usage of this dataset
# This is celery task which will execute in separate worker thread
try:
tasks.setRunDiskspace.delay(exp.pk)
except:
logger.errors.exception(traceback.format_exc())
else:
if exp.ftpStatus != RUN_STATUS_MISSING:
exp.ftpStatus = get_percent(exp)
exp.save()
logger.errors.info("FTP status: Transferring")
#--------------------------------
# Handle auto-run analysis
#--------------------------------
composite_exists, thumbnail_exists = reports_exist(exp)
logger.errors.debug("Reports Exist? %s %s" % (composite_exists, thumbnail_exists))
if not composite_exists:
logger.errors.debug("No auto-run analysis has been started")
# Check if auto-run for whole chip has been requested
if exp.autoAnalyze:
logger.errors.debug(" auto-run whole chip analysis has been requested")
#if check_analyze_early(exp) or check_for_completion(exp):
if ready_to_process(exp) or check_for_completion(exp):
logger.errors.info(" Start a whole chip auto-run analysis job")
generate_http_post(exp, projectName)
else:
logger.errors.info(" Do not start a whole chip auto-run job yet")
else:
logger.errors.debug(" auto-run whole chip analysis has not been requested")
else:
logger.errors.debug("composite report already exists")
if not thumbnail_exists:
# Check if auto-run for thumbnail has been requested
# But only if its a block dataset
if 'tiled' in exp.rawdatastyle:
if autorun_thumbnail(exp):
logger.errors.debug("auto-run thumbnail analysis has been requested")
if ready_to_process_thumbnail(exp) or check_for_completion(exp):
logger.errors.info(" Start a thumbnail auto-run analysis job")
generate_http_post(exp, projectName, DO_THUMBNAIL)
else:
logger.errors.info(" Do not start a thumbnail auto-run job yet")
else:
logger.errors.debug("auto-run thumbnail analysis has not been requested")
else:
logger.errors.debug("This is not a block dataset; no thumbnail to process")
else:
logger.errors.debug("thumbnail report already exists")
else:
logger.errors.debug("exp is empty")
else:
logger.errors.warn("folder %s is not added due to invalid experiment status from exp_kwargs. " % f)
else:
# No explog.txt exists; probably an engineering test run
logger.errors.debug("no %s was found in %s" % (LOG_BASENAME, f))
except:
logger.errors.exception(traceback.format_exc())
logger.current_folder = '(none)'
def __init__(self, name, config):
self.name = name
self.dat_path = os.path.join(config.get('global', 'results'), name)
self.analysis_path = os.path.join(
config.get('global', 'analysisresults'), name)
self.sigproc_results_path = os.path.join(
config.get('global', 'analysisresults'), name, "onboard_results",
"sigproc_results")
self.status = 'new'
self.last_flow = -1
try:
if not os.path.exists(self.sigproc_results_path):
os.makedirs(self.sigproc_results_path)
except:
logger.error(traceback.format_exc())
explog_file = os.path.join(self.analysis_path, "explog.txt")
if not os.path.exists(explog_file):
shutil.copyfile(os.path.join(self.dat_path, "explog.txt"),
os.path.join(self.analysis_path, "explog.txt"))
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
if not os.path.exists(
os.path.join(self.dat_path, 'onboard_results')):
os.symlink(os.path.join(self.analysis_path, 'onboard_results'),
os.path.join(self.dat_path, 'onboard_results'))
except:
logger.error(traceback.format_exc())
try:
self.explog_file = explog_file
# parse explog.txt
(head, tail) = os.path.split(explog_file)
explogtext = load_log(head, tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get(
'oninstranalysis', 'no')
self.exp_usesynchdats = 'yes' in self.explogdict.get(
'use_synchdats', 'no')
self.exp_oia_during_run = 'yes' in self.explogdict.get(
'oia_during_run', 'yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get(
'ecc_enabled', 'no')
self.exp_planned_run_guid = self.explogdict.get(
'planned_run_guid', '')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_seqkitplanname = self.explogdict.get('seqkitplanname', '')
self.exp_chipversion = self.explogdict.get('chipversion', '')
if not self.exp_chipversion or not self.exp_chipversion.startswith(
'P'):
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
try:
logger.info('connect to:%s user:%s password:%s' %
(TSUrl, TSUserName, TSPasswd))
headers = {
"Authorization":
"Basic " + string.strip(
base64.encodestring(TSUserName + ':' + TSPasswd)),
"Content-type":
"application/json",
"Accept":
"application/json"
}
conn = httplib.HTTPConnection(TSUrl)
if self.exp_planned_run_guid:
# TODO, use original plan data from file system
request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&planGUID=%s&sequenceKitName=%s&limit=20" % (
self.exp_chipversion, self.exp_planned_run_guid,
self.exp_seqkitplanname)
else:
logger.debug("use default chip args")
request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&sequenceKitName=%s&limit=20" % (
self.exp_chipversion, self.exp_seqkitplanname)
#http://frenzy.ite/rundb/api/v1/analysisargs/getargs/?format=json&chipType=P1.2.18&planGUID=&sequenceKitName=ProtonI200Kit-v2
logger.info('request: %s http://%s%s' % (self.explogdict.get(
'runname', 'unknown'), TSUrl, request))
conn.request("GET", request, "", headers)
response = conn.getresponse()
logger.info('%s %s' % (response.status, response.reason))
data = response.read()
analysisargs = json.loads(data)
logger.debug("Plan: %s" % analysisargs)
f = open(
os.path.join(self.sigproc_results_path,
"retrieved_args.json"), 'w')
f.write(json.dumps(analysisargs, indent=4))
f.close()
if analysisargs:
self.exp_beadfindArgs_thumbnail = analysisargs[
'thumbnailbeadfindargs']
self.exp_analysisArgs_thumbnail = analysisargs[
'thumbnailanalysisargs']
self.exp_beadfindArgs_block = analysisargs['beadfindargs']
self.exp_analysisArgs_block = analysisargs['analysisargs']
self.libraryKey = 'TCAG'
self.tfKey = 'ATCG'
else:
raise Exception("Analysis args are empty")
except:
raise
logger.info("thumbnail beadfindArgs: '%s'" %
self.exp_beadfindArgs_thumbnail)
logger.info("thumbnail analysisArgs: '%s'" %
self.exp_analysisArgs_thumbnail)
logger.info("block beadfindArgs: '%s'" %
self.exp_beadfindArgs_block)
logger.info("block analysisArgs: '%s'" %
self.exp_analysisArgs_block)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint(
'global', 'block_to_process_start')
self.block_to_process_end = config.getint('global',
'block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
#!/usr/bin/env python
# Copyright (C) 2013 Ion Torrent Systems, Inc. All Rights Reserved
'''Parses explog.txt and explog_final.txt'''
from ion.utils.explogparser import load_log_path
from ion.utils.explogparser import parse_log
explog_path = "./explog.txt"
# Parse the explog.txt file
text = load_log_path(explog_path)
dict = parse_log(text)
#NOTE: keywords will have spaces replaced with underscores and all letters lowercased.
keywords = [
"experimenterrorlog",
"experimentinfolog",
"explog_done",
"experiment_name",
"f",
]
for key, value in dict.items():
if any(key.startswith(item) for item in keywords):
print "(%s) ==> %s" % (key, value)
def __init__(self, name, config):
self.name = name
self.dat_path = os.path.join(config.get('global', 'results'), name)
self.analysis_path = os.path.join(config.get('global', 'analysisresults'), name)
self.sigproc_results_path = os.path.join(
config.get('global', 'analysisresults'), name, "onboard_results", "sigproc_results")
self.status = 'new'
self.last_flow = -1
try:
if not os.path.exists(self.sigproc_results_path):
os.makedirs(self.sigproc_results_path)
except:
logger.error(traceback.format_exc())
explog_file = os.path.join(self.analysis_path, "explog.txt")
if not os.path.exists(explog_file):
shutil.copyfile(os.path.join(self.dat_path, "explog.txt"),
os.path.join(self.analysis_path, "explog.txt"))
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
if not os.path.exists(os.path.join(self.dat_path, 'onboard_results')):
os.symlink(os.path.join(self.analysis_path, 'onboard_results'),
os.path.join(self.dat_path, 'onboard_results'))
except:
logger.error(traceback.format_exc())
try:
self.explog_file = explog_file
# parse explog.txt
(head, tail) = os.path.split(explog_file)
explogtext = load_log(head, tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis', 'no')
self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats', 'no')
self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run', 'yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled', 'no')
self.exp_planned_run_guid = self.explogdict.get('planned_run_guid', '')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_seqkitplanname = self.explogdict.get('seqkitplanname', '')
self.exp_chipversion = self.explogdict.get('chipversion', '')
if not self.exp_chipversion:
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
try:
analysisargs = {}
if os.path.exists(os.path.join(self.dat_path, "reanalysis_args.json")):
f = open(os.path.join(self.dat_path, "reanalysis_args.json"), 'r')
analysisargs = json.load(f)
f.close()
logger.info("use analysisargs from %s" %
os.path.join(self.dat_path, "reanalysis_args.json"))
elif os.path.exists(os.path.join(self.dat_path, "planned_run.json")):
f = open(os.path.join(self.dat_path, "planned_run.json"), 'r')
analysisargs = json.load(f)['objects'][0]
f.close()
logger.info("use analysisargs from %s" % os.path.join(self.dat_path, "planned_run.json"))
else:
logger.info("trying to fetch planned run for %s" %
os.path.join(self.dat_path, "planned_run.json"))
http_command = "python /software/testing/get_reanalysis_args.py -e %s" % (name)
args = shlex.split(http_command.encode('utf8'))
logger.info("run process: %s" % (args))
outfile = open(os.path.join(self.dat_path, 'get_reanalysis_args_oia.log'), "w")
p = subprocess.Popen(args, stdout=outfile, stderr=subprocess.STDOUT)
# add popen process to block
ret = p.wait()
outfile.close()
if os.path.exists(os.path.join(self.dat_path, "planned_run.json")):
with open(os.path.join(self.dat_path, "planned_run.json"), 'r') as f:
try:
analysisargs = json.load(f)['objects'][0]
logger.info("Using Plan information from %s" %
os.path.join(self.dat_path, "planned_run.json"))
except IndexError:
raise Exception("File %s is empty",
os.path.join(self.dat_path, "planned_run.json"))
if len(analysisargs) > 0:
self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs']
self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs']
self.exp_beadfindArgs_block = analysisargs['beadfindargs']
self.exp_analysisArgs_block = analysisargs['analysisargs']
self.exp_prebasecallerArgs_block = analysisargs['prebasecallerargs']
self.libraryKey = analysisargs['libraryKey']
self.tfKey = analysisargs['tfKey']
else:
raise Exception("There are no Analysis args available")
except:
raise
logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail)
logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail)
logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block)
logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block)
logger.info("block prebasecallerArgs: '%s'" % self.exp_prebasecallerArgs_block)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint('global', 'block_to_process_start')
self.block_to_process_end = config.getint('global', 'block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
#!/usr/bin/env python
# Copyright (C) 2013 Ion Torrent Systems, Inc. All Rights Reserved
'''Parses explog.txt and explog_final.txt'''
from ion.utils.explogparser import load_log_path
from ion.utils.explogparser import parse_log
explog_path = "./explog.txt"
# Parse the explog.txt file
text = load_log_path(explog_path)
dict = parse_log(text)
#NOTE: keywords will have spaces replaced with underscores and all letters lowercased.
keywords = [
"experimenterrorlog",
"experimentinfolog",
"explog_done",
"experiment_name",
"f",
]
for key,value in dict.items():
if any(key.startswith(item) for item in keywords):
print "(%s) ==> %s" % (key, value)
def crawl(folders, logger):
"""Crawl over ``folders``, reporting information to the ``CrawlLog``
``logger``."""
if folders:
logger.errors.info("checking %d directories" % len(folders))
for f in folders:
try:
text = load_log(f, LOG_BASENAME)
logger.errors.debug("checking directory %s" % f)
#------------------------------------
# Check for a valid explog.txt file
#------------------------------------
if text is not None:
logger.current_folder = f
# Catch any errors parsing this log file and continue to the next experiment.
try:
d = parse_log(text)
except:
logger.errors.info("Error parsing explog, skipping %s" % f)
logger.errors.exception(traceback.format_exc())
continue
exp = None
exp_set = models.Experiment.objects.filter(unique=f)
if exp_set:
# Experiment already in database
exp = exp_set[0]
if exp.ftpStatus == RUN_STATUS_COMPLETE:
continue
else:
# Need to create experiment for this folder
try:
planObj, expObj, easObj = get_planned_exp_objects(d, f, logger.errors)
exp = update_exp_objects_from_log(d, f, planObj, expObj, easObj, logger.errors)
except:
logger.errors.info("Error updating experiment objects, skipping %s" % f)
logger.errors.exception(traceback.format_exc())
continue
if exp is not None:
logger.errors.info("%s" % f)
#--------------------------------
# Update FTP Transfer Status
#--------------------------------
if check_for_completion(exp):
exptxt = load_log(f, LOG_FINAL_BASENAME)
if exptxt is not None:
exp.log = json.dumps(parse_log(exptxt), indent=4)
else:
# explog_final exists, but is not readable yet.
continue
# FTP transfer is complete
if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT:
logger.errors.info("FTP status: Aborted")
else:
exp.ftpStatus = RUN_STATUS_COMPLETE
exp.save()
# Measure and record disk space usage of this dataset
# This is celery task which will execute in separate worker thread
try:
tasks.setRunDiskspace.delay(exp.pk)
except:
logger.errors.exception(traceback.format_exc())
else:
if exp.ftpStatus != RUN_STATUS_MISSING:
exp.ftpStatus = get_filecount(exp)
exp.save()
logger.errors.info("FTP status: Transferring")
#--------------------------------
# Handle auto-run analysis
#--------------------------------
composite_exists, thumbnail_exists = reports_exist(exp)
logger.errors.debug("Reports Exist? %s %s" % (composite_exists, thumbnail_exists))
if not composite_exists:
logger.errors.debug("No auto-run analysis has been started")
# Check if auto-run for whole chip has been requested
if exp.autoAnalyze:
logger.errors.debug(" auto-run whole chip analysis has been requested")
#if check_analyze_early(exp) or check_for_completion(exp):
if ready_to_process(exp) or check_for_completion(exp):
logger.errors.info(" Start a whole chip auto-run analysis job")
generate_http_post(exp, logger)
else:
logger.errors.info(" Do not start a whole chip auto-run job yet")
else:
logger.errors.debug(" auto-run whole chip analysis has not been requested")
else:
logger.errors.debug("composite report already exists")
if not thumbnail_exists:
# Check if auto-run for thumbnail has been requested
# But only if its a block dataset
if 'tiled' in exp.rawdatastyle:
if autorun_thumbnail(exp, logger):
logger.errors.debug("auto-run thumbnail analysis has been requested")
if ready_to_process_thumbnail(exp) or check_for_completion(exp):
logger.errors.info(" Start a thumbnail auto-run analysis job")
generate_http_post(exp, logger, DO_THUMBNAIL)
else:
logger.errors.info(" Do not start a thumbnail auto-run job yet")
else:
logger.errors.debug("auto-run thumbnail analysis has not been requested")
else:
logger.errors.debug("This is not a block dataset; no thumbnail to process")
else:
logger.errors.debug("thumbnail report already exists")
else:
logger.errors.debug("exp is empty")
else:
# No explog.txt exists; probably an engineering test run
logger.errors.debug("no %s was found in %s" % (LOG_BASENAME, f))
except:
logger.errors.exception(traceback.format_exc())
logger.current_folder = '(none)'
def __init__(self,name,config):
self.name = name
self.dat_path = os.path.join(config.get('global','results'), name)
self.analysis_path = os.path.join(config.get('global','analysisresults'), name)
self.sigproc_results_path = os.path.join(config.get('global','analysisresults'), name, "onboard_results", "sigproc_results")
self.status = 'new'
self.last_flow = -1
try:
if not os.path.exists( self.sigproc_results_path ):
os.makedirs( self.sigproc_results_path )
except:
logger.error(traceback.format_exc())
explog_file = os.path.join(self.analysis_path, "explog.txt")
if not os.path.exists(explog_file):
shutil.copyfile(os.path.join(self.dat_path, "explog.txt"), os.path.join(self.analysis_path, "explog.txt"))
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
if not os.path.exists( os.path.join(self.dat_path, 'onboard_results') ):
os.symlink( os.path.join(self.analysis_path,'onboard_results'), os.path.join(self.dat_path,'onboard_results') )
except:
logger.error(traceback.format_exc())
try:
self.explog_file = explog_file
# parse explog.txt
(head,tail) = os.path.split(explog_file)
explogtext = load_log(head,tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
self.exp_planned_run_guid = self.explogdict.get('planned_run_guid','')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_seqkitplanname = self.explogdict.get('seqkitplanname','')
self.exp_chipversion = self.explogdict.get('chipversion','')
if not self.exp_chipversion:
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
try:
logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd))
headers = {
"Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
"Content-type": "application/json",
"Accept": "application/json" }
conn = httplib.HTTPConnection(TSUrl)
if self.exp_planned_run_guid:
# TODO, use original plan data from file system
request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&planGUID=%s&sequenceKitName=%s&limit=20" % (self.exp_chipversion, self.exp_planned_run_guid, self.exp_seqkitplanname)
else:
logger.debug("use default chip args")
request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&sequenceKitName=%s&limit=20" % (self.exp_chipversion, self.exp_seqkitplanname)
#http://frenzy.ite/rundb/api/v1/analysisargs/getargs/?format=json&chipType=P1.2.18&planGUID=&sequenceKitName=ProtonI200Kit-v2
logger.info('request: %s http://%s%s' % (self.explogdict.get('runname','unknown'), TSUrl, request))
conn.request("GET", request, "", headers)
response = conn.getresponse()
logger.info('%s %s' % (response.status, response.reason))
data = response.read()
analysisargs = json.loads(data)
logger.debug("Plan: %s" % analysisargs)
f = open(os.path.join(self.sigproc_results_path, "retrieved_args.json"),'w')
f.write(json.dumps(analysisargs, indent=4))
f.close()
if analysisargs:
self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs']
self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs']
self.exp_beadfindArgs_block = analysisargs['beadfindargs']
self.exp_analysisArgs_block = analysisargs['analysisargs']
self.libraryKey = 'TCAG'
self.tfKey = 'ATCG'
else:
raise Exception("Analysis args are empty")
except:
raise
logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail)
logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail)
logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block)
logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint('global','block_to_process_start')
self.block_to_process_end = config.getint('global','block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
def __init__(self, name, config):
self.name = name
self.dat_path = os.path.join(config.get('global', 'results'), name)
self.sigproc_results_path = os.path.join(self.dat_path,
"onboard_results",
"sigproc_results")
self.status = 'new'
self.last_flow = -1
explog_file = os.path.join(self.dat_path, "explog.txt")
if not os.path.exists(explog_file):
raise Exception("%s doesn't exist" % explog_file)
try:
self.explog_file = explog_file
# parse explog.txt
(head, tail) = os.path.split(explog_file)
explogtext = load_log(head, tail)
self.explogdict = parse_log(explogtext)
self.exp_flows = int(self.explogdict["flows"])
self.exp_oninstranalysis = 'yes' in self.explogdict.get(
'oninstranalysis', 'no')
self.exp_usesynchdats = 'yes' in self.explogdict.get(
'use_synchdats', 'no')
self.exp_oia_during_run = 'yes' in self.explogdict.get(
'oia_during_run', 'yes')
self.exp_ecc_enabled = 'yes' in self.explogdict.get(
'ecc_enabled', 'no')
self.exp_planned_run_short_id = self.explogdict.get(
'planned_run_short_id', '')
self.exp_chiptype = self.explogdict["chiptype"]
self.exp_chipversion = self.explogdict.get('chipversion', '')
if not self.exp_chipversion or not self.exp_chipversion.startswith(
'P'):
self.exp_chipversion = self.exp_chiptype
logger.info("planned_run_short_id: '%s'" %
self.exp_planned_run_short_id)
logger.info("chiptype: '%s'" % self.exp_chiptype)
logger.info("chipversion: '%s'" % self.exp_chipversion)
if os.path.exists('/software/config/DataCollect.config'):
f = open('/software/config/DataCollect.config')
text = f.readlines()
f.close()
for line in text:
[key, value] = line.split(':')
key = key.strip()
value = value.strip()
logger.info('%s %s' % (key, value))
if key == 'TSUrl':
TSUrl = value
elif key == 'TSUserName':
TSUserName = value
elif key == 'TSPasswd':
TSPasswd = value
else:
continue
else:
raise
logger.info('connect to:%s user:%s password:%s' %
(TSUrl, TSUserName, TSPasswd))
headers = {
"Authorization":
"Basic " +
string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
"Content-type":
"application/json",
"Accept":
"application/json"
}
try:
plan_found = False
if self.exp_planned_run_short_id:
# get plan data
conn = httplib.HTTPConnection(TSUrl)
conn.request(
"GET",
"/rundb/api/v1/plannedexperiment/?format=json&planShortID=%s&limit=20"
% self.exp_planned_run_short_id, "", headers)
response = conn.getresponse()
logger.info('%s %s' % (response.status, response.reason))
data = response.read()
# Need to limit plans
plans = json.loads(data)
logger.debug("number of plans found on %s: %s" %
(TSUrl, len(plans['objects'])))
logger.debug("search for plan with planShortID: '%s'" %
self.exp_planned_run_short_id)
plan = [
plan for plan in plans['objects']
if plan.get('planShortID', 'not_available') ==
self.exp_planned_run_short_id
]
logger.debug(
"number of plans with planShortID '%s' found: %s" %
(self.exp_planned_run_short_id, len(plan)))
if plan:
logger.debug("Plan: %s" % plan[0])
self.exp_beadfindArgs = plan[0].get(
'beadfindargs', 'justBeadFind')
self.exp_analysisArgs = plan[0].get(
'analysisargs', 'Analysis')
self.libraryKey = plan[0].get('libraryKey', 'TCAG')
self.tfKey = plan[0].get('tfKey', 'ATCG')
plan_found = True
else:
logger.debug("Problem with args in plan '%s'" %
self.exp_planned_run_short_id)
if not plan_found:
logger.debug("plan not available")
# TODO: user might starts run during TS upgrade
conn = httplib.HTTPConnection(TSUrl)
conn.request("GET",
"/rundb/api/v1/analysisargs/?format=json", "",
headers)
response = conn.getresponse()
data = response.read()
allanalysisargs = json.loads(data)
chip_found = False
logger.info(
'chipversion test: search for chipversion: [%s]' %
self.exp_chipversion)
for analysisargs in allanalysisargs['objects']:
logger.info(
'chipversion test: compare with chipversion: [%s]'
% analysisargs['name'])
if analysisargs[
'name'] == "default_" + self.exp_chipversion:
logger.info(
'chipversion test: found analysis args for chipversion: [%s]'
% analysisargs['name'])
logger.info(analysisargs)
self.exp_beadfindArgs = analysisargs[
'beadfindargs']
self.exp_analysisArgs = analysisargs[
'analysisargs']
self.libraryKey = 'TCAG'
self.tfKey = 'ATCG'
chip_found = True
break
if not chip_found:
raise Exception("Chiptype %s not found" %
self.exp_chipversion)
except:
logger.error(traceback.format_exc())
self.exp_beadfindArgs = 'justBeadFind --beadfind-minlivesnr 3 --region-size=216x224 --total-timeout 602'
self.exp_analysisArgs = 'Analysis --from-beadfind --clonal-filter-bkgmodel on --region-size=216x224 --bkg-bfmask-update off --gpuWorkLoad 1 --total-timeout 602'
self.libraryKey = "TCAG"
self.tfKey = "ATCG"
logger.info("beadfindArgs: '%s'" % self.exp_beadfindArgs)
logger.info("analysisArgs: '%s'" % self.exp_analysisArgs)
except:
logger.error(traceback.format_exc())
raise
self.blocks = []
try:
self.block_to_process_start = config.getint(
'global', 'block_to_process_start')
self.block_to_process_end = config.getint('global',
'block_to_process_end')
self.discover_blocks()
except:
logger.error(traceback.format_exc())
raise
