def newExperiment(explog_path): '''Create Experiment record''' # Parse the explog.txt file text = load_log_path(explog_path) dict = parse_log(text) # Create the Experiment Record folder = os.path.dirname(explog_path) # Test if Experiment object already exists try: newExp = models.Experiment.objects.get(unique=folder) except: newExp = None if newExp is None: try: expArgs,st = exp_kwargs(dict,folder) newExp = models.Experiment(**expArgs) newExp.save() except: newExp = None print traceback.format_exc() return newExp
def updateruninfo(request): '''Replaces crawler.py code Ensure this function/action is idempotent''' folder = request.POST.get('datapath') logger.debug("updateruninfo looking at %s" % folder) # parse explog.txt. LOG_BASENAME = "explog.txt" text = load_log(folder, LOG_BASENAME) if text is None: payload = "Cannot read %s" % (os.path.join(folder, LOG_BASENAME)) logger.warn(payload) return http.HttpResponseServerError(payload) else: try: explog_parsed = parse_log(text) except: logger.warn("Error parsing %s, skipping %s" % (LOG_BASENAME, folder)) logger.error(traceback.format_exc()) return http.HttpResponseServerError('%s %s' % (folder, traceback.format_exc())) # Update experiment, plan, eas objects for dataset in this folder try: planObj, expObj, easObj = get_planned_exp_objects(explog_parsed, folder) update_exp_objects_from_log(explog_parsed, folder, planObj, expObj, easObj) save_serialized_json(folder, planObj, expObj, easObj) except: logger.error(traceback.format_exc()) return http.HttpResponseServerError('%s %s' % (folder, traceback.format_exc())) return http.HttpResponse("%s database objects have been updated" % folder, mimetype='text/plain')
def newExperiment(explog_path, plan_json=''): '''Create Experiment record''' folder = os.path.dirname(explog_path) # Test if Experiment object already exists try: newExp = models.Experiment.objects.get(unique=folder) except: newExp = None if newExp is None: # Parse the explog.txt file text = load_log_path(explog_path) explog = parse_log(text) explog["planned_run_short_id"] = '' # don't allow getting plan by shortId - other plans may exist with that id try: planObj, expObj, easObj = get_planned_exp_objects(explog,folder) newExp = update_exp_objects_from_log(explog,folder, planObj, expObj, easObj) if plan_json: update_plan_info(plan_json, planObj, easObj) except: newExp = None print traceback.format_exc() return newExp
def update_expobj_ftpcompleted(_expobj, folder): '''Update Experiment object with completed ftp status''' # Store final explog in database exptxt = load_log(folder, LOG_FINAL_BASENAME) if exptxt is not None: _expobj.log = json.dumps(parse_log(exptxt), indent=4) else: # explog_final exists, but is not readable yet. return # Set FTP transfer status to complete if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT: logger.errors.info("FTP status: Aborted") else: _expobj.ftpStatus = RUN_STATUS_COMPLETE logger.errors.info("FTP status: Complete") # Save experiment object _expobj.save() # Measure and record disk space usage of this dataset # This is celery task which will execute in separate worker thread try: tasks.setRunDiskspace.delay(_expobj.pk) except: logger.errors.error(traceback.format_exc()) raise
def setup_explog(self): """This method will find the experiment log and return it's location""" # First look in raw data directory original = os.path.join(self.raw_data_dir, EXPLOG_FILENAME) if not os.path.exists(original): # Next look in parent of raw data (case:Proton data) original = os.path.join(os.path.dirname(self.raw_data_dir), EXPLOG_FILENAME) # Next look in the report directory if not os.path.exists(original): original = os.path.join(self.analysis_dir, EXPLOG_FILENAME) # Next look in the pgm_logs.zip file if not os.path.exists(original) and os.path.exists(os.path.join(self.results_dir, 'pgm_logs.zip')): original = os.path.join(self.raw_data_dir, EXPLOG_FILENAME) with zipfile.ZipFile(os.path.join(self.results_dir, 'pgm_logs.zip'), mode='r') as pgm_zip_hangle: explog_info = pgm_zip_hangle.getinfo(EXPLOG_FILENAME) pgm_zip_hangle.extract(explog_info, self.raw_data_dir) # read in the exp log with open(original, 'r') as original_handle: explog = parse_log(original_handle.read()) # HACK ALERT! In order to make sure we don't go over the maximum length of the experiment name (currently 128 characters) by the # appending of the _foreign string in the from_wells_analysis.py logic, we are going to have to add a check to see if it can fit # into the data base constraints with the appending of the foreign string if len(explog['experiment_name'] + "_foreign") > 128: raise Exception("We cannot transfer this result due to the length of the experiment name.") return original
def newExperiment(explog_path): '''Create Experiment record''' # Parse the explog.txt file text = load_log_path(explog_path) dict = parse_log(text) # Create the Experiment Record folder = os.path.dirname(explog_path) # Test if Experiment object already exists try: newExp = models.Experiment.objects.get(unique=folder) except: newExp = None if newExp is None: try: expArgs, st = exp_kwargs(dict, folder) newExp = models.Experiment(**expArgs) newExp.save() except: newExp = None print traceback.format_exc() return newExp
def __init__(self,name,config): self.name = name self.dat_path = os.path.join(config.get('global','results'), name) self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results") self.status = 'new' explog_file = os.path.join(self.dat_path, "explog.txt") if not os.path.exists(explog_file): raise Exception("%s doesn't exist" % explog_file) try: self.explog_file = explog_file # parse explog.txt (head,tail) = os.path.split(explog_file) explogtext = load_log(head,tail) self.explogdict = parse_log(explogtext) self.exp_flows = int(self.explogdict["flows"]) self.exp_oninstranalysis = 'yes' in self.explogdict['oninstranalysis'] #yes, no self.exp_usesynchdats = 'yes' in self.explogdict['use_synchdats'] #yes, no except: logger.error(traceback.format_exc()) raise self.blocks = [] try: self.block_to_process_start = int(config.get('global','block_to_process_start')) self.block_to_process_end = int(config.get('global','block_to_process_end')) self.discover_blocks() except: logger.error(traceback.format_exc()) raise
def generate_project_name(exp): """generate project name from parsing explog or input""" if len(sys.argv) > 3: project_name = sys.argv[3] else: fpath = load_log(exp.expDir, 'explog.txt') explog = parse_log(fpath) crawler.exp_kwargs(explog, exp.expDir) project_name = explog.get('project', '') return project_name
def generate_project_name(exp): """generate project name from parsing explog or input""" if len(sys.argv) > 3: project_name = sys.argv[3] else: fpath = load_log(exp.expDir, "explog.txt") explog = parse_log(fpath) crawler.exp_kwargs(explog, exp.expDir) project_name = explog.get("project", "") return project_name
def generate_project_name(exp): """generate project name from parsing explog or input""" if len(sys.argv) > 5: project_name = sys.argv[5] else: fpath = load_log(exp.expDir, 'explog.txt') explog = parse_log(fpath) #crawler.exp_kwargs(explog, exp.expDir, logger.errors) project_name = explog.get('project', '') return project_name
def newExperiment(_explog_path, _plan_json=''): '''Create Experiment record''' folder = os.path.dirname(_explog_path) # Test if Experiment object already exists try: _newExp = models.Experiment.objects.get(unique=folder) print("DEBUG: Experiment exists in database: %s" % folder) except: print("DEBUG: Experiment does not exist in database") _newExp = None if _newExp is None: # Parse the explog.txt file text = load_log_path(_explog_path) explog = parse_log(text) explog[ "planned_run_short_id"] = '' # don't allow getting plan by shortId - other plans may exist with that id try: ret_val = generate_updateruninfo_post(folder, logger) if ret_val == "Generated POST": # Get experiment object exp_set = models.Experiment.objects.filter(unique=folder) if exp_set: # Experiment object exists in database _newExp = exp_set[0] else: print( "ERROR: Could not update/generate new Experiment record in database" ) print(ret_val) return None # Append to expName to indicate imported dataset. _newExp.expName += "_foreign" chefLog_parsed = isChefInfoAvailable(folder) if chefLog_parsed: update_chefSummary(_newExp, chefLog_parsed) _newExp.save() if _plan_json: planObj = _newExp.plan easObj = _newExp.get_EAS() update_plan_info(_plan_json, planObj, easObj) except: print("DEBUG: There was an error adding the experiment") _newExp = None print(traceback.format_exc()) return _newExp
def newExperiment(_explog_path, _plan_json=''): '''Create Experiment record''' folder = os.path.dirname(_explog_path) # Test if Experiment object already exists try: _newExp = models.Experiment.objects.get(unique=folder) print("DEBUG: Experiment exists in database: %s" % folder) except: print("DEBUG: Experiment does not exist in database") _newExp = None if _newExp is None: # Parse the explog.txt file text = load_log_path(_explog_path) explog = parse_log(text) explog[ "planned_run_short_id"] = '' # don't allow getting plan by shortId - other plans may exist with that id try: ret_val = generate_updateruninfo_post(folder, logger) if ret_val == "Generated POST": # Get experiment object exp_set = models.Experiment.objects.filter(unique=folder) if exp_set: # Experiment object exists in database _newExp = exp_set[0] else: print("ERROR: Could not update/generate new Experiment record in database") print(ret_val) return None # Append to expName to indicate imported dataset. _newExp.expName += "_foreign" chefLog_parsed = isChefInfoAvailable(folder) if chefLog_parsed: update_chefSummary(_newExp, chefLog_parsed) _newExp.save() except: print("DEBUG: There was an error adding the experiment") _newExp = None print(traceback.format_exc()) if _plan_json: update_plan_info(_plan_json, _newExp.plan, _newExp.get_EAS()) return _newExp
def newExperiment(_explog_path, _plan_json=""): """Create Experiment record""" folder = os.path.dirname(_explog_path) # Test if Experiment object already exists try: _newExp = models.Experiment.objects.get(unique=folder) print "DEBUG: Experiment exists in database: %s" % (folder) except: print "DEBUG: Experiment does not exist in database" _newExp = None if _newExp is None: # Parse the explog.txt file text = load_log_path(_explog_path) explog = parse_log(text) explog["planned_run_short_id"] = "" # don't allow getting plan by shortId - other plans may exist with that id try: ret_val = generate_updateruninfo_post(folder, logger) if ret_val == "Generated POST": # Get experiment object exp_set = models.Experiment.objects.filter(unique=folder) if exp_set: # Experiment object exists in database _newExp = exp_set[0] else: print "ERROR: Could not update/generate new Experiment record in database" print ret_val return None # Append to expName to indicate imported dataset. _newExp.expName += "_foreign" _newExp.save() if _plan_json: planObj = _newExp.plan easObj = _newExp.get_EAS() update_plan_info(_plan_json, planObj, easObj) except: print "DEBUG: There was an error adding the experiment" _newExp = None print traceback.format_exc() return _newExp
def update_expobj_ftpcompleted(_expobj, folder): '''Update Experiment object with completed ftp status''' # Store final explog in database exptxt = load_log(folder, LOG_FINAL_BASENAME) if exptxt is not None: _expobj.log = json.dumps(parse_log(exptxt), indent=4) else: # explog_final exists, but is not readable yet. return # Set FTP transfer status to complete if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT: logger.errors.info("FTP status: Aborted") else: _expobj.ftpStatus = RUN_STATUS_COMPLETE logger.errors.info("FTP status: Complete") # Save experiment object _expobj.save() return
def updateruninfo(request): """Replaces crawler.py code Ensure this function/action is idempotent""" folder = request.POST.get("datapath") logger.debug("updateruninfo looking at %s" % folder) # parse explog.txt. LOG_BASENAME = "explog.txt" text = load_log(folder, LOG_BASENAME) if text is None: payload = "Cannot read %s" % (os.path.join(folder, LOG_BASENAME)) logger.warn(payload) return http.HttpResponseServerError(payload) else: try: explog_parsed = parse_log(text) except Exception: logger.warn("Error parsing %s, skipping %s" % (LOG_BASENAME, folder)) logger.error(traceback.format_exc()) return http.HttpResponseServerError( "%s %s" % (folder, traceback.format_exc())) # Update experiment, plan, eas objects for dataset in this folder try: planObj, expObj, easObj = get_planned_exp_objects( explog_parsed, folder) update_exp_objects_from_log(explog_parsed, folder, planObj, expObj, easObj) save_serialized_json(folder, planObj, expObj, easObj) except Exception: logger.error(traceback.format_exc()) return http.HttpResponseServerError( "%s %s" % (folder, traceback.format_exc())) return http.HttpResponse("%s database objects have been updated" % folder, mimetype="text/plain")
def __init__(self,results,name,config): self.name = name self.dat_path = os.path.join(results, name) self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results") self.status = 'new' explog_file = os.path.join(results, name, "explog.txt") if os.path.exists(explog_file): self.explog_file = explog_file # parse explog.txt (head,tail) = os.path.split(explog_file) explogtext = load_log(head,tail) self.explogdict = parse_log(explogtext) self.exp_flows = int(self.explogdict["flows"]) self.exp_oninstranalysis = 'yes' in self.explogdict['oninstranalysis']#yes, no self.exp_usesynchdats = 'yes' in self.explogdict['use_synchdats']#yes, no else: self.explogdict = None self.explog_file = None self.exp_flows = None self.exp_oninstranalysis = None self.exp_usesynchdats = None self.blocks = [] if self.explogdict: try: self.discover_blocks(0,95)#TODO block_to_process_start,block_to_process_end) except: logger.error(traceback.format_exc()) pass else: logger.error(traceback.format_exc())
def __init__(self,name,config): self.name = name self.dat_path = os.path.join(config.get('global','results'), name) self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results") self.status = 'new' explog_file = os.path.join(self.dat_path, "explog.txt") if not os.path.exists(explog_file): raise Exception("%s doesn't exist" % explog_file) try: self.explog_file = explog_file # parse explog.txt (head,tail) = os.path.split(explog_file) explogtext = load_log(head,tail) self.explogdict = parse_log(explogtext) self.exp_flows = int(self.explogdict["flows"]) self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no') self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no') self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes') self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no') self.exp_planned_run_short_id = self.explogdict.get('planned_run_short_id','') self.exp_chiptype = self.explogdict["chiptype"] logger.info('chiptype:-%s-' % self.exp_chiptype) self.exp_runtype = self.explogdict["runtype"] logger.info('runtype:-%s-' % self.exp_runtype) ''' if os.path.exists(os.path.join(self.dat_path, "chips.json")): chips = json.load(open('os.path.join(self.dat_path, "chips.json")','r')) else: logger.error('use default args /usr/share/ion/oia/chips.json') chips = json.load(open('/usr/share/ion/oia/chips.json','r')) ''' if os.path.exists('/software/config/DataCollect.config'): f = open('/software/config/DataCollect.config') text = f.readlines() f.close() for line in text: [key, value] = line.split(':') key = key.strip() value = value.strip() logger.info('%s %s' % (key, value)) if key == 'TSUrl': TSUrl = value elif key == 'TSUserName': TSUserName = value elif key == 'TSPasswd': TSPasswd = value else: continue else: raise logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd)) headers = { "Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)), "Content-type": "application/json", "Accept": "application/json" } # get chip data conn = httplib.HTTPConnection(TSUrl) conn.request("GET", "/rundb/api/v1/chip/?format=json", "", headers) response = conn.getresponse() data = response.read() chips = json.loads(data) # get plan data conn = httplib.HTTPConnection(TSUrl) conn.request("GET", "/rundb/api/v1/plannedexperiment/?format=json", "", headers) response = conn.getresponse() logger.info('%s %s' % (response.status, response.reason)) data = response.read() # Need to limit plans #logger.debug(data) plans = json.loads(data) for chip in chips['objects']: logger.info(chip) logger.info('chiptype test: test normal chiptype: [%s]' % self.exp_chiptype) if chip['name']==self.exp_chiptype: logger.info('chiptype test: found normal chiptype: [%s]' % chip['name']) self.exp_beadfindArgs = chip['beadfindargs'] self.exp_analysisArgs = chip['analysisargs'] logger.info('chiptype test: test special chiptype/runtype: [%s%s]' % (self.exp_chiptype,self.exp_runtype)) if chip['name']==self.exp_chiptype+self.exp_runtype: logger.info('chiptype test: found special chiptype/runtype: [%s]' % chip['name']) self.exp_beadfindArgs = chip['beadfindargs'] self.exp_analysisArgs = chip['analysisargs'] break logger.info('beadfindArgs:-%s-' % self.exp_beadfindArgs) logger.info('analysisArgs:-%s-' % self.exp_analysisArgs) plan = [plan for plan in plans['objects'] if plan.get('planShortID','not_available') == self.exp_planned_run_short_id] if plan: logger.debug("Plan: %s" % plan[0]) self.libraryKey=plan[0].get('libraryKey','TCAG') # TODO self.tfKey=plan[0].get('tfKey','ATCG') # TODO else: logger.debug("no plan") self.libraryKey="TCAG" self.tfKey="ATCG" except: logger.error(traceback.format_exc()) raise self.blocks = [] try: self.block_to_process_start = config.getint('global','block_to_process_start') self.block_to_process_end = config.getint('global','block_to_process_end') self.discover_blocks() except: logger.error(traceback.format_exc()) raise
def __init__(self,name,config): self.name = name self.dat_path = os.path.join(config.get('global','results'), name) self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results") self.status = 'new' self.last_flow = -1 explog_file = os.path.join(self.dat_path, "explog.txt") if not os.path.exists(explog_file): raise Exception("%s doesn't exist" % explog_file) try: self.explog_file = explog_file # parse explog.txt (head,tail) = os.path.split(explog_file) explogtext = load_log(head,tail) self.explogdict = parse_log(explogtext) self.exp_flows = int(self.explogdict["flows"]) self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no') self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no') self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes') self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no') self.exp_planned_run_short_id = self.explogdict.get('planned_run_short_id','') self.exp_chiptype = self.explogdict["chiptype"] self.exp_chipversion = self.explogdict.get('chipversion','') if not self.exp_chipversion or not self.exp_chipversion.startswith('P'): self.exp_chipversion = self.exp_chiptype logger.info("planned_run_short_id: '%s'" % self.exp_planned_run_short_id) logger.info("chiptype: '%s'" % self.exp_chiptype) logger.info("chipversion: '%s'" % self.exp_chipversion) if os.path.exists('/software/config/DataCollect.config'): f = open('/software/config/DataCollect.config') text = f.readlines() f.close() for line in text: [key, value] = line.split(':') key = key.strip() value = value.strip() logger.info('%s %s' % (key, value)) if key == 'TSUrl': TSUrl = value elif key == 'TSUserName': TSUserName = value elif key == 'TSPasswd': TSPasswd = value else: continue else: raise logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd)) headers = { "Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)), "Content-type": "application/json", "Accept": "application/json" } try: plan_found = False if self.exp_planned_run_short_id: # get plan data conn = httplib.HTTPConnection(TSUrl) conn.request("GET", "/rundb/api/v1/plannedexperiment/?format=json&planShortID=%s&limit=20" % self.exp_planned_run_short_id, "", headers) response = conn.getresponse() logger.info('%s %s' % (response.status, response.reason)) data = response.read() # Need to limit plans plans = json.loads(data) logger.debug("number of plans found on %s: %s" % (TSUrl,len(plans['objects']))) logger.debug("search for plan with planShortID: '%s'" % self.exp_planned_run_short_id) plan = [plan for plan in plans['objects'] if plan.get('planShortID','not_available') == self.exp_planned_run_short_id] logger.debug("number of plans with planShortID '%s' found: %s" % (self.exp_planned_run_short_id,len(plan))) if plan: logger.debug("Plan: %s" % plan[0]) self.exp_beadfindArgs = plan[0].get('beadfindargs','justBeadFind') self.exp_analysisArgs = plan[0].get('analysisargs','Analysis') self.libraryKey = plan[0].get('libraryKey','TCAG') self.tfKey = plan[0].get('tfKey','ATCG') plan_found = True else: logger.debug("Problem with args in plan '%s'" % self.exp_planned_run_short_id) if not plan_found: logger.debug("plan not available") # TODO: user might starts run during TS upgrade conn = httplib.HTTPConnection(TSUrl) conn.request("GET", "/rundb/api/v1/analysisargs/?format=json", "", headers) response = conn.getresponse() data = response.read() allanalysisargs = json.loads(data) chip_found = False logger.info('chipversion test: search for chipversion: [%s]' % self.exp_chipversion) for analysisargs in allanalysisargs['objects']: logger.info('chipversion test: compare with chipversion: [%s]' % analysisargs['name']) if analysisargs['name']=="default_"+self.exp_chipversion: logger.info('chipversion test: found analysis args for chipversion: [%s]' % analysisargs['name']) logger.info(analysisargs) self.exp_beadfindArgs = analysisargs['beadfindargs'] self.exp_analysisArgs = analysisargs['analysisargs'] self.libraryKey = 'TCAG' self.tfKey = 'ATCG' chip_found = True break if not chip_found: raise Exception("Chiptype %s not found" % self.exp_chipversion) except: logger.error(traceback.format_exc()) self.exp_beadfindArgs = 'justBeadFind --beadfind-minlivesnr 3 --region-size=216x224 --total-timeout 602' self.exp_analysisArgs = 'Analysis --from-beadfind --clonal-filter-bkgmodel on --region-size=216x224 --bkg-bfmask-update off --gpuWorkLoad 1 --total-timeout 602' self.libraryKey = "TCAG" self.tfKey = "ATCG" logger.info("beadfindArgs: '%s'" % self.exp_beadfindArgs) logger.info("analysisArgs: '%s'" % self.exp_analysisArgs) except: logger.error(traceback.format_exc()) raise self.blocks = [] try: self.block_to_process_start = config.getint('global','block_to_process_start') self.block_to_process_end = config.getint('global','block_to_process_end') self.discover_blocks() except: logger.error(traceback.format_exc()) raise
def crawl(folders, logger): """Crawl over ``folders``, reporting information to the ``CrawlLog`` ``logger``.""" if folders: logger.errors.info("checking %d directories" % len(folders)) for f in folders: try: text = load_log(f, LOG_BASENAME) logger.errors.debug("checking directory %s" % f) #------------------------------------ # Check for a valid explog.txt file #------------------------------------ if text is not None: logger.current_folder = f # Catch any errors parsing this log file and continue to the next experiment. try: d = parse_log(text) except: logger.errors.info("Error parsing explog, skipping %s" % f) logger.errors.exception(traceback.format_exc()) continue exp = None exp_set = models.Experiment.objects.filter(unique=f) if exp_set: # Experiment already in database exp = exp_set[0] if exp.ftpStatus == RUN_STATUS_COMPLETE: continue else: # Need to create experiment for this folder try: planObj, expObj, easObj = get_planned_exp_objects(d, f) exp = update_exp_objects_from_log( d, f, planObj, expObj, easObj) except: logger.errors.info( "Error updating experiment objects, skipping %s" % f) logger.errors.exception(traceback.format_exc()) continue if exp is not None: logger.errors.info("%s" % f) #-------------------------------- # Update FTP Transfer Status #-------------------------------- if check_for_completion(exp): exptxt = load_log(f, LOG_FINAL_BASENAME) if exptxt is not None: exp.log = json.dumps(parse_log(exptxt), indent=4) else: # explog_final exists, but is not readable yet. continue # FTP transfer is complete if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT: logger.errors.info("FTP status: Aborted") else: exp.ftpStatus = RUN_STATUS_COMPLETE exp.save() # Measure and record disk space usage of this dataset # This is celery task which will execute in separate worker thread try: tasks.setRunDiskspace.delay(exp.pk) except: logger.errors.exception(traceback.format_exc()) else: if exp.ftpStatus != RUN_STATUS_MISSING: exp.ftpStatus = get_percent(exp) exp.save() logger.errors.info("FTP status: Transferring") #-------------------------------- # Handle auto-run analysis #-------------------------------- composite_exists, thumbnail_exists = reports_exist(exp) logger.errors.debug("Reports Exist? %s %s" % (composite_exists, thumbnail_exists)) if not composite_exists: logger.errors.debug( "No auto-run analysis has been started") # Check if auto-run for whole chip has been requested if exp.autoAnalyze: logger.errors.debug( " auto-run whole chip analysis has been requested" ) #if check_analyze_early(exp) or check_for_completion(exp): if ready_to_process(exp) or check_for_completion( exp): logger.errors.info( " Start a whole chip auto-run analysis job" ) generate_http_post(exp) else: logger.errors.info( " Do not start a whole chip auto-run job yet" ) else: logger.errors.debug( " auto-run whole chip analysis has not been requested" ) else: logger.errors.debug("composite report already exists") if not thumbnail_exists: # Check if auto-run for thumbnail has been requested # But only if its a block dataset if 'tiled' in exp.rawdatastyle: if autorun_thumbnail(exp): logger.errors.debug( "auto-run thumbnail analysis has been requested" ) if ready_to_process_thumbnail( exp) or check_for_completion(exp): logger.errors.info( " Start a thumbnail auto-run analysis job" ) generate_http_post(exp, DO_THUMBNAIL) else: logger.errors.info( " Do not start a thumbnail auto-run job yet" ) else: logger.errors.debug( "auto-run thumbnail analysis has not been requested" ) else: logger.errors.debug( "This is not a block dataset; no thumbnail to process" ) else: logger.errors.debug("thumbnail report already exists") else: logger.errors.debug("exp is empty") else: # No explog.txt exists; probably an engineering test run logger.errors.debug("no %s was found in %s" % (LOG_BASENAME, f)) except: logger.errors.exception(traceback.format_exc()) logger.current_folder = '(none)'
def __init__(self,name,config): self.name = name self.dat_path = os.path.join(config.get('global','results'), name) self.analysis_path = os.path.join(config.get('global','analysisresults'), name) self.sigproc_results_path = os.path.join(config.get('global','analysisresults'), name, "onboard_results", "sigproc_results") self.status = 'new' self.last_flow = -1 try: if not os.path.exists( self.sigproc_results_path ): os.makedirs( self.sigproc_results_path ) except: logger.error(traceback.format_exc()) explog_file = os.path.join(self.analysis_path, "explog.txt") if not os.path.exists(explog_file): shutil.copyfile(os.path.join(self.dat_path, "explog.txt"), os.path.join(self.analysis_path, "explog.txt")) if not os.path.exists(explog_file): raise Exception("%s doesn't exist" % explog_file) try: if not os.path.exists( os.path.join(self.dat_path, 'onboard_results') ): os.symlink( os.path.join(self.analysis_path,'onboard_results'), os.path.join(self.dat_path,'onboard_results') ) except: logger.error(traceback.format_exc()) try: self.explog_file = explog_file # parse explog.txt (head,tail) = os.path.split(explog_file) explogtext = load_log(head,tail) self.explogdict = parse_log(explogtext) self.exp_flows = int(self.explogdict["flows"]) self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no') self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no') self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes') self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no') self.exp_planned_run_guid = self.explogdict.get('planned_run_guid','') self.exp_chiptype = self.explogdict["chiptype"] self.exp_seqkitplanname = self.explogdict.get('seqkitplanname','') self.exp_chipversion = self.explogdict.get('chipversion','') if not self.exp_chipversion: self.exp_chipversion = self.exp_chiptype logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid) logger.info("chiptype: '%s'" % self.exp_chiptype) logger.info("chipversion: '%s'" % self.exp_chipversion) try: f = open(os.path.join(self.dat_path, "planned_run.json"),'r') analysisargs = json.load(f)['objects'][0] f.close() if analysisargs: self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs'] self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs'] self.exp_beadfindArgs_block = analysisargs['beadfindargs'] self.exp_analysisArgs_block = analysisargs['analysisargs'] self.libraryKey = analysisargs['libraryKey'] self.tfKey = analysisargs['tfKey'] else: raise Exception("Analysis args are empty") except: raise logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail) logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail) logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block) logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block) except: logger.error(traceback.format_exc()) raise self.blocks = [] try: self.block_to_process_start = config.getint('global','block_to_process_start') self.block_to_process_end = config.getint('global','block_to_process_end') self.discover_blocks() except: logger.error(traceback.format_exc()) raise
def crawl(folders, logger): """Crawl over ``folders``, reporting information to the ``CrawlLog`` ``logger``.""" if folders: logger.errors.info("checking %d directories" % len(folders)) for f in folders: try: text = load_log(f, LOG_BASENAME) logger.errors.debug("checking directory %s" % f) #------------------------------------ # Check for a valid explog.txt file #------------------------------------ if text is not None: logger.current_folder = f # Catch any errors parsing this log file and continue to the next experiment. d = parse_log(text) kwargs, status = exp_kwargs(d, f) if status is True: exp = exp_from_kwargs(kwargs, logger, False) projectName = d.get('project', '') # part of explog but not a field in Experiment table if exp is not None: logger.errors.info("%s" % f) #make sure 1-1 exp-plan relationship is enforced when new experiment is created planObj = exp.plan if planObj and exp.pk == None and len(models.Experiment.objects.filter(plan__pk = planObj.pk)) > 0: logger.errors.info('WARNING: Plan %s is already associated to an experiment, a copy will be created' % planObj.pk) planObj.pk = None planObj.save() exp.plan = planObj #-------------------------------- # Update FTP Transfer Status #-------------------------------- if check_for_completion(exp): # FTP transfer is complete if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT: logger.errors.info("FTP status: Aborted") else: exp.ftpStatus = RUN_STATUS_COMPLETE exp.log = json.dumps(parse_log(load_log(f, LOG_FINAL_BASENAME)), indent=4) exp.save() # Measure and record disk space usage of this dataset # This is celery task which will execute in separate worker thread try: tasks.setRunDiskspace.delay(exp.pk) except: logger.errors.exception(traceback.format_exc()) else: if exp.ftpStatus != RUN_STATUS_MISSING: exp.ftpStatus = get_percent(exp) exp.save() logger.errors.info("FTP status: Transferring") #-------------------------------- # Handle auto-run analysis #-------------------------------- composite_exists, thumbnail_exists = reports_exist(exp) logger.errors.debug("Reports Exist? %s %s" % (composite_exists, thumbnail_exists)) if not composite_exists: logger.errors.debug("No auto-run analysis has been started") # Check if auto-run for whole chip has been requested if exp.autoAnalyze: logger.errors.debug(" auto-run whole chip analysis has been requested") #if check_analyze_early(exp) or check_for_completion(exp): if ready_to_process(exp) or check_for_completion(exp): logger.errors.info(" Start a whole chip auto-run analysis job") generate_http_post(exp, projectName) else: logger.errors.info(" Do not start a whole chip auto-run job yet") else: logger.errors.debug(" auto-run whole chip analysis has not been requested") else: logger.errors.debug("composite report already exists") if not thumbnail_exists: # Check if auto-run for thumbnail has been requested # But only if its a block dataset if 'tiled' in exp.rawdatastyle: if autorun_thumbnail(exp): logger.errors.debug("auto-run thumbnail analysis has been requested") if ready_to_process_thumbnail(exp) or check_for_completion(exp): logger.errors.info(" Start a thumbnail auto-run analysis job") generate_http_post(exp, projectName, DO_THUMBNAIL) else: logger.errors.info(" Do not start a thumbnail auto-run job yet") else: logger.errors.debug("auto-run thumbnail analysis has not been requested") else: logger.errors.debug("This is not a block dataset; no thumbnail to process") else: logger.errors.debug("thumbnail report already exists") else: logger.errors.debug("exp is empty") else: logger.errors.warn("folder %s is not added due to invalid experiment status from exp_kwargs. " % f) else: # No explog.txt exists; probably an engineering test run logger.errors.debug("no %s was found in %s" % (LOG_BASENAME, f)) except: logger.errors.exception(traceback.format_exc()) logger.current_folder = '(none)'
def __init__(self, name, config): self.name = name self.dat_path = os.path.join(config.get('global', 'results'), name) self.analysis_path = os.path.join( config.get('global', 'analysisresults'), name) self.sigproc_results_path = os.path.join( config.get('global', 'analysisresults'), name, "onboard_results", "sigproc_results") self.status = 'new' self.last_flow = -1 try: if not os.path.exists(self.sigproc_results_path): os.makedirs(self.sigproc_results_path) except: logger.error(traceback.format_exc()) explog_file = os.path.join(self.analysis_path, "explog.txt") if not os.path.exists(explog_file): shutil.copyfile(os.path.join(self.dat_path, "explog.txt"), os.path.join(self.analysis_path, "explog.txt")) if not os.path.exists(explog_file): raise Exception("%s doesn't exist" % explog_file) try: if not os.path.exists( os.path.join(self.dat_path, 'onboard_results')): os.symlink(os.path.join(self.analysis_path, 'onboard_results'), os.path.join(self.dat_path, 'onboard_results')) except: logger.error(traceback.format_exc()) try: self.explog_file = explog_file # parse explog.txt (head, tail) = os.path.split(explog_file) explogtext = load_log(head, tail) self.explogdict = parse_log(explogtext) self.exp_flows = int(self.explogdict["flows"]) self.exp_oninstranalysis = 'yes' in self.explogdict.get( 'oninstranalysis', 'no') self.exp_usesynchdats = 'yes' in self.explogdict.get( 'use_synchdats', 'no') self.exp_oia_during_run = 'yes' in self.explogdict.get( 'oia_during_run', 'yes') self.exp_ecc_enabled = 'yes' in self.explogdict.get( 'ecc_enabled', 'no') self.exp_planned_run_guid = self.explogdict.get( 'planned_run_guid', '') self.exp_chiptype = self.explogdict["chiptype"] self.exp_seqkitplanname = self.explogdict.get('seqkitplanname', '') self.exp_chipversion = self.explogdict.get('chipversion', '') if not self.exp_chipversion or not self.exp_chipversion.startswith( 'P'): self.exp_chipversion = self.exp_chiptype logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid) logger.info("chiptype: '%s'" % self.exp_chiptype) logger.info("chipversion: '%s'" % self.exp_chipversion) try: logger.info('connect to:%s user:%s password:%s' % (TSUrl, TSUserName, TSPasswd)) headers = { "Authorization": "Basic " + string.strip( base64.encodestring(TSUserName + ':' + TSPasswd)), "Content-type": "application/json", "Accept": "application/json" } conn = httplib.HTTPConnection(TSUrl) if self.exp_planned_run_guid: # TODO, use original plan data from file system request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&planGUID=%s&sequenceKitName=%s&limit=20" % ( self.exp_chipversion, self.exp_planned_run_guid, self.exp_seqkitplanname) else: logger.debug("use default chip args") request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&sequenceKitName=%s&limit=20" % ( self.exp_chipversion, self.exp_seqkitplanname) #http://frenzy.ite/rundb/api/v1/analysisargs/getargs/?format=json&chipType=P1.2.18&planGUID=&sequenceKitName=ProtonI200Kit-v2 logger.info('request: %s http://%s%s' % (self.explogdict.get( 'runname', 'unknown'), TSUrl, request)) conn.request("GET", request, "", headers) response = conn.getresponse() logger.info('%s %s' % (response.status, response.reason)) data = response.read() analysisargs = json.loads(data) logger.debug("Plan: %s" % analysisargs) f = open( os.path.join(self.sigproc_results_path, "retrieved_args.json"), 'w') f.write(json.dumps(analysisargs, indent=4)) f.close() if analysisargs: self.exp_beadfindArgs_thumbnail = analysisargs[ 'thumbnailbeadfindargs'] self.exp_analysisArgs_thumbnail = analysisargs[ 'thumbnailanalysisargs'] self.exp_beadfindArgs_block = analysisargs['beadfindargs'] self.exp_analysisArgs_block = analysisargs['analysisargs'] self.libraryKey = 'TCAG' self.tfKey = 'ATCG' else: raise Exception("Analysis args are empty") except: raise logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail) logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail) logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block) logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block) except: logger.error(traceback.format_exc()) raise self.blocks = [] try: self.block_to_process_start = config.getint( 'global', 'block_to_process_start') self.block_to_process_end = config.getint('global', 'block_to_process_end') self.discover_blocks() except: logger.error(traceback.format_exc()) raise
#!/usr/bin/env python # Copyright (C) 2013 Ion Torrent Systems, Inc. All Rights Reserved '''Parses explog.txt and explog_final.txt''' from ion.utils.explogparser import load_log_path from ion.utils.explogparser import parse_log explog_path = "./explog.txt" # Parse the explog.txt file text = load_log_path(explog_path) dict = parse_log(text) #NOTE: keywords will have spaces replaced with underscores and all letters lowercased. keywords = [ "experimenterrorlog", "experimentinfolog", "explog_done", "experiment_name", "f", ] for key, value in dict.items(): if any(key.startswith(item) for item in keywords): print "(%s) ==> %s" % (key, value)
def __init__(self, name, config): self.name = name self.dat_path = os.path.join(config.get('global', 'results'), name) self.analysis_path = os.path.join(config.get('global', 'analysisresults'), name) self.sigproc_results_path = os.path.join( config.get('global', 'analysisresults'), name, "onboard_results", "sigproc_results") self.status = 'new' self.last_flow = -1 try: if not os.path.exists(self.sigproc_results_path): os.makedirs(self.sigproc_results_path) except: logger.error(traceback.format_exc()) explog_file = os.path.join(self.analysis_path, "explog.txt") if not os.path.exists(explog_file): shutil.copyfile(os.path.join(self.dat_path, "explog.txt"), os.path.join(self.analysis_path, "explog.txt")) if not os.path.exists(explog_file): raise Exception("%s doesn't exist" % explog_file) try: if not os.path.exists(os.path.join(self.dat_path, 'onboard_results')): os.symlink(os.path.join(self.analysis_path, 'onboard_results'), os.path.join(self.dat_path, 'onboard_results')) except: logger.error(traceback.format_exc()) try: self.explog_file = explog_file # parse explog.txt (head, tail) = os.path.split(explog_file) explogtext = load_log(head, tail) self.explogdict = parse_log(explogtext) self.exp_flows = int(self.explogdict["flows"]) self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis', 'no') self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats', 'no') self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run', 'yes') self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled', 'no') self.exp_planned_run_guid = self.explogdict.get('planned_run_guid', '') self.exp_chiptype = self.explogdict["chiptype"] self.exp_seqkitplanname = self.explogdict.get('seqkitplanname', '') self.exp_chipversion = self.explogdict.get('chipversion', '') if not self.exp_chipversion: self.exp_chipversion = self.exp_chiptype logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid) logger.info("chiptype: '%s'" % self.exp_chiptype) logger.info("chipversion: '%s'" % self.exp_chipversion) try: analysisargs = {} if os.path.exists(os.path.join(self.dat_path, "reanalysis_args.json")): f = open(os.path.join(self.dat_path, "reanalysis_args.json"), 'r') analysisargs = json.load(f) f.close() logger.info("use analysisargs from %s" % os.path.join(self.dat_path, "reanalysis_args.json")) elif os.path.exists(os.path.join(self.dat_path, "planned_run.json")): f = open(os.path.join(self.dat_path, "planned_run.json"), 'r') analysisargs = json.load(f)['objects'][0] f.close() logger.info("use analysisargs from %s" % os.path.join(self.dat_path, "planned_run.json")) else: logger.info("trying to fetch planned run for %s" % os.path.join(self.dat_path, "planned_run.json")) http_command = "python /software/testing/get_reanalysis_args.py -e %s" % (name) args = shlex.split(http_command.encode('utf8')) logger.info("run process: %s" % (args)) outfile = open(os.path.join(self.dat_path, 'get_reanalysis_args_oia.log'), "w") p = subprocess.Popen(args, stdout=outfile, stderr=subprocess.STDOUT) # add popen process to block ret = p.wait() outfile.close() if os.path.exists(os.path.join(self.dat_path, "planned_run.json")): with open(os.path.join(self.dat_path, "planned_run.json"), 'r') as f: try: analysisargs = json.load(f)['objects'][0] logger.info("Using Plan information from %s" % os.path.join(self.dat_path, "planned_run.json")) except IndexError: raise Exception("File %s is empty", os.path.join(self.dat_path, "planned_run.json")) if len(analysisargs) > 0: self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs'] self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs'] self.exp_beadfindArgs_block = analysisargs['beadfindargs'] self.exp_analysisArgs_block = analysisargs['analysisargs'] self.exp_prebasecallerArgs_block = analysisargs['prebasecallerargs'] self.libraryKey = analysisargs['libraryKey'] self.tfKey = analysisargs['tfKey'] else: raise Exception("There are no Analysis args available") except: raise logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail) logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail) logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block) logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block) logger.info("block prebasecallerArgs: '%s'" % self.exp_prebasecallerArgs_block) except: logger.error(traceback.format_exc()) raise self.blocks = [] try: self.block_to_process_start = config.getint('global', 'block_to_process_start') self.block_to_process_end = config.getint('global', 'block_to_process_end') self.discover_blocks() except: logger.error(traceback.format_exc()) raise
#!/usr/bin/env python # Copyright (C) 2013 Ion Torrent Systems, Inc. All Rights Reserved '''Parses explog.txt and explog_final.txt''' from ion.utils.explogparser import load_log_path from ion.utils.explogparser import parse_log explog_path = "./explog.txt" # Parse the explog.txt file text = load_log_path(explog_path) dict = parse_log(text) #NOTE: keywords will have spaces replaced with underscores and all letters lowercased. keywords = [ "experimenterrorlog", "experimentinfolog", "explog_done", "experiment_name", "f", ] for key,value in dict.items(): if any(key.startswith(item) for item in keywords): print "(%s) ==> %s" % (key, value)
def crawl(folders, logger): """Crawl over ``folders``, reporting information to the ``CrawlLog`` ``logger``.""" if folders: logger.errors.info("checking %d directories" % len(folders)) for f in folders: try: text = load_log(f, LOG_BASENAME) logger.errors.debug("checking directory %s" % f) #------------------------------------ # Check for a valid explog.txt file #------------------------------------ if text is not None: logger.current_folder = f # Catch any errors parsing this log file and continue to the next experiment. try: d = parse_log(text) except: logger.errors.info("Error parsing explog, skipping %s" % f) logger.errors.exception(traceback.format_exc()) continue exp = None exp_set = models.Experiment.objects.filter(unique=f) if exp_set: # Experiment already in database exp = exp_set[0] if exp.ftpStatus == RUN_STATUS_COMPLETE: continue else: # Need to create experiment for this folder try: planObj, expObj, easObj = get_planned_exp_objects(d, f, logger.errors) exp = update_exp_objects_from_log(d, f, planObj, expObj, easObj, logger.errors) except: logger.errors.info("Error updating experiment objects, skipping %s" % f) logger.errors.exception(traceback.format_exc()) continue if exp is not None: logger.errors.info("%s" % f) #-------------------------------- # Update FTP Transfer Status #-------------------------------- if check_for_completion(exp): exptxt = load_log(f, LOG_FINAL_BASENAME) if exptxt is not None: exp.log = json.dumps(parse_log(exptxt), indent=4) else: # explog_final exists, but is not readable yet. continue # FTP transfer is complete if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT: logger.errors.info("FTP status: Aborted") else: exp.ftpStatus = RUN_STATUS_COMPLETE exp.save() # Measure and record disk space usage of this dataset # This is celery task which will execute in separate worker thread try: tasks.setRunDiskspace.delay(exp.pk) except: logger.errors.exception(traceback.format_exc()) else: if exp.ftpStatus != RUN_STATUS_MISSING: exp.ftpStatus = get_filecount(exp) exp.save() logger.errors.info("FTP status: Transferring") #-------------------------------- # Handle auto-run analysis #-------------------------------- composite_exists, thumbnail_exists = reports_exist(exp) logger.errors.debug("Reports Exist? %s %s" % (composite_exists, thumbnail_exists)) if not composite_exists: logger.errors.debug("No auto-run analysis has been started") # Check if auto-run for whole chip has been requested if exp.autoAnalyze: logger.errors.debug(" auto-run whole chip analysis has been requested") #if check_analyze_early(exp) or check_for_completion(exp): if ready_to_process(exp) or check_for_completion(exp): logger.errors.info(" Start a whole chip auto-run analysis job") generate_http_post(exp, logger) else: logger.errors.info(" Do not start a whole chip auto-run job yet") else: logger.errors.debug(" auto-run whole chip analysis has not been requested") else: logger.errors.debug("composite report already exists") if not thumbnail_exists: # Check if auto-run for thumbnail has been requested # But only if its a block dataset if 'tiled' in exp.rawdatastyle: if autorun_thumbnail(exp, logger): logger.errors.debug("auto-run thumbnail analysis has been requested") if ready_to_process_thumbnail(exp) or check_for_completion(exp): logger.errors.info(" Start a thumbnail auto-run analysis job") generate_http_post(exp, logger, DO_THUMBNAIL) else: logger.errors.info(" Do not start a thumbnail auto-run job yet") else: logger.errors.debug("auto-run thumbnail analysis has not been requested") else: logger.errors.debug("This is not a block dataset; no thumbnail to process") else: logger.errors.debug("thumbnail report already exists") else: logger.errors.debug("exp is empty") else: # No explog.txt exists; probably an engineering test run logger.errors.debug("no %s was found in %s" % (LOG_BASENAME, f)) except: logger.errors.exception(traceback.format_exc()) logger.current_folder = '(none)'
def __init__(self,name,config): self.name = name self.dat_path = os.path.join(config.get('global','results'), name) self.analysis_path = os.path.join(config.get('global','analysisresults'), name) self.sigproc_results_path = os.path.join(config.get('global','analysisresults'), name, "onboard_results", "sigproc_results") self.status = 'new' self.last_flow = -1 try: if not os.path.exists( self.sigproc_results_path ): os.makedirs( self.sigproc_results_path ) except: logger.error(traceback.format_exc()) explog_file = os.path.join(self.analysis_path, "explog.txt") if not os.path.exists(explog_file): shutil.copyfile(os.path.join(self.dat_path, "explog.txt"), os.path.join(self.analysis_path, "explog.txt")) if not os.path.exists(explog_file): raise Exception("%s doesn't exist" % explog_file) try: if not os.path.exists( os.path.join(self.dat_path, 'onboard_results') ): os.symlink( os.path.join(self.analysis_path,'onboard_results'), os.path.join(self.dat_path,'onboard_results') ) except: logger.error(traceback.format_exc()) try: self.explog_file = explog_file # parse explog.txt (head,tail) = os.path.split(explog_file) explogtext = load_log(head,tail) self.explogdict = parse_log(explogtext) self.exp_flows = int(self.explogdict["flows"]) self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no') self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no') self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes') self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no') self.exp_planned_run_guid = self.explogdict.get('planned_run_guid','') self.exp_chiptype = self.explogdict["chiptype"] self.exp_seqkitplanname = self.explogdict.get('seqkitplanname','') self.exp_chipversion = self.explogdict.get('chipversion','') if not self.exp_chipversion: self.exp_chipversion = self.exp_chiptype logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid) logger.info("chiptype: '%s'" % self.exp_chiptype) logger.info("chipversion: '%s'" % self.exp_chipversion) try: logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd)) headers = { "Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)), "Content-type": "application/json", "Accept": "application/json" } conn = httplib.HTTPConnection(TSUrl) if self.exp_planned_run_guid: # TODO, use original plan data from file system request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&planGUID=%s&sequenceKitName=%s&limit=20" % (self.exp_chipversion, self.exp_planned_run_guid, self.exp_seqkitplanname) else: logger.debug("use default chip args") request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&sequenceKitName=%s&limit=20" % (self.exp_chipversion, self.exp_seqkitplanname) #http://frenzy.ite/rundb/api/v1/analysisargs/getargs/?format=json&chipType=P1.2.18&planGUID=&sequenceKitName=ProtonI200Kit-v2 logger.info('request: %s http://%s%s' % (self.explogdict.get('runname','unknown'), TSUrl, request)) conn.request("GET", request, "", headers) response = conn.getresponse() logger.info('%s %s' % (response.status, response.reason)) data = response.read() analysisargs = json.loads(data) logger.debug("Plan: %s" % analysisargs) f = open(os.path.join(self.sigproc_results_path, "retrieved_args.json"),'w') f.write(json.dumps(analysisargs, indent=4)) f.close() if analysisargs: self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs'] self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs'] self.exp_beadfindArgs_block = analysisargs['beadfindargs'] self.exp_analysisArgs_block = analysisargs['analysisargs'] self.libraryKey = 'TCAG' self.tfKey = 'ATCG' else: raise Exception("Analysis args are empty") except: raise logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail) logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail) logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block) logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block) except: logger.error(traceback.format_exc()) raise self.blocks = [] try: self.block_to_process_start = config.getint('global','block_to_process_start') self.block_to_process_end = config.getint('global','block_to_process_end') self.discover_blocks() except: logger.error(traceback.format_exc()) raise
def __init__(self, name, config): self.name = name self.dat_path = os.path.join(config.get('global', 'results'), name) self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results") self.status = 'new' self.last_flow = -1 explog_file = os.path.join(self.dat_path, "explog.txt") if not os.path.exists(explog_file): raise Exception("%s doesn't exist" % explog_file) try: self.explog_file = explog_file # parse explog.txt (head, tail) = os.path.split(explog_file) explogtext = load_log(head, tail) self.explogdict = parse_log(explogtext) self.exp_flows = int(self.explogdict["flows"]) self.exp_oninstranalysis = 'yes' in self.explogdict.get( 'oninstranalysis', 'no') self.exp_usesynchdats = 'yes' in self.explogdict.get( 'use_synchdats', 'no') self.exp_oia_during_run = 'yes' in self.explogdict.get( 'oia_during_run', 'yes') self.exp_ecc_enabled = 'yes' in self.explogdict.get( 'ecc_enabled', 'no') self.exp_planned_run_short_id = self.explogdict.get( 'planned_run_short_id', '') self.exp_chiptype = self.explogdict["chiptype"] self.exp_chipversion = self.explogdict.get('chipversion', '') if not self.exp_chipversion or not self.exp_chipversion.startswith( 'P'): self.exp_chipversion = self.exp_chiptype logger.info("planned_run_short_id: '%s'" % self.exp_planned_run_short_id) logger.info("chiptype: '%s'" % self.exp_chiptype) logger.info("chipversion: '%s'" % self.exp_chipversion) if os.path.exists('/software/config/DataCollect.config'): f = open('/software/config/DataCollect.config') text = f.readlines() f.close() for line in text: [key, value] = line.split(':') key = key.strip() value = value.strip() logger.info('%s %s' % (key, value)) if key == 'TSUrl': TSUrl = value elif key == 'TSUserName': TSUserName = value elif key == 'TSPasswd': TSPasswd = value else: continue else: raise logger.info('connect to:%s user:%s password:%s' % (TSUrl, TSUserName, TSPasswd)) headers = { "Authorization": "Basic " + string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)), "Content-type": "application/json", "Accept": "application/json" } try: plan_found = False if self.exp_planned_run_short_id: # get plan data conn = httplib.HTTPConnection(TSUrl) conn.request( "GET", "/rundb/api/v1/plannedexperiment/?format=json&planShortID=%s&limit=20" % self.exp_planned_run_short_id, "", headers) response = conn.getresponse() logger.info('%s %s' % (response.status, response.reason)) data = response.read() # Need to limit plans plans = json.loads(data) logger.debug("number of plans found on %s: %s" % (TSUrl, len(plans['objects']))) logger.debug("search for plan with planShortID: '%s'" % self.exp_planned_run_short_id) plan = [ plan for plan in plans['objects'] if plan.get('planShortID', 'not_available') == self.exp_planned_run_short_id ] logger.debug( "number of plans with planShortID '%s' found: %s" % (self.exp_planned_run_short_id, len(plan))) if plan: logger.debug("Plan: %s" % plan[0]) self.exp_beadfindArgs = plan[0].get( 'beadfindargs', 'justBeadFind') self.exp_analysisArgs = plan[0].get( 'analysisargs', 'Analysis') self.libraryKey = plan[0].get('libraryKey', 'TCAG') self.tfKey = plan[0].get('tfKey', 'ATCG') plan_found = True else: logger.debug("Problem with args in plan '%s'" % self.exp_planned_run_short_id) if not plan_found: logger.debug("plan not available") # TODO: user might starts run during TS upgrade conn = httplib.HTTPConnection(TSUrl) conn.request("GET", "/rundb/api/v1/analysisargs/?format=json", "", headers) response = conn.getresponse() data = response.read() allanalysisargs = json.loads(data) chip_found = False logger.info( 'chipversion test: search for chipversion: [%s]' % self.exp_chipversion) for analysisargs in allanalysisargs['objects']: logger.info( 'chipversion test: compare with chipversion: [%s]' % analysisargs['name']) if analysisargs[ 'name'] == "default_" + self.exp_chipversion: logger.info( 'chipversion test: found analysis args for chipversion: [%s]' % analysisargs['name']) logger.info(analysisargs) self.exp_beadfindArgs = analysisargs[ 'beadfindargs'] self.exp_analysisArgs = analysisargs[ 'analysisargs'] self.libraryKey = 'TCAG' self.tfKey = 'ATCG' chip_found = True break if not chip_found: raise Exception("Chiptype %s not found" % self.exp_chipversion) except: logger.error(traceback.format_exc()) self.exp_beadfindArgs = 'justBeadFind --beadfind-minlivesnr 3 --region-size=216x224 --total-timeout 602' self.exp_analysisArgs = 'Analysis --from-beadfind --clonal-filter-bkgmodel on --region-size=216x224 --bkg-bfmask-update off --gpuWorkLoad 1 --total-timeout 602' self.libraryKey = "TCAG" self.tfKey = "ATCG" logger.info("beadfindArgs: '%s'" % self.exp_beadfindArgs) logger.info("analysisArgs: '%s'" % self.exp_analysisArgs) except: logger.error(traceback.format_exc()) raise self.blocks = [] try: self.block_to_process_start = config.getint( 'global', 'block_to_process_start') self.block_to_process_end = config.getint('global', 'block_to_process_end') self.discover_blocks() except: logger.error(traceback.format_exc()) raise