Beispiel #1
0
def updateruninfo(request):
    '''Replaces crawler.py code
    Ensure this function/action is idempotent'''

    folder = request.POST.get('datapath')
    logger.debug("updateruninfo looking at %s" % folder)

    # parse explog.txt.
    LOG_BASENAME = "explog.txt"
    text = load_log(folder, LOG_BASENAME)
    if text is None:
        payload = "Cannot read %s" % (os.path.join(folder, LOG_BASENAME))
        logger.warn(payload)
        return http.HttpResponseServerError(payload)
    else:
        try:
            explog_parsed = parse_log(text)
        except:
            logger.warn("Error parsing %s, skipping %s" % (LOG_BASENAME, folder))
            logger.error(traceback.format_exc())
            return http.HttpResponseServerError('%s %s' % (folder, traceback.format_exc()))

        # Update experiment, plan, eas objects for dataset in this folder
        try:
            planObj, expObj, easObj = get_planned_exp_objects(explog_parsed, folder)
            update_exp_objects_from_log(explog_parsed, folder, planObj, expObj, easObj)
            save_serialized_json(folder, planObj, expObj, easObj)
        except:
            logger.error(traceback.format_exc())
            return http.HttpResponseServerError('%s %s' % (folder, traceback.format_exc()))

        return http.HttpResponse("%s database objects have been updated" % folder, mimetype='text/plain')
Beispiel #2
0
    def update_expobj_ftpcompleted(_expobj, folder):
        '''Update Experiment object with completed ftp status'''
        # Store final explog in database
        exptxt = load_log(folder, LOG_FINAL_BASENAME)
        if exptxt is not None:
            _expobj.log = json.dumps(parse_log(exptxt), indent=4)
        else:
            # explog_final exists, but is not readable yet.
            return

        # Set FTP transfer status to complete
        if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
            logger.errors.info("FTP status: Aborted")
        else:
            _expobj.ftpStatus = RUN_STATUS_COMPLETE
            logger.errors.info("FTP status: Complete")

        # Save experiment object
        _expobj.save()

        # Measure and record disk space usage of this dataset
        # This is celery task which will execute in separate worker thread
        try:
            tasks.setRunDiskspace.delay(_expobj.pk)
        except:
            logger.errors.error(traceback.format_exc())
            raise
Beispiel #3
0
    def __init__(self,name,config):

        self.name = name
        self.dat_path = os.path.join(config.get('global','results'), name)
        self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
        self.status = 'new'

        explog_file = os.path.join(self.dat_path, "explog.txt")
        if not os.path.exists(explog_file):
            raise Exception("%s doesn't exist" % explog_file)

        try:
            self.explog_file = explog_file

            # parse explog.txt
            (head,tail) = os.path.split(explog_file)
            explogtext = load_log(head,tail)

            self.explogdict = parse_log(explogtext)
            self.exp_flows = int(self.explogdict["flows"])
            self.exp_oninstranalysis = 'yes' in self.explogdict['oninstranalysis'] #yes, no
            self.exp_usesynchdats = 'yes' in self.explogdict['use_synchdats'] #yes, no
        except:
            logger.error(traceback.format_exc())
            raise


        self.blocks = []
        try:
            self.block_to_process_start = int(config.get('global','block_to_process_start'))
            self.block_to_process_end = int(config.get('global','block_to_process_end'))
            self.discover_blocks()
        except:
            logger.error(traceback.format_exc())
            raise
Beispiel #4
0
    def update_expobj_ftpcompleted(_expobj, folder):
        '''Update Experiment object with completed ftp status'''
        # Store final explog in database
        exptxt = load_log(folder, LOG_FINAL_BASENAME)
        if exptxt is not None:
            _expobj.log = json.dumps(parse_log(exptxt), indent=4)
        else:
            # explog_final exists, but is not readable yet.
            return

        # Set FTP transfer status to complete
        if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
            logger.errors.info("FTP status: Aborted")
        else:
            _expobj.ftpStatus = RUN_STATUS_COMPLETE
            logger.errors.info("FTP status: Complete")

        # Save experiment object
        _expobj.save()

        # Measure and record disk space usage of this dataset
        # This is celery task which will execute in separate worker thread
        try:
            tasks.setRunDiskspace.delay(_expobj.pk)
        except:
            logger.errors.error(traceback.format_exc())
            raise
Beispiel #5
0
def generate_project_name(exp):
    """generate project name from parsing explog or input"""
    if len(sys.argv) > 3:
        project_name = sys.argv[3]
    else:
        fpath = load_log(exp.expDir, "explog.txt")
        explog = parse_log(fpath)
        crawler.exp_kwargs(explog, exp.expDir)
        project_name = explog.get("project", "")
    return project_name
Beispiel #6
0
def generate_project_name(exp):
    """generate project name from parsing explog or input"""
    if len(sys.argv) > 5:
        project_name = sys.argv[5]
    else:
        fpath = load_log(exp.expDir, 'explog.txt')
        explog = parse_log(fpath)
        #crawler.exp_kwargs(explog, exp.expDir, logger.errors)
        project_name = explog.get('project', '')
    return project_name
Beispiel #7
0
def generate_project_name(exp):
    """generate project name from parsing explog or input"""
    if len(sys.argv) > 3:
        project_name = sys.argv[3]
    else:
        fpath = load_log(exp.expDir, 'explog.txt')
        explog = parse_log(fpath)
        crawler.exp_kwargs(explog, exp.expDir)
        project_name = explog.get('project', '')
    return project_name
Beispiel #8
0
def isChefInfoAvailable(folder):
    # parse chef_param.json
    chefLog_parsed = {}
    JSON_BASENAME = "chef_params.json"
    chefLog = load_log(folder, JSON_BASENAME)
    if chefLog is not None:
        try:
            chefLog_parsed = json.loads(chefLog)
        except:
            logger.warn("Error parsing %s, skipping %s" % (JSON_BASENAME, folder))
            logger.error(traceback.format_exc())
            print(traceback.format_exc())

    return chefLog_parsed
Beispiel #9
0
def isChefInfoAvailable(folder):
    # parse chef_param.json
    chefLog_parsed = {}
    JSON_BASENAME = "chef_params.json"
    chefLog = load_log(folder, JSON_BASENAME)
    if chefLog is None:
        payload = "Chef summary info not available read %s" % (os.path.join(
            folder, JSON_BASENAME))
        logger.warn(payload)
    else:
        try:
            chefLog_parsed = json.loads(chefLog)
        except:
            logger.warn("Error parsing %s, skipping %s" %
                        (JSON_BASENAME, folder))
            logger.error(traceback.format_exc())
            print(traceback.format_exc())

    return chefLog_parsed
Beispiel #10
0
    def update_expobj_ftpcompleted(_expobj, folder):
        '''Update Experiment object with completed ftp status'''
        # Store final explog in database
        exptxt = load_log(folder, LOG_FINAL_BASENAME)
        if exptxt is not None:
            _expobj.log = json.dumps(parse_log(exptxt), indent=4)
        else:
            # explog_final exists, but is not readable yet.
            return

        # Set FTP transfer status to complete
        if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
            logger.errors.info("FTP status: Aborted")
        else:
            _expobj.ftpStatus = RUN_STATUS_COMPLETE
            logger.errors.info("FTP status: Complete")

        # Save experiment object
        _expobj.save()

        return
Beispiel #11
0
    def update_expobj_ftpcompleted(_expobj, folder):
        '''Update Experiment object with completed ftp status'''
        # Store final explog in database
        exptxt = load_log(folder, LOG_FINAL_BASENAME)
        if exptxt is not None:
            _expobj.log = json.dumps(parse_log(exptxt), indent=4)
        else:
            # explog_final exists, but is not readable yet.
            return

        # Set FTP transfer status to complete
        if _expobj.ftpStatus == RUN_STATUS_ABORT or _expobj.ftpStatus == RUN_STATUS_SYS_CRIT:
            logger.errors.info("FTP status: Aborted")
        else:
            _expobj.ftpStatus = RUN_STATUS_COMPLETE
            logger.errors.info("FTP status: Complete")

        # Save experiment object
        _expobj.save()
        
        return
Beispiel #12
0
def updateruninfo(request):
    """Replaces crawler.py code
    Ensure this function/action is idempotent"""

    folder = request.POST.get("datapath")
    logger.debug("updateruninfo looking at %s" % folder)

    # parse explog.txt.
    LOG_BASENAME = "explog.txt"
    text = load_log(folder, LOG_BASENAME)
    if text is None:
        payload = "Cannot read %s" % (os.path.join(folder, LOG_BASENAME))
        logger.warn(payload)
        return http.HttpResponseServerError(payload)
    else:
        try:
            explog_parsed = parse_log(text)
        except Exception:
            logger.warn("Error parsing %s, skipping %s" %
                        (LOG_BASENAME, folder))
            logger.error(traceback.format_exc())
            return http.HttpResponseServerError(
                "%s %s" % (folder, traceback.format_exc()))

        # Update experiment, plan, eas objects for dataset in this folder
        try:
            planObj, expObj, easObj = get_planned_exp_objects(
                explog_parsed, folder)
            update_exp_objects_from_log(explog_parsed, folder, planObj, expObj,
                                        easObj)
            save_serialized_json(folder, planObj, expObj, easObj)
        except Exception:
            logger.error(traceback.format_exc())
            return http.HttpResponseServerError(
                "%s %s" % (folder, traceback.format_exc()))

        return http.HttpResponse("%s database objects have been updated" %
                                 folder,
                                 mimetype="text/plain")
Beispiel #13
0
    def __init__(self,results,name,config):

        self.name = name
        self.dat_path = os.path.join(results, name)
        self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
        self.status = 'new'

        explog_file = os.path.join(results, name, "explog.txt")
        if os.path.exists(explog_file):

            self.explog_file = explog_file

            # parse explog.txt
            (head,tail) = os.path.split(explog_file)
            explogtext = load_log(head,tail)

            self.explogdict = parse_log(explogtext)
            self.exp_flows = int(self.explogdict["flows"])
            self.exp_oninstranalysis = 'yes' in self.explogdict['oninstranalysis']#yes, no
            self.exp_usesynchdats = 'yes' in self.explogdict['use_synchdats']#yes, no
        else:
            self.explogdict = None
            self.explog_file = None
            self.exp_flows = None
            self.exp_oninstranalysis = None
            self.exp_usesynchdats = None

        self.blocks = []
        if self.explogdict:
            try:
                self.discover_blocks(0,95)#TODO block_to_process_start,block_to_process_end)
            except:
                logger.error(traceback.format_exc())
                pass
        else:
             logger.error(traceback.format_exc())
Beispiel #14
0
Datei: oiad.py Projekt: aidjek/TS
    def __init__(self,name,config):

        self.name = name
        self.dat_path = os.path.join(config.get('global','results'), name)
        self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
        self.status = 'new'

        explog_file = os.path.join(self.dat_path, "explog.txt")
        if not os.path.exists(explog_file):
            raise Exception("%s doesn't exist" % explog_file)

        try:
            self.explog_file = explog_file

            # parse explog.txt
            (head,tail) = os.path.split(explog_file)
            explogtext = load_log(head,tail)

            self.explogdict = parse_log(explogtext)
            self.exp_flows = int(self.explogdict["flows"])
            self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
            self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
            self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
            self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
            self.exp_planned_run_short_id = self.explogdict.get('planned_run_short_id','')
            self.exp_chiptype = self.explogdict["chiptype"]
            logger.info('chiptype:-%s-' % self.exp_chiptype)
            self.exp_runtype = self.explogdict["runtype"]
            logger.info('runtype:-%s-' % self.exp_runtype)

            '''
            if os.path.exists(os.path.join(self.dat_path, "chips.json")):
                chips = json.load(open('os.path.join(self.dat_path, "chips.json")','r'))
            else:
                logger.error('use default args /usr/share/ion/oia/chips.json')
                chips = json.load(open('/usr/share/ion/oia/chips.json','r'))
            '''

            if os.path.exists('/software/config/DataCollect.config'):
                f = open('/software/config/DataCollect.config')
                text = f.readlines()
                f.close()
                for line in text:
                    [key, value] = line.split(':')
                    key = key.strip()
                    value = value.strip()
                    logger.info('%s %s' % (key, value))
                    if key == 'TSUrl':
                        TSUrl = value
                    elif key == 'TSUserName':
                        TSUserName = value
                    elif key == 'TSPasswd':
                        TSPasswd = value
                    else:
                        continue
            else:
                raise

            logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd))

            headers = {
                "Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
                "Content-type": "application/json",
                "Accept": "application/json" }

            # get chip data
            conn = httplib.HTTPConnection(TSUrl)
            conn.request("GET", "/rundb/api/v1/chip/?format=json", "", headers)
            response = conn.getresponse()
            data = response.read()
            chips = json.loads(data)

            # get plan data
            conn = httplib.HTTPConnection(TSUrl)
            conn.request("GET", "/rundb/api/v1/plannedexperiment/?format=json", "", headers)
            response = conn.getresponse()
            logger.info('%s %s' % (response.status, response.reason))
            data = response.read()
            # Need to limit plans
            #logger.debug(data)
            plans = json.loads(data)

            for chip in chips['objects']:
                logger.info(chip)
                logger.info('chiptype test: test normal chiptype: [%s]' % self.exp_chiptype)
                if chip['name']==self.exp_chiptype:
                    logger.info('chiptype test: found normal chiptype: [%s]' % chip['name'])
                    self.exp_beadfindArgs = chip['beadfindargs']
                    self.exp_analysisArgs = chip['analysisargs']
                logger.info('chiptype test: test special chiptype/runtype: [%s%s]' % (self.exp_chiptype,self.exp_runtype))
                if chip['name']==self.exp_chiptype+self.exp_runtype:
                    logger.info('chiptype test: found special chiptype/runtype: [%s]' % chip['name'])
                    self.exp_beadfindArgs = chip['beadfindargs']
                    self.exp_analysisArgs = chip['analysisargs']
                    break
            logger.info('beadfindArgs:-%s-' % self.exp_beadfindArgs)
            logger.info('analysisArgs:-%s-' % self.exp_analysisArgs)

            plan = [plan for plan in plans['objects'] if plan.get('planShortID','not_available') == self.exp_planned_run_short_id]
            if plan:
                logger.debug("Plan: %s" % plan[0])
                self.libraryKey=plan[0].get('libraryKey','TCAG') # TODO
                self.tfKey=plan[0].get('tfKey','ATCG') # TODO
            else:
                logger.debug("no plan")
                self.libraryKey="TCAG"
                self.tfKey="ATCG"


        except:
            logger.error(traceback.format_exc())
            raise


        self.blocks = []
        try:
            self.block_to_process_start = config.getint('global','block_to_process_start')
            self.block_to_process_end = config.getint('global','block_to_process_end')
            self.discover_blocks()
        except:
            logger.error(traceback.format_exc())
            raise
Beispiel #15
0
    def __init__(self, name, config):

        self.name = name
        self.dat_path = os.path.join(config.get('global', 'results'), name)
        self.analysis_path = os.path.join(
            config.get('global', 'analysisresults'), name)
        self.sigproc_results_path = os.path.join(
            config.get('global', 'analysisresults'), name, "onboard_results",
            "sigproc_results")
        self.status = 'new'
        self.last_flow = -1

        try:
            if not os.path.exists(self.sigproc_results_path):
                os.makedirs(self.sigproc_results_path)
        except:
            logger.error(traceback.format_exc())

        explog_file = os.path.join(self.analysis_path, "explog.txt")
        if not os.path.exists(explog_file):
            shutil.copyfile(os.path.join(self.dat_path, "explog.txt"),
                            os.path.join(self.analysis_path, "explog.txt"))
        if not os.path.exists(explog_file):
            raise Exception("%s doesn't exist" % explog_file)

        try:
            if not os.path.exists(
                    os.path.join(self.dat_path, 'onboard_results')):
                os.symlink(os.path.join(self.analysis_path, 'onboard_results'),
                           os.path.join(self.dat_path, 'onboard_results'))
        except:
            logger.error(traceback.format_exc())

        try:
            self.explog_file = explog_file

            # parse explog.txt
            (head, tail) = os.path.split(explog_file)
            explogtext = load_log(head, tail)

            self.explogdict = parse_log(explogtext)
            self.exp_flows = int(self.explogdict["flows"])
            self.exp_oninstranalysis = 'yes' in self.explogdict.get(
                'oninstranalysis', 'no')
            self.exp_usesynchdats = 'yes' in self.explogdict.get(
                'use_synchdats', 'no')
            self.exp_oia_during_run = 'yes' in self.explogdict.get(
                'oia_during_run', 'yes')
            self.exp_ecc_enabled = 'yes' in self.explogdict.get(
                'ecc_enabled', 'no')
            self.exp_planned_run_guid = self.explogdict.get(
                'planned_run_guid', '')
            self.exp_chiptype = self.explogdict["chiptype"]
            self.exp_seqkitplanname = self.explogdict.get('seqkitplanname', '')

            self.exp_chipversion = self.explogdict.get('chipversion', '')
            if not self.exp_chipversion or not self.exp_chipversion.startswith(
                    'P'):
                self.exp_chipversion = self.exp_chiptype

            logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
            logger.info("chiptype: '%s'" % self.exp_chiptype)
            logger.info("chipversion: '%s'" % self.exp_chipversion)

            try:

                logger.info('connect to:%s user:%s password:%s' %
                            (TSUrl, TSUserName, TSPasswd))

                headers = {
                    "Authorization":
                    "Basic " + string.strip(
                        base64.encodestring(TSUserName + ':' + TSPasswd)),
                    "Content-type":
                    "application/json",
                    "Accept":
                    "application/json"
                }

                conn = httplib.HTTPConnection(TSUrl)
                if self.exp_planned_run_guid:
                    # TODO, use original plan data from file system
                    request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&planGUID=%s&sequenceKitName=%s&limit=20" % (
                        self.exp_chipversion, self.exp_planned_run_guid,
                        self.exp_seqkitplanname)
                else:
                    logger.debug("use default chip args")
                    request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&sequenceKitName=%s&limit=20" % (
                        self.exp_chipversion, self.exp_seqkitplanname)

                #http://frenzy.ite/rundb/api/v1/analysisargs/getargs/?format=json&chipType=P1.2.18&planGUID=&sequenceKitName=ProtonI200Kit-v2
                logger.info('request: %s http://%s%s' % (self.explogdict.get(
                    'runname', 'unknown'), TSUrl, request))
                conn.request("GET", request, "", headers)

                response = conn.getresponse()
                logger.info('%s %s' % (response.status, response.reason))
                data = response.read()
                analysisargs = json.loads(data)
                logger.debug("Plan: %s" % analysisargs)

                f = open(
                    os.path.join(self.sigproc_results_path,
                                 "retrieved_args.json"), 'w')
                f.write(json.dumps(analysisargs, indent=4))
                f.close()

                if analysisargs:
                    self.exp_beadfindArgs_thumbnail = analysisargs[
                        'thumbnailbeadfindargs']
                    self.exp_analysisArgs_thumbnail = analysisargs[
                        'thumbnailanalysisargs']
                    self.exp_beadfindArgs_block = analysisargs['beadfindargs']
                    self.exp_analysisArgs_block = analysisargs['analysisargs']
                    self.libraryKey = 'TCAG'
                    self.tfKey = 'ATCG'
                else:
                    raise Exception("Analysis args are empty")
            except:
                raise

            logger.info("thumbnail beadfindArgs: '%s'" %
                        self.exp_beadfindArgs_thumbnail)
            logger.info("thumbnail analysisArgs: '%s'" %
                        self.exp_analysisArgs_thumbnail)
            logger.info("block beadfindArgs: '%s'" %
                        self.exp_beadfindArgs_block)
            logger.info("block analysisArgs: '%s'" %
                        self.exp_analysisArgs_block)

        except:
            logger.error(traceback.format_exc())
            raise

        self.blocks = []
        try:
            self.block_to_process_start = config.getint(
                'global', 'block_to_process_start')
            self.block_to_process_end = config.getint('global',
                                                      'block_to_process_end')
            self.discover_blocks()
        except:
            logger.error(traceback.format_exc())
            raise
Beispiel #16
0
def crawl(folders, logger):
    """Crawl over ``folders``, reporting information to the ``CrawlLog``
    ``logger``."""
    if folders:
        logger.errors.info("checking %d directories" % len(folders))

    for f in folders:
        try:
            text = load_log(f, LOG_BASENAME)
            logger.errors.debug("checking directory %s" % f)
            #------------------------------------
            # Check for a valid explog.txt file
            #------------------------------------
            if text is not None:

                logger.current_folder = f
                # Catch any errors parsing this log file and continue to the next experiment.
                try:
                    d = parse_log(text)
                except:
                    logger.errors.info("Error parsing explog, skipping %s" % f)
                    logger.errors.exception(traceback.format_exc())
                    continue

                exp = None
                exp_set = models.Experiment.objects.filter(unique=f)
                if exp_set:
                    # Experiment already in database
                    exp = exp_set[0]
                    if exp.ftpStatus == RUN_STATUS_COMPLETE:
                        continue
                else:
                    # Need to create experiment for this folder
                    try:
                        planObj, expObj, easObj = get_planned_exp_objects(d, f)
                        exp = update_exp_objects_from_log(
                            d, f, planObj, expObj, easObj)
                    except:
                        logger.errors.info(
                            "Error updating experiment objects, skipping %s" %
                            f)
                        logger.errors.exception(traceback.format_exc())
                        continue

                if exp is not None:

                    logger.errors.info("%s" % f)

                    #--------------------------------
                    # Update FTP Transfer Status
                    #--------------------------------
                    if check_for_completion(exp):
                        exptxt = load_log(f, LOG_FINAL_BASENAME)
                        if exptxt is not None:
                            exp.log = json.dumps(parse_log(exptxt), indent=4)
                        else:
                            # explog_final exists, but is not readable yet.
                            continue

                        # FTP transfer is complete
                        if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT:
                            logger.errors.info("FTP status: Aborted")
                        else:
                            exp.ftpStatus = RUN_STATUS_COMPLETE
                        exp.save()

                        # Measure and record disk space usage of this dataset
                        # This is celery task which will execute in separate worker thread
                        try:
                            tasks.setRunDiskspace.delay(exp.pk)
                        except:
                            logger.errors.exception(traceback.format_exc())

                    else:
                        if exp.ftpStatus != RUN_STATUS_MISSING:
                            exp.ftpStatus = get_percent(exp)
                            exp.save()
                            logger.errors.info("FTP status: Transferring")

                    #--------------------------------
                    # Handle auto-run analysis
                    #--------------------------------
                    composite_exists, thumbnail_exists = reports_exist(exp)
                    logger.errors.debug("Reports Exist? %s %s" %
                                        (composite_exists, thumbnail_exists))
                    if not composite_exists:
                        logger.errors.debug(
                            "No auto-run analysis has been started")
                        # Check if auto-run for whole chip has been requested
                        if exp.autoAnalyze:
                            logger.errors.debug(
                                "  auto-run whole chip analysis has been requested"
                            )
                            #if check_analyze_early(exp) or check_for_completion(exp):
                            if ready_to_process(exp) or check_for_completion(
                                    exp):
                                logger.errors.info(
                                    "  Start a whole chip auto-run analysis job"
                                )
                                generate_http_post(exp)
                            else:
                                logger.errors.info(
                                    "  Do not start a whole chip auto-run job yet"
                                )
                        else:
                            logger.errors.debug(
                                "  auto-run whole chip analysis has not been requested"
                            )
                    else:
                        logger.errors.debug("composite report already exists")

                    if not thumbnail_exists:
                        # Check if auto-run for thumbnail has been requested
                        # But only if its a block dataset
                        if 'tiled' in exp.rawdatastyle:
                            if autorun_thumbnail(exp):
                                logger.errors.debug(
                                    "auto-run thumbnail analysis has been requested"
                                )
                                if ready_to_process_thumbnail(
                                        exp) or check_for_completion(exp):
                                    logger.errors.info(
                                        "  Start a thumbnail auto-run analysis job"
                                    )
                                    generate_http_post(exp, DO_THUMBNAIL)
                                else:
                                    logger.errors.info(
                                        "  Do not start a thumbnail auto-run job yet"
                                    )
                            else:
                                logger.errors.debug(
                                    "auto-run thumbnail analysis has not been requested"
                                )
                        else:
                            logger.errors.debug(
                                "This is not a block dataset; no thumbnail to process"
                            )
                    else:
                        logger.errors.debug("thumbnail report already exists")
                else:
                    logger.errors.debug("exp is empty")
            else:
                # No explog.txt exists; probably an engineering test run
                logger.errors.debug("no %s was found in %s" %
                                    (LOG_BASENAME, f))
        except:
            logger.errors.exception(traceback.format_exc())

    logger.current_folder = '(none)'
Beispiel #17
0
Datei: oiad.py Projekt: LBragg/TS
    def __init__(self,name,config):

        self.name = name
        self.dat_path = os.path.join(config.get('global','results'), name)
        self.sigproc_results_path = os.path.join(self.dat_path, "onboard_results", "sigproc_results")
        self.status = 'new'
        self.last_flow = -1

        explog_file = os.path.join(self.dat_path, "explog.txt")
        if not os.path.exists(explog_file):
            raise Exception("%s doesn't exist" % explog_file)

        try:
            self.explog_file = explog_file

            # parse explog.txt
            (head,tail) = os.path.split(explog_file)
            explogtext = load_log(head,tail)

            self.explogdict = parse_log(explogtext)
            self.exp_flows = int(self.explogdict["flows"])
            self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
            self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
            self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
            self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
            self.exp_planned_run_short_id = self.explogdict.get('planned_run_short_id','')
            self.exp_chiptype = self.explogdict["chiptype"]

            self.exp_chipversion = self.explogdict.get('chipversion','')
            if not self.exp_chipversion or not self.exp_chipversion.startswith('P'):
                self.exp_chipversion = self.exp_chiptype

            logger.info("planned_run_short_id: '%s'" % self.exp_planned_run_short_id)
            logger.info("chiptype: '%s'" % self.exp_chiptype)
            logger.info("chipversion: '%s'" % self.exp_chipversion)


            if os.path.exists('/software/config/DataCollect.config'):
                f = open('/software/config/DataCollect.config')
                text = f.readlines()
                f.close()
                for line in text:
                    [key, value] = line.split(':')
                    key = key.strip()
                    value = value.strip()
                    logger.info('%s %s' % (key, value))
                    if key == 'TSUrl':
                        TSUrl = value
                    elif key == 'TSUserName':
                        TSUserName = value
                    elif key == 'TSPasswd':
                        TSPasswd = value
                    else:
                        continue
            else:
                raise

            logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd))

            headers = {
                "Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
                "Content-type": "application/json",
                "Accept": "application/json" }

            try:
                plan_found = False
                if self.exp_planned_run_short_id:
                    # get plan data
                    conn = httplib.HTTPConnection(TSUrl)
                    conn.request("GET", "/rundb/api/v1/plannedexperiment/?format=json&planShortID=%s&limit=20" % self.exp_planned_run_short_id, "", headers)
                    response = conn.getresponse()
                    logger.info('%s %s' % (response.status, response.reason))
                    data = response.read()
                    # Need to limit plans
                    plans = json.loads(data)
                    logger.debug("number of plans found on %s: %s" % (TSUrl,len(plans['objects'])))
                    logger.debug("search for plan with planShortID: '%s'" % self.exp_planned_run_short_id)

                    plan = [plan for plan in plans['objects'] if plan.get('planShortID','not_available') == self.exp_planned_run_short_id]
                    logger.debug("number of plans with planShortID '%s' found: %s" % (self.exp_planned_run_short_id,len(plan)))
                    if plan:
                        logger.debug("Plan: %s" % plan[0])
                        self.exp_beadfindArgs = plan[0].get('beadfindargs','justBeadFind')
                        self.exp_analysisArgs = plan[0].get('analysisargs','Analysis')
                        self.libraryKey       = plan[0].get('libraryKey','TCAG')
                        self.tfKey            = plan[0].get('tfKey','ATCG')
                        plan_found = True
                    else:
                        logger.debug("Problem with args in plan '%s'" % self.exp_planned_run_short_id)

                if not plan_found:
                    logger.debug("plan not available")
                    # TODO: user might starts run during TS upgrade
                    conn = httplib.HTTPConnection(TSUrl)
                    conn.request("GET", "/rundb/api/v1/analysisargs/?format=json", "", headers)
                    response = conn.getresponse()
                    data = response.read()
                    allanalysisargs = json.loads(data)

                    chip_found = False
                    logger.info('chipversion test: search for chipversion: [%s]' % self.exp_chipversion)

                    for analysisargs in allanalysisargs['objects']:
                        logger.info('chipversion test: compare with chipversion: [%s]' % analysisargs['name'])
                        if analysisargs['name']=="default_"+self.exp_chipversion:
                            logger.info('chipversion test: found analysis args for chipversion: [%s]' % analysisargs['name'])
                            logger.info(analysisargs)
                            self.exp_beadfindArgs = analysisargs['beadfindargs']
                            self.exp_analysisArgs = analysisargs['analysisargs']
                            self.libraryKey       = 'TCAG'
                            self.tfKey            = 'ATCG'
                            chip_found = True
                            break
                    if not chip_found:
                        raise Exception("Chiptype %s not found" % self.exp_chipversion)
            except:
                logger.error(traceback.format_exc())
                self.exp_beadfindArgs = 'justBeadFind --beadfind-minlivesnr 3 --region-size=216x224 --total-timeout 602'
                self.exp_analysisArgs = 'Analysis --from-beadfind --clonal-filter-bkgmodel on --region-size=216x224 --bkg-bfmask-update off --gpuWorkLoad 1 --total-timeout 602'
                self.libraryKey       = "TCAG"
                self.tfKey            = "ATCG"

            logger.info("beadfindArgs: '%s'" % self.exp_beadfindArgs)
            logger.info("analysisArgs: '%s'" % self.exp_analysisArgs)


        except:
            logger.error(traceback.format_exc())
            raise


        self.blocks = []
        try:
            self.block_to_process_start = config.getint('global','block_to_process_start')
            self.block_to_process_end = config.getint('global','block_to_process_end')
            self.discover_blocks()
        except:
            logger.error(traceback.format_exc())
            raise
Beispiel #18
0
def crawl(folders, logger):
    """Crawl over ``folders``, reporting information to the ``CrawlLog``
    ``logger``."""
    if folders:
        logger.errors.info("checking %d directories" % len(folders))

    for f in folders:
        try:
            text = load_log(f, LOG_BASENAME)
            logger.errors.debug("checking directory %s" % f)
            #------------------------------------
            # Check for a valid explog.txt file
            #------------------------------------
            if text is not None:
    
                logger.current_folder = f
                # Catch any errors parsing this log file and continue to the next experiment.
                d = parse_log(text)
                kwargs, status = exp_kwargs(d, f)
    
                if status is True:
                    exp = exp_from_kwargs(kwargs, logger, False)
                    projectName = d.get('project', '')   # part of explog but not a field in Experiment table
                    if exp is not None:
    
                        logger.errors.info("%s" % f)
                        
                        #make sure 1-1 exp-plan relationship is enforced when new experiment is created
                        planObj = exp.plan                    
                        if planObj and exp.pk == None and len(models.Experiment.objects.filter(plan__pk = planObj.pk)) > 0:
                            logger.errors.info('WARNING: Plan %s is already associated to an experiment, a copy will be created' % planObj.pk)
                            planObj.pk = None
                            planObj.save()
                            exp.plan = planObj
    
                        #--------------------------------
                        # Update FTP Transfer Status
                        #--------------------------------
                        if check_for_completion(exp):
                            # FTP transfer is complete
                            if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT:
                                logger.errors.info("FTP status: Aborted")
                            else:
                                exp.ftpStatus = RUN_STATUS_COMPLETE
    
                            exp.log = json.dumps(parse_log(load_log(f, LOG_FINAL_BASENAME)), indent=4)
                            exp.save()
    
                            # Measure and record disk space usage of this dataset
                            # This is celery task which will execute in separate worker thread
                            try:
                                tasks.setRunDiskspace.delay(exp.pk)
                            except:
                                logger.errors.exception(traceback.format_exc())
    
                        else:
                            if exp.ftpStatus != RUN_STATUS_MISSING:
                                exp.ftpStatus = get_percent(exp)
                                exp.save()
                                logger.errors.info("FTP status: Transferring")
    
                        #--------------------------------
                        # Handle auto-run analysis
                        #--------------------------------
                        composite_exists, thumbnail_exists = reports_exist(exp)
                        logger.errors.debug("Reports Exist? %s %s" % (composite_exists, thumbnail_exists))
                        if not composite_exists:
                            logger.errors.debug("No auto-run analysis has been started")
                            # Check if auto-run for whole chip has been requested
                            if exp.autoAnalyze:
                                logger.errors.debug("  auto-run whole chip analysis has been requested")
                                #if check_analyze_early(exp) or check_for_completion(exp):
                                if ready_to_process(exp) or check_for_completion(exp):
                                    logger.errors.info("  Start a whole chip auto-run analysis job")
                                    generate_http_post(exp, projectName)
                                else:
                                    logger.errors.info("  Do not start a whole chip auto-run job yet")
                            else:
                                logger.errors.debug("  auto-run whole chip analysis has not been requested")
                        else:
                            logger.errors.debug("composite report already exists")
    
                        if not thumbnail_exists:
                            # Check if auto-run for thumbnail has been requested
                            # But only if its a block dataset
                            if 'tiled' in exp.rawdatastyle:
                                if autorun_thumbnail(exp):
                                    logger.errors.debug("auto-run thumbnail analysis has been requested")
                                    if ready_to_process_thumbnail(exp) or check_for_completion(exp):
                                        logger.errors.info("  Start a thumbnail auto-run analysis job")
                                        generate_http_post(exp, projectName, DO_THUMBNAIL)
                                    else:
                                        logger.errors.info("  Do not start a thumbnail auto-run job yet")
                                else:
                                    logger.errors.debug("auto-run thumbnail analysis has not been requested")
                            else:
                                logger.errors.debug("This is not a block dataset; no thumbnail to process")
                        else:
                            logger.errors.debug("thumbnail report already exists")
                    else:
                        logger.errors.debug("exp is empty")
                else:
                    logger.errors.warn("folder %s is not added due to invalid experiment status from exp_kwargs. " % f)
            else:
                # No explog.txt exists; probably an engineering test run
                logger.errors.debug("no %s was found in %s" % (LOG_BASENAME, f))
        except:
            logger.errors.exception(traceback.format_exc())
            
    logger.current_folder = '(none)'
Beispiel #19
0
    def __init__(self,name,config):

        self.name = name
        self.dat_path             = os.path.join(config.get('global','results'), name)
        self.analysis_path        = os.path.join(config.get('global','analysisresults'), name)
        self.sigproc_results_path = os.path.join(config.get('global','analysisresults'), name, "onboard_results", "sigproc_results")
        self.status = 'new'
        self.last_flow = -1

        try:
            if not os.path.exists( self.sigproc_results_path ):
                os.makedirs( self.sigproc_results_path )
        except:
            logger.error(traceback.format_exc())

        explog_file = os.path.join(self.analysis_path, "explog.txt")
        if not os.path.exists(explog_file):
            shutil.copyfile(os.path.join(self.dat_path, "explog.txt"), os.path.join(self.analysis_path, "explog.txt"))
        if not os.path.exists(explog_file):
            raise Exception("%s doesn't exist" % explog_file)

        try:
            if not os.path.exists( os.path.join(self.dat_path, 'onboard_results') ):
                os.symlink( os.path.join(self.analysis_path,'onboard_results'), os.path.join(self.dat_path,'onboard_results') )
        except:
            logger.error(traceback.format_exc())

        try:
            self.explog_file = explog_file

            # parse explog.txt
            (head,tail) = os.path.split(explog_file)
            explogtext = load_log(head,tail)

            self.explogdict = parse_log(explogtext)
            self.exp_flows = int(self.explogdict["flows"])
            self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
            self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
            self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
            self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
            self.exp_planned_run_guid = self.explogdict.get('planned_run_guid','')
            self.exp_chiptype = self.explogdict["chiptype"]
            self.exp_seqkitplanname = self.explogdict.get('seqkitplanname','')

            self.exp_chipversion = self.explogdict.get('chipversion','')
            if not self.exp_chipversion:
                self.exp_chipversion = self.exp_chiptype

            logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
            logger.info("chiptype: '%s'" % self.exp_chiptype)
            logger.info("chipversion: '%s'" % self.exp_chipversion)


            try:
                f = open(os.path.join(self.dat_path, "planned_run.json"),'r')
                analysisargs = json.load(f)['objects'][0]
                f.close()

                if analysisargs:
                    self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs']
                    self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs']
                    self.exp_beadfindArgs_block     = analysisargs['beadfindargs']
                    self.exp_analysisArgs_block     = analysisargs['analysisargs']
                    self.libraryKey                 = analysisargs['libraryKey']
                    self.tfKey                      = analysisargs['tfKey']
                else:
                    raise Exception("Analysis args are empty")
            except:
                raise

            logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail)
            logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail)
            logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block)
            logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block)


        except:
            logger.error(traceback.format_exc())
            raise


        self.blocks = []
        try:
            self.block_to_process_start = config.getint('global','block_to_process_start')
            self.block_to_process_end = config.getint('global','block_to_process_end')
            self.discover_blocks()
        except:
            logger.error(traceback.format_exc())
            raise
Beispiel #20
0
    def __init__(self, name, config):

        self.name = name
        self.dat_path = os.path.join(config.get('global', 'results'), name)
        self.analysis_path = os.path.join(config.get('global', 'analysisresults'), name)
        self.sigproc_results_path = os.path.join(
            config.get('global', 'analysisresults'), name, "onboard_results", "sigproc_results")
        self.status = 'new'
        self.last_flow = -1

        try:
            if not os.path.exists(self.sigproc_results_path):
                os.makedirs(self.sigproc_results_path)
        except:
            logger.error(traceback.format_exc())

        explog_file = os.path.join(self.analysis_path, "explog.txt")
        if not os.path.exists(explog_file):
            shutil.copyfile(os.path.join(self.dat_path, "explog.txt"),
                            os.path.join(self.analysis_path, "explog.txt"))
        if not os.path.exists(explog_file):
            raise Exception("%s doesn't exist" % explog_file)

        try:
            if not os.path.exists(os.path.join(self.dat_path, 'onboard_results')):
                os.symlink(os.path.join(self.analysis_path, 'onboard_results'),
                           os.path.join(self.dat_path, 'onboard_results'))
        except:
            logger.error(traceback.format_exc())

        try:
            self.explog_file = explog_file

            # parse explog.txt
            (head, tail) = os.path.split(explog_file)
            explogtext = load_log(head, tail)

            self.explogdict = parse_log(explogtext)
            self.exp_flows = int(self.explogdict["flows"])
            self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis', 'no')
            self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats', 'no')
            self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run', 'yes')
            self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled', 'no')
            self.exp_planned_run_guid = self.explogdict.get('planned_run_guid', '')
            self.exp_chiptype = self.explogdict["chiptype"]
            self.exp_seqkitplanname = self.explogdict.get('seqkitplanname', '')

            self.exp_chipversion = self.explogdict.get('chipversion', '')
            if not self.exp_chipversion:
                self.exp_chipversion = self.exp_chiptype

            logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
            logger.info("chiptype: '%s'" % self.exp_chiptype)
            logger.info("chipversion: '%s'" % self.exp_chipversion)

            try:
                analysisargs = {}
                if os.path.exists(os.path.join(self.dat_path, "reanalysis_args.json")):
                    f = open(os.path.join(self.dat_path, "reanalysis_args.json"), 'r')
                    analysisargs = json.load(f)
                    f.close()
                    logger.info("use analysisargs from %s" %
                                os.path.join(self.dat_path, "reanalysis_args.json"))
                elif os.path.exists(os.path.join(self.dat_path, "planned_run.json")):
                    f = open(os.path.join(self.dat_path, "planned_run.json"), 'r')
                    analysisargs = json.load(f)['objects'][0]
                    f.close()
                    logger.info("use analysisargs from %s" % os.path.join(self.dat_path, "planned_run.json"))
                else:
                    logger.info("trying to fetch planned run for %s" %
                                os.path.join(self.dat_path, "planned_run.json"))
                    http_command = "python /software/testing/get_reanalysis_args.py -e %s" % (name)
                    args = shlex.split(http_command.encode('utf8'))
                    logger.info("run process: %s" % (args))
                    outfile = open(os.path.join(self.dat_path, 'get_reanalysis_args_oia.log'), "w")
                    p = subprocess.Popen(args, stdout=outfile, stderr=subprocess.STDOUT)
                    # add popen process to block
                    ret = p.wait()
                    outfile.close()
                    if os.path.exists(os.path.join(self.dat_path, "planned_run.json")):
                        with open(os.path.join(self.dat_path, "planned_run.json"), 'r') as f:
                            try:
                                analysisargs = json.load(f)['objects'][0]
                                logger.info("Using Plan information from %s" %
                                            os.path.join(self.dat_path, "planned_run.json"))
                            except IndexError:
                                raise Exception("File %s is empty",
                                                os.path.join(self.dat_path, "planned_run.json"))

                if len(analysisargs) > 0:
                    self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs']
                    self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs']
                    self.exp_beadfindArgs_block = analysisargs['beadfindargs']
                    self.exp_analysisArgs_block = analysisargs['analysisargs']
                    self.exp_prebasecallerArgs_block = analysisargs['prebasecallerargs']
                    self.libraryKey = analysisargs['libraryKey']
                    self.tfKey = analysisargs['tfKey']
                else:
                    raise Exception("There are no Analysis args available")
            except:
                raise

            logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail)
            logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail)
            logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block)
            logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block)
            logger.info("block prebasecallerArgs: '%s'" % self.exp_prebasecallerArgs_block)

        except:
            logger.error(traceback.format_exc())
            raise

        self.blocks = []
        try:
            self.block_to_process_start = config.getint('global', 'block_to_process_start')
            self.block_to_process_end = config.getint('global', 'block_to_process_end')
            self.discover_blocks()
        except:
            logger.error(traceback.format_exc())
            raise
Beispiel #21
0
def crawl(folders, logger):
    """Crawl over ``folders``, reporting information to the ``CrawlLog``
    ``logger``."""
    if folders:
        logger.errors.info("checking %d directories" % len(folders))

    for f in folders:
        try:
            text = load_log(f, LOG_BASENAME)
            logger.errors.debug("checking directory %s" % f)
            #------------------------------------
            # Check for a valid explog.txt file
            #------------------------------------
            if text is not None:

                logger.current_folder = f
                # Catch any errors parsing this log file and continue to the next experiment.
                try:
                        d = parse_log(text)
                except:
                	logger.errors.info("Error parsing explog, skipping %s" % f)
                	logger.errors.exception(traceback.format_exc())
                	continue

                exp = None
                exp_set = models.Experiment.objects.filter(unique=f)
                if exp_set:
                    # Experiment already in database
                    exp = exp_set[0]
                    if exp.ftpStatus == RUN_STATUS_COMPLETE:
                        continue
                else:
                    # Need to create experiment for this folder
                    try:
                        planObj, expObj, easObj = get_planned_exp_objects(d, f, logger.errors)
                        exp = update_exp_objects_from_log(d, f, planObj, expObj, easObj, logger.errors)
                    except:
                        logger.errors.info("Error updating experiment objects, skipping %s" % f)
                        logger.errors.exception(traceback.format_exc())
                        continue

                if exp is not None:

                    logger.errors.info("%s" % f)

                    #--------------------------------
                    # Update FTP Transfer Status
                    #--------------------------------
                    if check_for_completion(exp):
                        exptxt = load_log(f, LOG_FINAL_BASENAME)
                        if exptxt is not None:
                            exp.log = json.dumps(parse_log(exptxt), indent=4)
                        else:
                            # explog_final exists, but is not readable yet.
                            continue

                        # FTP transfer is complete
                        if exp.ftpStatus == RUN_STATUS_ABORT or exp.ftpStatus == RUN_STATUS_SYS_CRIT:
                            logger.errors.info("FTP status: Aborted")
                        else:
                            exp.ftpStatus = RUN_STATUS_COMPLETE
                        exp.save()

                        # Measure and record disk space usage of this dataset
                        # This is celery task which will execute in separate worker thread
                        try:
                            tasks.setRunDiskspace.delay(exp.pk)
                        except:
                            logger.errors.exception(traceback.format_exc())

                    else:
                        if exp.ftpStatus != RUN_STATUS_MISSING:
                            exp.ftpStatus = get_filecount(exp)
                            exp.save()
                            logger.errors.info("FTP status: Transferring")

                    #--------------------------------
                    # Handle auto-run analysis
                    #--------------------------------
                    composite_exists, thumbnail_exists = reports_exist(exp)
                    logger.errors.debug("Reports Exist? %s %s" % (composite_exists, thumbnail_exists))
                    if not composite_exists:
                        logger.errors.debug("No auto-run analysis has been started")
                        # Check if auto-run for whole chip has been requested
                        if exp.autoAnalyze:
                            logger.errors.debug("  auto-run whole chip analysis has been requested")
                            #if check_analyze_early(exp) or check_for_completion(exp):
                            if ready_to_process(exp) or check_for_completion(exp):
                                logger.errors.info("  Start a whole chip auto-run analysis job")
                                generate_http_post(exp, logger)
                            else:
                                logger.errors.info("  Do not start a whole chip auto-run job yet")
                        else:
                            logger.errors.debug("  auto-run whole chip analysis has not been requested")
                    else:
                        logger.errors.debug("composite report already exists")

                    if not thumbnail_exists:
                        # Check if auto-run for thumbnail has been requested
                        # But only if its a block dataset
                        if 'tiled' in exp.rawdatastyle:
                            if autorun_thumbnail(exp, logger):
                                logger.errors.debug("auto-run thumbnail analysis has been requested")
                                if ready_to_process_thumbnail(exp) or check_for_completion(exp):
                                    logger.errors.info("  Start a thumbnail auto-run analysis job")
                                    generate_http_post(exp, logger, DO_THUMBNAIL)
                                else:
                                    logger.errors.info("  Do not start a thumbnail auto-run job yet")
                            else:
                                logger.errors.debug("auto-run thumbnail analysis has not been requested")
                        else:
                            logger.errors.debug("This is not a block dataset; no thumbnail to process")
                    else:
                        logger.errors.debug("thumbnail report already exists")
                else:
                    logger.errors.debug("exp is empty")
            else:
                # No explog.txt exists; probably an engineering test run
                logger.errors.debug("no %s was found in %s" % (LOG_BASENAME, f))
        except:
            logger.errors.exception(traceback.format_exc())

    logger.current_folder = '(none)'
Beispiel #22
0
Datei: oiad.py Projekt: rb94/TS
    def __init__(self,name,config):

        self.name = name
        self.dat_path             = os.path.join(config.get('global','results'), name)
        self.analysis_path        = os.path.join(config.get('global','analysisresults'), name)
        self.sigproc_results_path = os.path.join(config.get('global','analysisresults'), name, "onboard_results", "sigproc_results")
        self.status = 'new'
        self.last_flow = -1

        try:
            if not os.path.exists( self.sigproc_results_path ):
                os.makedirs( self.sigproc_results_path )
        except:
            logger.error(traceback.format_exc())

        explog_file = os.path.join(self.analysis_path, "explog.txt")
        if not os.path.exists(explog_file):
            shutil.copyfile(os.path.join(self.dat_path, "explog.txt"), os.path.join(self.analysis_path, "explog.txt"))
        if not os.path.exists(explog_file):
            raise Exception("%s doesn't exist" % explog_file)

        try:
            if not os.path.exists( os.path.join(self.dat_path, 'onboard_results') ):
                os.symlink( os.path.join(self.analysis_path,'onboard_results'), os.path.join(self.dat_path,'onboard_results') )
        except:
            logger.error(traceback.format_exc())

        try:
            self.explog_file = explog_file

            # parse explog.txt
            (head,tail) = os.path.split(explog_file)
            explogtext = load_log(head,tail)

            self.explogdict = parse_log(explogtext)
            self.exp_flows = int(self.explogdict["flows"])
            self.exp_oninstranalysis = 'yes' in self.explogdict.get('oninstranalysis','no')
            self.exp_usesynchdats = 'yes' in self.explogdict.get('use_synchdats','no')
            self.exp_oia_during_run = 'yes' in self.explogdict.get('oia_during_run','yes')
            self.exp_ecc_enabled = 'yes' in self.explogdict.get('ecc_enabled','no')
            self.exp_planned_run_guid = self.explogdict.get('planned_run_guid','')
            self.exp_chiptype = self.explogdict["chiptype"]
            self.exp_seqkitplanname = self.explogdict.get('seqkitplanname','')

            self.exp_chipversion = self.explogdict.get('chipversion','')
            if not self.exp_chipversion:
                self.exp_chipversion = self.exp_chiptype

            logger.info("planned_run_guid: '%s'" % self.exp_planned_run_guid)
            logger.info("chiptype: '%s'" % self.exp_chiptype)
            logger.info("chipversion: '%s'" % self.exp_chipversion)


            try:

                logger.info('connect to:%s user:%s password:%s' % (TSUrl,TSUserName,TSPasswd))

                headers = {
                "Authorization": "Basic "+ string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
                "Content-type": "application/json",
                "Accept": "application/json" }

                conn = httplib.HTTPConnection(TSUrl)
                if self.exp_planned_run_guid:
                    # TODO, use original plan data from file system
                    request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&planGUID=%s&sequenceKitName=%s&limit=20" % (self.exp_chipversion, self.exp_planned_run_guid, self.exp_seqkitplanname)
                else:
                    logger.debug("use default chip args")
                    request = "/rundb/api/v1/analysisargs/getargs/?format=json&chipType=%s&sequenceKitName=%s&limit=20" % (self.exp_chipversion, self.exp_seqkitplanname)

                #http://frenzy.ite/rundb/api/v1/analysisargs/getargs/?format=json&chipType=P1.2.18&planGUID=&sequenceKitName=ProtonI200Kit-v2
                logger.info('request: %s http://%s%s' % (self.explogdict.get('runname','unknown'), TSUrl, request))
                conn.request("GET", request, "", headers)

                response = conn.getresponse()
                logger.info('%s %s' % (response.status, response.reason))
                data = response.read()
                analysisargs = json.loads(data)
                logger.debug("Plan: %s" % analysisargs)

                f = open(os.path.join(self.sigproc_results_path, "retrieved_args.json"),'w')
                f.write(json.dumps(analysisargs, indent=4))
                f.close()

                if analysisargs:
                    self.exp_beadfindArgs_thumbnail = analysisargs['thumbnailbeadfindargs'] 
                    self.exp_analysisArgs_thumbnail = analysisargs['thumbnailanalysisargs']
                    self.exp_beadfindArgs_block     = analysisargs['beadfindargs']
                    self.exp_analysisArgs_block     = analysisargs['analysisargs']
                    self.libraryKey                 = 'TCAG'
                    self.tfKey                      = 'ATCG'
                else:
                    raise Exception("Analysis args are empty")
            except:
                raise

            logger.info("thumbnail beadfindArgs: '%s'" % self.exp_beadfindArgs_thumbnail)
            logger.info("thumbnail analysisArgs: '%s'" % self.exp_analysisArgs_thumbnail)
            logger.info("block beadfindArgs: '%s'" % self.exp_beadfindArgs_block)
            logger.info("block analysisArgs: '%s'" % self.exp_analysisArgs_block)


        except:
            logger.error(traceback.format_exc())
            raise


        self.blocks = []
        try:
            self.block_to_process_start = config.getint('global','block_to_process_start')
            self.block_to_process_end = config.getint('global','block_to_process_end')
            self.discover_blocks()
        except:
            logger.error(traceback.format_exc())
            raise
Beispiel #23
0
Datei: oiad.py Projekt: dkeren/TS
    def __init__(self, name, config):

        self.name = name
        self.dat_path = os.path.join(config.get('global', 'results'), name)
        self.sigproc_results_path = os.path.join(self.dat_path,
                                                 "onboard_results",
                                                 "sigproc_results")
        self.status = 'new'
        self.last_flow = -1

        explog_file = os.path.join(self.dat_path, "explog.txt")
        if not os.path.exists(explog_file):
            raise Exception("%s doesn't exist" % explog_file)

        try:
            self.explog_file = explog_file

            # parse explog.txt
            (head, tail) = os.path.split(explog_file)
            explogtext = load_log(head, tail)

            self.explogdict = parse_log(explogtext)
            self.exp_flows = int(self.explogdict["flows"])
            self.exp_oninstranalysis = 'yes' in self.explogdict.get(
                'oninstranalysis', 'no')
            self.exp_usesynchdats = 'yes' in self.explogdict.get(
                'use_synchdats', 'no')
            self.exp_oia_during_run = 'yes' in self.explogdict.get(
                'oia_during_run', 'yes')
            self.exp_ecc_enabled = 'yes' in self.explogdict.get(
                'ecc_enabled', 'no')
            self.exp_planned_run_short_id = self.explogdict.get(
                'planned_run_short_id', '')
            self.exp_chiptype = self.explogdict["chiptype"]

            self.exp_chipversion = self.explogdict.get('chipversion', '')
            if not self.exp_chipversion or not self.exp_chipversion.startswith(
                    'P'):
                self.exp_chipversion = self.exp_chiptype

            logger.info("planned_run_short_id: '%s'" %
                        self.exp_planned_run_short_id)
            logger.info("chiptype: '%s'" % self.exp_chiptype)
            logger.info("chipversion: '%s'" % self.exp_chipversion)

            if os.path.exists('/software/config/DataCollect.config'):
                f = open('/software/config/DataCollect.config')
                text = f.readlines()
                f.close()
                for line in text:
                    [key, value] = line.split(':')
                    key = key.strip()
                    value = value.strip()
                    logger.info('%s %s' % (key, value))
                    if key == 'TSUrl':
                        TSUrl = value
                    elif key == 'TSUserName':
                        TSUserName = value
                    elif key == 'TSPasswd':
                        TSPasswd = value
                    else:
                        continue
            else:
                raise

            logger.info('connect to:%s user:%s password:%s' %
                        (TSUrl, TSUserName, TSPasswd))

            headers = {
                "Authorization":
                "Basic " +
                string.strip(base64.encodestring(TSUserName + ':' + TSPasswd)),
                "Content-type":
                "application/json",
                "Accept":
                "application/json"
            }

            try:
                plan_found = False
                if self.exp_planned_run_short_id:
                    # get plan data
                    conn = httplib.HTTPConnection(TSUrl)
                    conn.request(
                        "GET",
                        "/rundb/api/v1/plannedexperiment/?format=json&planShortID=%s&limit=20"
                        % self.exp_planned_run_short_id, "", headers)
                    response = conn.getresponse()
                    logger.info('%s %s' % (response.status, response.reason))
                    data = response.read()
                    # Need to limit plans
                    plans = json.loads(data)
                    logger.debug("number of plans found on %s: %s" %
                                 (TSUrl, len(plans['objects'])))
                    logger.debug("search for plan with planShortID: '%s'" %
                                 self.exp_planned_run_short_id)

                    plan = [
                        plan for plan in plans['objects']
                        if plan.get('planShortID', 'not_available') ==
                        self.exp_planned_run_short_id
                    ]
                    logger.debug(
                        "number of plans with planShortID '%s' found: %s" %
                        (self.exp_planned_run_short_id, len(plan)))
                    if plan:
                        logger.debug("Plan: %s" % plan[0])
                        self.exp_beadfindArgs = plan[0].get(
                            'beadfindargs', 'justBeadFind')
                        self.exp_analysisArgs = plan[0].get(
                            'analysisargs', 'Analysis')
                        self.libraryKey = plan[0].get('libraryKey', 'TCAG')
                        self.tfKey = plan[0].get('tfKey', 'ATCG')
                        plan_found = True
                    else:
                        logger.debug("Problem with args in plan '%s'" %
                                     self.exp_planned_run_short_id)

                if not plan_found:
                    logger.debug("plan not available")
                    # TODO: user might starts run during TS upgrade
                    conn = httplib.HTTPConnection(TSUrl)
                    conn.request("GET",
                                 "/rundb/api/v1/analysisargs/?format=json", "",
                                 headers)
                    response = conn.getresponse()
                    data = response.read()
                    allanalysisargs = json.loads(data)

                    chip_found = False
                    logger.info(
                        'chipversion test: search for chipversion: [%s]' %
                        self.exp_chipversion)

                    for analysisargs in allanalysisargs['objects']:
                        logger.info(
                            'chipversion test: compare with chipversion: [%s]'
                            % analysisargs['name'])
                        if analysisargs[
                                'name'] == "default_" + self.exp_chipversion:
                            logger.info(
                                'chipversion test: found analysis args for chipversion: [%s]'
                                % analysisargs['name'])
                            logger.info(analysisargs)
                            self.exp_beadfindArgs = analysisargs[
                                'beadfindargs']
                            self.exp_analysisArgs = analysisargs[
                                'analysisargs']
                            self.libraryKey = 'TCAG'
                            self.tfKey = 'ATCG'
                            chip_found = True
                            break
                    if not chip_found:
                        raise Exception("Chiptype %s not found" %
                                        self.exp_chipversion)
            except:
                logger.error(traceback.format_exc())
                self.exp_beadfindArgs = 'justBeadFind --beadfind-minlivesnr 3 --region-size=216x224 --total-timeout 602'
                self.exp_analysisArgs = 'Analysis --from-beadfind --clonal-filter-bkgmodel on --region-size=216x224 --bkg-bfmask-update off --gpuWorkLoad 1 --total-timeout 602'
                self.libraryKey = "TCAG"
                self.tfKey = "ATCG"

            logger.info("beadfindArgs: '%s'" % self.exp_beadfindArgs)
            logger.info("analysisArgs: '%s'" % self.exp_analysisArgs)

        except:
            logger.error(traceback.format_exc())
            raise

        self.blocks = []
        try:
            self.block_to_process_start = config.getint(
                'global', 'block_to_process_start')
            self.block_to_process_end = config.getint('global',
                                                      'block_to_process_end')
            self.discover_blocks()
        except:
            logger.error(traceback.format_exc())
            raise