def Request(self, run=None, **kwjob): """Ask for an available host and block resources. """ info = self.action('available_host', **kwjob) host = info.get(self.d_crit['host'][0], None) if _DBG_ALLOC: print3('job allocated on %s : %s' % (host, pformat(kwjob))) if host is not None: status = ALLOCATED for key in self.job_keys: if isnum(kwjob.get(key)): self.action('add', host, key + 'run', kwjob[key]) else: status = NORESOURCE for key, lim in self.limit.items(): if kwjob.get(key) and (\ (self.d_crit[key][2] and kwjob[key] > lim) or \ (not self.d_crit[key][2] and kwjob[key] < lim) ): status = OVERLIMIT if run: run.DBG("OVERLIMIT %s : requested = %s limit = %s (reverse = %s)" \ % (key, kwjob[key], lim, self.d_crit[key][2])) break self.action('store_job', host, **kwjob) if _DBG_ALLOC: print3(self.Load()) return host, status
def __setid(self, value): """Positionne le champ idkey de l'objet """ if not self.idkey in self.refs: self.values[self.idkey] = value elif __verbose__: print3('Warning <setid>: %s is an ITEM object.' % repr(self.idkey))
def __init__(self, config_file, run=None, version_path=None): """config_file : filename of the 'config.txt' file to read run : AsterRun object (optional) version_path : directory of the version (if it is not the dirname of 'config_file'). """ # ----- initialisations self.config = {} self.filename = config_file self.dirn = version_path or get_absolute_dirname(config_file) self.verbose = False self.debug = False # ----- reference to AsterRun object which manages the execution self.run = run if run != None: self.verbose = run['verbose'] self.debug = run['debug'] # ----- set optional/defaults values (ALWAYS AS LIST !) self.config = DEFAULTS.copy() # ----- read config file if not osp.isfile(config_file): self._mess(ufmt(_(u'file not found : %s'), config_file), '<F>_FILE_NOT_FOUND') f = open(config_file, 'r') content = f.read() f.close() self.config.update(self._parse(content)) if self.debug: print3('<DBG> <init> AsterConfig :') print3(self)
def ended(self, job, opts, itemid, calcul, res): """Call when a job is ended.""" line = self.summary_line(job, opts, res) # printing line is not thread safe but it's only printing! print3(line) add_to_tail(self.reptrav, line) # count nook for each thread gravity = self.run.GetGrav(calcul.diag) error = gravity == -9 or gravity >= self.run.GetGrav('NOOK') if error: self.nbnook[opts['threadid']] += 1 add_to_tail(self.resudir, line, filename='NOOK') add_to_tail(self.resudir, line, filename='RESULTAT') output_filename = _(u'no error or flashdir not defined') # copy output/error to flashdir if self.flashdir != None: try: if not osp.isdir(self.flashdir): self.run.MkDir(self.flashdir, niverr='SILENT') except OSError: pass self.run.Copy(self.flashdir, calcul.flash('output'), calcul.flash('error'), calcul.flash('export'), niverr='<A>_ALARM') output_filename = osp.join(self.flashdir, osp.basename(calcul.flash('output'))) result = [job, opts] result.extend(res) result.append(output_filename) # clean flasheur calcul.kill() return result
def add_filter(self, typ, *args): """Ajoute un ou plusieurs filtres. typ = 'regexp', 'para' ou 'user'. """ if typ == 'command': assert len(args) == 1 expr = self.decode_cmde(args[0]) if expr is not None: self.filter.append(FILTER_REGEXP(expr)) elif typ == 'regexp': assert len(args) == 1 expr = args[0] if expr is not None: self.filter.append(FILTER_REGEXP(expr)) elif typ == 'para': assert len(args) == 1 self.filter.append(FILTER_PARA(args[0])) elif typ == 'user': assert len(args) == 1 d = globals().copy() execfile(args[0], d) if d.has_key('user_filter'): if type(d['user_filter']) in (list, tuple): l_filter = d['user_filter'] else: l_filter = [ d['user_filter'], ] for ff in l_filter: assert isinstance(ff, FILTER) self.filter.extend(l_filter) else: print3('>>> unknown filter type (%s) : ignored.' % typ)
def getrepr(self, refs=True): """Affichage. Si 'refs'=False, on n'imprime pas les sous-items. """ txt = [] for k, v in self.values.items(): if k in self.refs.keys(): item = None if isinstance(self[k], ITEM): item = v idref = v.get(v.idkey) else: item = eval('%s(%s)' % (self.refs[k], 'c=self.c')) try: item.read({item.idkey:v}) idref = v except (ConnectDBError, ReadDBError), msg: item = None if __verbose__: print3(msg) if item != None and (item.__class__.__name__ != self.__class__.__name__\ or idref != self.get(self.idkey)): if not refs: txt.append(fmt_vr % (k, idref)) else: txt.append(fmt_id % (k, idref)) txt.append(os.linesep.join([' > %s' \ % l for l in repr(item).split(os.linesep)])) txt.append(fmt_fin) else: txt.append(fmt_vr % (k, v)) else: if type(v) == str: v = to_unicode(v) txt.append(fmt_val % (k, v))
def AsDict(self): """Retourne un dictionnaire avec toutes les valeurs, valeurs du champ "primaire" pour les sous-items. """ def repr_val(val): v = val if type(val) in (str, unicode): mat = re.search(re_date, val) if mat != None: v = mat.group(1) return v dico = {} for k, v in self.values.items(): if k.startswith('__'): continue if k in self.refs.keys(): item = None if isinstance(self[k], ITEM): item = v idref = v.get(v.idkey) else: item = eval('%s(%s)' % (self.refs[k], 'c=self.c')) try: item.read({item.idkey:v}) idref = v except (ConnectDBError, ReadDBError), msg: item = None if __verbose__: print3(msg) if item != None and (item.__class__.__name__ != self.__class__.__name__\ or idref != self.get(self.idkey)): dico[k] = repr_val(item.GetPrimValue()) elif not v is None: dico[k] = repr_val(v)
def Sortie(self, exit_code): """Exit by printing diagnostic if defined or exit code if not null""" # print important messages msg = self.get_important_messages() if len(msg) > 0: print3(msg) # raise an exception to be catched by the Worker object if not is_main_thread(): raise TaskAbort(msg) # helps to locate error in PROGRAM_ERROR case or in verbose mode if self.diag == '<F>_PROGRAM_ERROR' \ or (exit_code != 0 and (self['debug'] or self['verbose'] or not self.ExitOnFatalError)): self.DBG(u'### Raise RunAsterError exception because of program error or debug mode ###') raise RunAsterError, exit_code # stop all timers if self.print_timer and hasattr(self, 'timer'): print3(os.linesep, self.timer) if self.print_timer: # related to non silent actions print3(self.program + ' ' + self.__version__) if self.diag: # change <E> to <F> self.diag = self.diag.replace('<E>', '<F>') print3(self.fmt['diag'] % self.diag) if exit_code == 0 and \ ("NOOK" in self.diag or self.diag == "NO_TEST_RESU"): exit_code = 1 if self.PrintExitCode or exit_code != 0: print3(self.fmt['exit'] % exit_code) self._clear_tmp() self.DBG("exit %s" % exit_code) sys.exit(exit_code)
def VerbIgnore(self, verbose=None): """End message in verbose mode """ if verbose == None: verbose = self.verbose if verbose: print3(label['SKIP'])
def update(self, table, line, cond): """Update 'line' (as a list of dict "column:value") into the 'table' where 'cond' is respected. Return the number of updated lines. """ if not hasattr(self.cursor, 'fetchone'): print3(_(u'Connection not opened.')) return 0 query = ['UPDATE %s SET' % table] lset = [] for k, v in line.items(): if v != None: if type(v) is LongType: v = '%d' % v lset.append('%s=%s' % (k, repr(v))) query.append(', '.join(lset)) query.append('WHERE') lwh = [] for k, v in cond.items(): if v != None: if type(v) is LongType: v = '%d' % v lwh.append('%s=%s' % (k, repr(v))) query.append(' AND '.join(lwh)) query.append(';') if self.verbose: print3('### Query : %s' % (' '.join(query))) try: nb = self.cursor.execute(' '.join(query)) except: traceback.print_exc() raise MySQLError, 'ERROR query = %s' % (' '.join(query)) return nb
def get_first(self, values=None): """Return the most available host. """ if values is None: values = self.infos[:] if len(values) == 0: return {} for crit, rev in self.l_crit: values.sort(reverse=rev) if _DBG_TRI: print3(crit) pprint(values) val0 = values[0][crit] new = [ values[0], ] for info in values[1:]: if info[crit] != val0: break new.append(info) values = new if len(values) == 1: break if _DBG_TRI: print3('--- FIN ---') pprint(values) return values[0]
def _read_rc(self, ficrc, destdict, optional=False, mcsimp=None): """Read a ressource file and store variables to 'destdict'.""" if osp.isfile(ficrc): read_rcfile(ficrc, destdict, mcsimp=mcsimp) elif not optional: print3(ufmt(_(u'file not found : %s'), ficrc)) self.Sortie(4)
def _mess(self, msg, cod='', store=False): """Just print a message """ if hasattr(self.run, 'Mess'): self.run.Mess(msg, cod, store) else: print3('%-18s %s' % (cod, msg))
def Insert(run, *args): """Insert an execution to the database. """ if len(args) != 1: run.parser.error(_(u"'--%s' takes exactly %d arguments (%d given)") % \ (run.current_action, 1, len(args))) iret = 0 if not run.config.get('astketud'): run.Mess(_(u"'astketud' is not defined in 'agla' configuration file."), '<F>_AGLA_ERROR') # 1. copy export file jn = run['num_job'] fprof = get_tmpname(run, run['tmp_user'], basename='etude_prof') kret = run.Copy(fprof, args[0], niverr='<F>_PROFILE_COPY') # 2. insert study in database cmd = '%(astketud)s %(profile)s' % { 'astketud': run['astketud'], 'profile': fprof, } iret, output = run.Shell(cmd) if iret != 0: run.Mess(output, '<F>_DB_ERROR') # astketud always returns exit=0 ! else: print3(output)
def Mess(self, msg, cod='', store=False): """Print a message sur stdout, 'cod' is an error code (format "<.>_...") <E> : continue, <F> : stop, <A> : alarm, continue, '' or INFO : for an info, SILENT : like an info without <INFO> string, TITLE : add a separator. If cod='<F>' (without a description), if exists last <E> error is transformed into <F>, else <F>_ABNORMAL_ABORT. If store is True, 'msg' is stored in print_on_exit dictionnary.""" # ----- gravite g0 = self.GetGrav(self.diag) g1 = self.GetGrav(cod) coderr = cod if cod == '<F>': if g0 < self.GetGrav('<E>'): coderr = '<F>_ABNORMAL_ABORT' else: coderr = self.diag.replace('<E>', '<F>') if g1 >= self.GetGrav('<A>'): self.DBG(u'Warning or error raised :', '%s %s' % (cod, msg), print_traceback=True) # ----- message if cod == '' or cod == 'INFO': fmt = self.fmt['msg_info'] coderr = '' elif cod == 'SILENT': fmt = self.fmt['silent'] coderr = '' elif cod == 'TITLE': fmt = self.fmt['title'] coderr = '' else: fmt = self.fmt['msg+cod'] # unknown flag if g1 == -9: coderr = '<I> '+coderr print3(ufmt(fmt, coderr, msg)) magic.get_stdout().flush() # ----- store in print_on_exit if store or (not self.ExitOnFatalError and g1 >= self.GetGrav('<S>')): k = '?' msg2 = msg mat = re.search('<(.)>', cod) if mat != None and mat.group(1) in ('A', 'S', 'E', 'F'): k = mat.group(1) msg2 = self.fmt['msg+cod'] % (coderr, msg) elif cod in ('OK', 'NOOK'): k = cod self.print_on_exit[k] = self.print_on_exit.get(k, []) + [msg2, ] # ----- diagnostic le plus défavorable if g1 > g0: self.diag = coderr if g1 == self.GetGrav('<F>'): self.Sortie(4)
def print_stats(self): """Affiche les stats sur les données lues/construites """ if self.verbose: print3('%6d messages appelés dans les sources' % len(self.d_msg_call.keys())) print3('%6d fichiers sources appellent le catalogue de messages' % len(self.d_msg_used.keys()))
def search_message(self, fich): """Retourne les messages utilisés dans 'fich'. """ try: txt = open(fich, 'r').read() except IOError, msg: print3(_(u'Error with file %s : %s') % (fich, msg)) return []
def _dbg(self, *args, **kargs): """Print debug informations """ if hasattr(self.run, 'DBG'): kargs['stack_id'] = kargs.get('stack_id', 1) self.run.DBG(*args, **kargs) elif self.debug: print3('<DBG> %s // %s' % (str(args), str(kargs)))
def _get_list_python(self): """Retourne la liste des fichiers python """ s_pyt = set() s_pyt.update(glob(osp.join(self.pyt, '*.py'))) s_pyt.update(glob(osp.join(self.pyt, '*', '*.py'))) s_pyt.update(glob(osp.join(self.capy, '*.capy'))) s_pyt.update(glob(osp.join(self.capy, '*', '*.capy'))) if self.build_obj.support('waf'): l_pyt = [osp.abspath(f) for f in s_pyt] l_pyt.sort() return l_pyt d_py = {} for f in s_pyt: typ = osp.splitext(f)[-1][1:] key = self.build_obj.GetCModif(typ, f) assert d_py.get(key) is None, 'ERROR : %s (old : %s)' % ( key, d_py[key]) d_py[key] = f # surcharge s_surch = set() for dsrc in self.surch_pyt: s_surch.update(glob(osp.join(dsrc, '*.py'))) s_surch.update(glob(osp.join(dsrc, '*', '*.py'))) s_surch.update(glob(osp.join(dsrc, '*.capy'))) s_surch.update(glob(osp.join(dsrc, '*', '*.capy'))) if len(s_surch) > 0: l_surch = list(s_surch) l_surch.sort() print3('%5d module(s) python en surcharge : ' % len(l_surch)) print3(os.linesep.join(l_surch)) # retirer des sources originaux ceux de la surcharge... s_suppr = set() for f in s_surch: typ = osp.splitext(f)[-1][1:] key = self.build_obj.GetCModif(typ, f) fexist = d_py.get(key) if fexist: s_suppr.add(fexist) if self.verbose: print3('suppression :', fexist) # ... et ceux de l'unigest if self.unig: iunig = 0 for typ in ('py', 'capy'): for f in self.unig[typ]: iunig += 1 fabs = osp.abspath(osp.join(self.repref, f)) key = self.build_obj.GetCModif(typ, fabs) s_suppr.add(d_py.get(key, '')) if self.verbose: print3('suppression :', fabs) if iunig > 0: print3('%5d module(s) python supprime(s).' % iunig) s_pyt.difference_update(s_suppr) s_pyt.update(s_surch) l_pyt = [osp.abspath(f) for f in s_pyt] l_pyt.sort() return l_pyt
def __init__(self, repref, fort=('bibfor', 'bibf90'), pyt='bibpyt', capy='catapy', cache_dict=None, verbose=True, force=False, ignore_comments=True, debug=False, surch_fort=[], surch_pyt=[], unig=None): """Initialisations.""" if type(fort) not in (list, tuple): fort = [ fort, ] if type(surch_fort) not in (list, tuple): surch_fort = [ surch_fort, ] if type(surch_pyt) not in (list, tuple): surch_pyt = [ surch_pyt, ] self.repref = repref self.fort = [osp.join(repref, rep) for rep in fort] self.pyt = osp.join(repref, pyt) self.capy = osp.join(repref, capy) self.surch_fort = [osp.abspath(d) for d in surch_fort] self.surch_pyt = [osp.abspath(d) for d in surch_pyt] self.verbose = verbose self.debug = debug self.force = force self.message_dir = 'Messages' self.cache_dict = cache_dict or osp.join('/tmp/', 'messages_dicts.pick') self.ignore_comments = ignore_comments self.wrk_fort = osp.join(os.getcwdu(), 'F') self.wrk_pyt = osp.join(os.getcwdu(), 'PY') self.unig = unig # init AsterConfig, AsterBuild objects fconf = osp.join(self.repref, 'config.txt') self.conf_obj = AsterConfig(fconf, run=None) self.build_obj = AsterBuild(run=None, conf=self.conf_obj) if unig: # use with_fordepla=False not to mark moved files as removed self.unig = unigest2dict(unig, self.conf_obj, with_fordepla=True) if self.verbose: print3('Repertoires :\n REF=%s\n FORTRAN=%s\n PYTHON=%s\n CAPY=%s' \ % (self.repref, self.fort, self.pyt, self.capy)) self.l_cata = self._get_list_cata() self.l_src, self.l_pyt = self._get_list_src()
def __getid(self): """Retourne la valeur du champ idkey de l'objet """ value = self.values[self.idkey] if not self.idkey in self.refs or not isinstance(value, ITEM): return value else: if __verbose__: print3('Warning <getid>: %s is an ITEM object.' % repr(self.idkey)) return value.get(value.idkey)
def copyfileD(run, df, icomm, ncomm): """Copy datas from `df` into current directory. Raise only <E> if an error occurs run CheckOK() after. """ dest = None # 1. ----- if logical unit is set : fort.* if df['ul'] != 0 or df['type'] in ('nom',): dest = 'fort.%d' % df['ul'] if df['type'] == 'nom': dest = osp.basename(df['path']) # exception for multiple command files (adding _N) if df['ul'] == 1: icomm += 1 format = '%%0%dd' % (int(log10(max(1, ncomm))) + 1) dest = dest + '.' + format % icomm # warning if file already exists if run.Exists(dest): run.Mess(ufmt(_(u"'%s' overwrites '%s'"), df['path'], dest), '<A>_COPY_DATA') if df['compr']: dest = dest + '.gz' # 2. ----- bases and directories (ul=0) else: # base if df['type'] in ('base', 'bhdf'): dest = osp.basename(df['path']) # ensi elif df['type'] == 'ensi': dest = 'DONNEES_ENSIGHT' # repe elif df['type'] == 'repe': dest = 'REPE_IN' if dest is not None: # 3. --- copy kret = run.Copy(dest, df['path'], niverr='<E>_COPY_ERROR', verbose=True) # 4. --- decompression if kret == 0 and df['compr']: kret, dest = run.Gunzip(dest, niverr='<E>_DECOMPRESSION', verbose=True) # 5. --- move the bases in main directory if df['type'] in ('base', 'bhdf'): for f in glob(osp.join(dest, '*')): run.Rename(f, osp.basename(f)) # force the file to be writable make_writable(dest) # clean text files if necessary if df['ul'] != 0 and run.IsTextFileWithCR(dest): file_cleanCR(dest) print3(ufmt(' ` ' + _(u'line terminators have been removed from %s'), dest)) return icomm
def get_export(prof, args, print_output=True, **kwargs): """Call --get_export action on a server.""" # read server informations serv = build_server_from_profile(prof, TYPES.EXEC) vers = kwargs.get('vers', magic.run['aster_vers']) iret, output, err = call_generic_service("get_export", serv, prof, args, {'vers': vers}) if print_output: print3(output) return iret, output
def release(self): """Release current lock. """ if self.lock is None: raise CopyError, _('No lock to release') flock = self.lock_filename() if DEBUG: print3(u' RELEASED by', self.leech) if os.path.exists(flock): os.remove(flock) self.lock = None
def _get_list_src(self): """Retourne la liste des routines fortran et python. """ l_f = self._get_list_fort() l_pyt = self._get_list_python() if self.verbose: print3('%5d routines fortran dans %s + %s' % (len(l_f), self.fort, self.surch_fort)) print3('%5d modules python dans %s + %s' % (len(l_pyt), self.pyt, self.surch_pyt)) return l_f, l_pyt
def _get_list_fort(self): """Retourne la liste des fichiers sources fortran/fortran90.""" s_f = set() for dsrc in self.fort: s_f.update(glob(osp.join(dsrc, '*.f'))) s_f.update(glob(osp.join(dsrc, '*', '*.f'))) s_f.update(glob(osp.join(dsrc, '*.F'))) s_f.update(glob(osp.join(dsrc, '*', '*.F'))) s_f.update(glob(osp.join(dsrc, '*.F90'))) s_f.update(glob(osp.join(dsrc, '*', '*.F90'))) if self.build_obj.support('waf'): l_f = [osp.abspath(f) for f in s_f] l_f.sort() return l_f d_f = {} for f in s_f: assert d_f.get( osp.basename(f)) is None, 'ERROR : %s (old : %s)' % ( f, d_f[osp.basename(f)]) d_f[osp.basename(f)] = f # surcharge s_surch = set() for dsrc in self.surch_fort: s_surch.update(glob(osp.join(dsrc, '*.f'))) s_surch.update(glob(osp.join(dsrc, '*', '*.f'))) s_surch.update(glob(osp.join(dsrc, '*.F'))) s_surch.update(glob(osp.join(dsrc, '*', '*.F'))) if len(s_surch) > 0: l_surch = list(s_surch) l_surch.sort() print3('%5d sources en surcharge : ' % len(l_surch)) print3(os.linesep.join(l_surch)) # retirer des sources originaux ceux de la surcharge... s_suppr = set() for f in s_surch: fexist = d_f.get(osp.basename(f)) if fexist: s_suppr.add(fexist) if self.verbose: print3('suppression :', fexist) # ... et ceux de l'unigest if self.unig: iunig = 0 for f in self.unig['f'] + self.unig['f90']: iunig += 1 s_suppr.add(d_f.get(osp.basename(f), '')) if self.verbose: print3('suppression :', f) if iunig > 0: print3('%5d source(s) supprime(s).' % iunig) s_f.difference_update(s_suppr) s_f.update(s_surch) l_f = [osp.abspath(f) for f in s_f] l_f.sort() return l_f
def __init__(self, astest_dir, all=False, verbose=True): """Initialisations """ self.astest_dir = force_list(astest_dir) self.liste_ct = [] self.alltest = all self.filter = [] self.liste_crit = [] self.list_para_test = list_para_test self.verbose = verbose if self.verbose: print3(ufmt(_(u'Directory of testcases : %s'), self.astest_dir))
def VerbStart(self, cmd, verbose=None): """Start message in verbose mode """ Lm = self._LineLen if verbose == None: verbose = self.verbose if verbose: pcmd = cmd if len(cmd) > Lm - 2 or cmd.count(os.linesep) > 0: pcmd = pcmd + os.linesep + ' ' * Lm print3(('%-' + str(Lm) + 's') % (pcmd, ), end=' ') magic.get_stdout().flush()
def soumbtc_batch(self, fbtc): """Run btc script in batch mode. """ self.scheduler.set_subpara(fbtc, change_input_script=True) iret, jobid, queue = self.scheduler.start() if iret != 0: print3(_(u"Error during submitting job !")) iret = 5 if jobid == '': print3(_(u"Empty jobid")) iret = 5 return iret, jobid, queue
def available_host(self, **kwjob): """Return the most available host accepting job parameters. """ if _DBG_CRIT: print3('job parameters : ', end=' ') pprint(kwjob) values = self.suitable_host(**kwjob) if _DBG_CRIT: print '%d suitable hosts : %s' \ % (len(values), [info[self.d_crit['host'][0]] for info in values]) avail = self.get_first(values) return avail