def SendMail(self, dest, author=None, subject='no subject', text=''): """Send a message by mail. Use ", " in `dest` to separate multiple recipients. """ sign = _(u'email sent at %s by as_run from %s') % (now(), local_host) sign = os.linesep*2 + '-'*len(sign) + os.linesep \ + sign \ + os.linesep + '-'*len(sign) + os.linesep try: import smtplib from asrun.common.utils import MIMETextClass if MIMETextClass is None: raise ImportError except ImportError: self._mess(_(u'Can not send mail from this machine'), '<A>_NO_MAILER') return dest = [s.strip() for s in dest.split(',')] mail_encoding = get_encoding() content = convert(to_unicode(text) + sign, mail_encoding) msg = MIMETextClass(content, _charset=mail_encoding) msg['Subject'] = convert(subject, mail_encoding) if author == None: author = '%s@%s' % (local_user, local_full_host) msg['From'] = author msg['To'] = ', '.join(dest) s = smtplib.SMTP() s.connect() s.sendmail(msg['From'], dest, msg.as_string()) s.close()
def Histor(run, *args): """Extract the content of some issues from REX database. """ if len(args) != 2: run.parser.error(_(u"'--%s' requires two arguments") % run.current_action) iret = 0 on_machref = run.get('rep_agla', 'local') != 'local' if not on_machref: run.Mess(_(u'Only available on the AGLA machine'), '<F>_AGLA_ERROR') # 0. check imports if not imports_succeed: run.Mess(_(u'Imports of REX interface failed.'), '<F>_IMPORT_ERROR') # 1. copy input file jn = run['num_job'] ffich = get_tmpname(run, run['tmp_user'], basename='hist_input') kret = run.Copy(ffich, args[0], niverr='<F>_COPYFILE') # 2. read input file hist_content = open(ffich, 'r').read() expr = re.compile('([0-9]+)', re.MULTILINE) l_nf = [int(n) for n in expr.findall(hist_content)] # 3. open database connection try: c = db.CONNECT('REX', rcdir=confdir, verbose=run['debug']) except Exception, msg: run.Mess(convert(msg), '<F>_DB_ERROR')
def AddToEnv(self, profile): """Read 'profile' file (with sh/bash/ksh syntax) and add updated variables to os.environ. """ if not os.path.isfile(profile): self._mess(ufmt(_(u'file not found : %s'), profile), '<A>_FILE_NOT_FOUND') return # read initial environment iret, out = self.Shell('%s env' % shell_cmd) self._dbg("env_init", out, all=True) env_init = env2dict(out) if iret != 0: self._mess(_(u'error getting environment'), '<E>_ABNORMAL_ABORT') return # read profile and dump modified environment iret, out = self.Shell('%s ". %s ; env"' % (shell_cmd, profile)) self._dbg("env_prof", out, all=True) env_prof = env2dict(out) if iret != 0: self._mess(ufmt(_(u'error reading profile : %s'), profile), '<E>_ABNORMAL_ABORT') return # "diff" for k, v in env_prof.items(): if env_init.get(k, None) != v: self._dbg('AddToEnv set : %s=%s' % (k, v)) os.environ[k] = convert(v) for k in [k for k in env_init.keys() if env_prof.get(k) is None]: self._dbg('unset %s ' % k, DBG=True) try: del os.environ[k] except: pass
def get_exec_command(self, cmd_in, add_tee=False, env=None): """Return command to run Code_Aster. """ run = magic.run # add source of environments files if env is not None and self.really(): envstr = [". %s" % f for f in env] envstr.append(cmd_in) cmd_in = " ; ".join(envstr) dict_val = { 'cmd_in' : cmd_in, 'var' : 'EXECUTION_CODE_ASTER_EXIT_%s' % run['num_job'], } if add_tee: cmd_in = """( %(cmd_in)s ; echo %(var)s=$? ) | tee fort.6""" % dict_val if not self.really(): if add_tee: cmd_in += """ ; exit `grep -a %(var)s fort.6 | head -1 """ \ """| sed -e 's/%(var)s=//'`""" % dict_val return cmd_in mpi_script = osp.join(self.global_reptrav, 'mpi_script.sh') if run.get('use_parallel_cp') in YES_VALUES: cp_cmd = '%s --with-as_run %s %s' \ % (osp.join(aster_root, 'bin', 'parallel_cp'), " ".join(run.get_remote_args()), self.global_reptrav) elif self.nbnode() > 1: cp_cmd = 'scp -r' else: cp_cmd = 'cp -r' dict_mpi_args = { 'cmd_to_run' : cmd_in, 'program' : mpi_script, 'cp_cmd' : cp_cmd, } dict_mpi_args.update(self.build_dict_mpi_args()) template = osp.join(datadir, 'mpirun_template') if not run.Exists(template): run.Mess(ufmt(_(u'file not found : %s'), template), '<F>_FILE_NOT_FOUND') content = open(template, 'r').read() % dict_mpi_args run.DBG(content, all=True) open(mpi_script, 'w').write(convert(content)) os.chmod(mpi_script, 0755) # add comment because cpu/system times are not counted by the timer self._add_timer_comment() # mpirun/mpiexec command = dict_mpi_args['mpirun_cmd'] % dict_mpi_args # need to initialize MPI session ? if dict_mpi_args['mpi_ini']: command = dict_mpi_args['mpi_ini'] % dict_mpi_args + " ; " + command # need to close MPI session ? if dict_mpi_args['mpi_end']: command = command + " ; " + dict_mpi_args['mpi_end'] % dict_mpi_args return command
def Tail(run, *args): """Output the last part of fort.6 file or filter lines matching a pattern. """ if len(args) < 5: run.parser.error(_(u"'--%s' takes at least %d arguments (%d given)") % \ (run.current_action, 5, len(args))) elif len(args) > 6: run.parser.error(_(u"'--%s' takes at most %d arguments (%d given)") % \ (run.current_action, 6, len(args))) # arguments njob, nomjob, mode, fdest, nbline = args[:5] expression = None if len(args) > 5: expression = args[5] # allow to customize of the executed function worker = Func_tail if run.get('schema_tail_exec'): worker = get_plugin(run['schema_tail_exec']) run.DBG("calling plugin : %s" % run['schema_tail_exec']) etat, diag, s_out = worker(run, njob, nomjob, mode, nbline, expression) print_tail_result(nomjob, njob, etat, diag) if s_out == "": s_out = _(u'the output is empty (job ended ?)') # exit if job isn't running run.PrintExitCode = False if run.get('result_to_output') or fdest == 'None': run.DBG(_(u'tail put to output')) print3(s_out) else: # send output file if run.IsRemote(fdest): ftmp = get_tmpname(run, run['tmp_user'], basename='tail') open(ftmp, 'w').write(convert(s_out)) jret = run.Copy(fdest, ftmp) else: fdest = run.PathOnly(fdest) open(fdest, 'w').write(convert(s_out)) run.DBG(ufmt(u'output written to : %s', fdest))
def AddComment(self, txt, init=False): """Add a comment to the result table. """ sign = sha1(convert(txt)).hexdigest() if self._comment_seen.get(sign): return self._comment_seen[sign] = True if init: self.comment = '' lines = self.comment.splitlines() lines.append(txt) self.comment = os.linesep.join([" %s" % line for line in lines])
def __convert_string(self, out_encoding, copy): """Convert all strings to `out_encoding`. If 'copy' is True a copy of `.values` is returned, if 'copy' is False `.values` is modified in place. """ if copy: dico = self.values.copy() else: dico = self.values dico = self.values for k, v in dico.items(): if type(v) in (str, unicode): dico[k] = convert(v, out_encoding) if copy: return dico
def exit(self, status=0, msg=None): """Call 'run.Sortie' method instead of 'sys.exit'.""" if msg: magic.get_stderr().write(convert(msg)) self.run.PrintExitCode = False self.run.Sortie(status)
class ITEM: """Enregistrement d'une table avec éventuellement des références à des enregistrements d'autres tables. (Je pense que hyperdb de Roundup pourrait être utilisée, mais je n'ai pas trouvé comment...!). Attributs : table : nom de la table associée à ce type d'enregistrement c : connexion MySQL refs : dictionnaire indiquant la sous-classe des ITEMs référencés idkey : vaut 'id' sauf pour les tables multi-link où il vaut 'linkid' primkeys : liste des clés "primaires" (permettant de retrouver un item) values : dictionnaire des champs default : contenu d'item par défaut inDB : True si on a vérifié que l'enregistrement est dans la base lu : True si on a déjà lu l'enregistrement dans la base cols : (cache) ordre des colonnes dans la table (pour SELECT * par ex) link_class : table où sont stockés les "multi-link" parent_class : la classe "parent" A priori, tant que lu=False, on n'est pas assuré que les champs qui font référence à un autre enregistrement contienne l'ITEM référencé, ils peuvent contenir que l'id. """ table = None link_class = None parent_class = None default = {} refs = {} primkeys = ['id'] idkey = 'id' db_encoding = 'utf-8' # this is used for Roundup schema where max ids are store in 'ids' table. tab_idmax = 'ids' def __init__(self, dini=None, c=None): """Initialisation avec dini qui est : - soit le dictionnaire des valeurs des champs de l'enregistrement, - soit la valeur du champ "primaire". Sont équivalents (en supposant que 'id' est la clé primaire): ITEM({'id' : num}, c) et ITEM(num, c) """ toread = False if dini == None: dini = {} elif not type(dini) is dict: dini = { self.idkey : dini } toread = True self.c = c self.values = self.default.copy() self.values.update(dini) self.inDB = False self.lu = False self.cols = None if toread: self.read() def __setitem__(self, key, value): """Positionne un champ de l'enregistrement """ self.values[key] = value def __getitem__(self, key): """Retourne la valeur d'un champ de l'enregistrement """ return self.values[key] def __setid(self, value): """Positionne le champ idkey de l'objet """ if not self.idkey in self.refs: self.values[self.idkey] = value elif __verbose__: print3('Warning <setid>: %s is an ITEM object.' % repr(self.idkey)) def __getid(self): """Retourne la valeur du champ idkey de l'objet """ value = self.values[self.idkey] if not self.idkey in self.refs or not isinstance(value, ITEM): return value else: if __verbose__: print3('Warning <getid>: %s is an ITEM object.' % repr(self.idkey)) return value.get(value.idkey) def get(self, key, default=None): """Retourne la valeur d'un champ de l'enregistrement ou default """ return self.values.get(key, default) def getrepr(self, refs=True): """Affichage. Si 'refs'=False, on n'imprime pas les sous-items. """ txt = [] for k, v in self.values.items(): if k in self.refs.keys(): item = None if isinstance(self[k], ITEM): item = v idref = v.get(v.idkey) else: item = eval('%s(%s)' % (self.refs[k], 'c=self.c')) try: item.read({item.idkey:v}) idref = v except (ConnectDBError, ReadDBError), msg: item = None if __verbose__: print3(msg) if item != None and (item.__class__.__name__ != self.__class__.__name__\ or idref != self.get(self.idkey)): if not refs: txt.append(fmt_vr % (k, idref)) else: txt.append(fmt_id % (k, idref)) txt.append(os.linesep.join([' > %s' \ % l for l in repr(item).split(os.linesep)])) txt.append(fmt_fin) else: txt.append(fmt_vr % (k, v)) else: if type(v) == str: v = to_unicode(v) txt.append(fmt_val % (k, v)) return convert(os.linesep.join(txt))
def local_shell(self, cmd, bg=False, verbose=False, follow_output=False, alt_comment=None, interact=False, separated_stderr=False, stack_id=3, **ignore_args): """Execute a command shell cmd : command bg : put command in background if True verbose : print status messages during execution if True follow_output : follow interactively output of command alt_comment : print this "alternative comment" instead of "cmd" Return : iret : exit code if bg = False, 0 if bg = True output : output lines (as string) """ if not alt_comment: alt_comment = cmd if len(cmd) > self.MaxCmdLen: self._mess((_(u'length of command shell greater '\ 'than %d characters.') % self.MaxCmdLen), '<A>_ALARM') self._dbg('cmd :', cmd, 'background : %s' % bg, 'follow_output : %s' % follow_output, all=True, stack_id=stack_id) self.VerbStart(alt_comment, verbose=verbose) if follow_output and verbose: print3(os.linesep + _(u'Command output :')) var_id = "EXIT_COMMAND_%s" % get_command_id() fout_name = get_tmpname_base(self._tmpdir, 'local_shell_output', user=local_user, node=local_host, pid="auto") ferr_name = get_tmpname_base(self._tmpdir, 'local_shell_error', user=local_user, node=local_host, pid="auto") new_cmd = get_command_line(cmd, bg, follow_output, separated_stderr, fout_name, ferr_name, var_id) # execution iret = os.system(convert(new_cmd)) output, error = "", "" try: output = to_unicode(open(fout_name, "r").read()) os.remove(fout_name) except: pass try: error = to_unicode(open(ferr_name, "r").read()) os.remove(ferr_name) except: pass if follow_output: # repeat header message self.VerbStart(alt_comment, verbose=verbose) mat = re.search('EXIT_CODE=([0-9]+)', output) if mat: iret = int(mat.group(1)) elif follow_output: # os.system returns exit code of tee mat = re.search("%s=([0-9]+)" % var_id, output) if mat: iret = int(mat.group(1)) self.VerbEnd(iret, output, verbose=verbose) if iret != 0: print('ERROR : iret = %s' % iret, '+++ STANDARD OUTPUT:', output, '+++ STANDARD ERROR:') if bg: iret = 0 if not separated_stderr: result = iret, output else: result = iret, output, error return result
class AsterCalcul(BaseCalcul): """This class read user's profile and (if needed) call as_run through or not a terminal, or just write a btc file... """ _supported_services = ( 'study', 'parametric_study', 'testcase', 'meshtool', 'stanley', 'convbase', 'distribution', 'exectool', 'multiple', ) def __init__(self, run, filename=None, prof=None, pid=None, differ_init=False): """Initializations """ BaseCalcul.__init__(self) assert filename or prof, 'none of (filename, prof) provided!' self.run = run if pid is None: self.pid = self.run['num_job'] else: self.pid = pid self.studyid = self.pid if prof is not None: self.prof = prof else: # ----- profile filename fprof = get_tmpname(self.run, self.run['tmp_user'], basename='profil_astk') self.run.ToDelete(fprof) kret = self.run.Copy(fprof, filename, niverr='<F>_PROFILE_COPY') self.prof = AsterProfil(fprof, self.run) if self.prof['nomjob'][0] == '': self.prof['nomjob'] = 'unnamed' # attributes self.dict_info = None self.as_exec_ref = self.run.get('as_exec_ref') self.diag = '?' self.__initialized = False if not differ_init: self.finalize_init() def finalize_init(self): """Finalize initialization. Allow to adapt prof object before customization.""" # decode service called self.decode_special_service() # add memory self.add_memory() # allow customization of calcul object if self.run['schema_calcul']: schem = get_plugin(self.run['schema_calcul']) self.run.DBG("calling plugin : %s" % self.run['schema_calcul']) self.prof = schem(self) self.__initialized = True def decode_special_service(self): """Return the profile modified for the "special" service. """ self.serv, self.prof = apply_special_service(self.prof, self.run) if self.serv == '': if self.prof['parent'][0] == 'parametric': self.serv = 'parametric_study' elif self.prof['parent'][0] == 'astout': self.serv = 'testcase' else: self.serv = 'study' self.prof['service'] = self.serv self.run.DBG("service name : %s" % self.serv) if self.serv not in self._supported_services: self.error(_(u'Unknown service : %s') % self.serv) def add_memory(self): """Add an amount of memory (MB) to the export parameters""" if self.serv in ('parametric_study', ): return conf = build_config_from_export(self.run, self.prof) self.run.DBG("memory to add: %s" % conf['ADDMEM'][0]) try: addmem = float(conf['ADDMEM'][0]) except ValueError: addmem = 0. if not addmem: return memory = float(self.prof['memjob'][0] or 0.) / 1024. + addmem self.prof.set_param_memory(memory) self.run.DBG("new memory parameters: memjob=%s memjeveux=%s" % \ (self.prof['memjob'][0], self.prof.args['memjeveux'])) def build_dict_info(self, opts): """Build a dictionnary grouping all parameters. """ sep = "-----------------------------------------------" self.mode = self.prof['mode'][0] if not self.mode or self.run.get(self.mode) not in YES_VALUES: self.mode = self.prof['mode'] = "interactif" if self.mode == 'batch': self.scheduler = BatchSystemFactory(self.run, self.prof) node = self.prof['noeud'][0] or self.prof['serveur'][0] self.dict_info = { 'sep': sep, 'export': self.prof.get_filename(), 'mcli': self.prof['mclient'][0], 'ucli': self.prof['uclient'][0], 'serv': self.prof['serveur'][0], 'user': self.prof['username'][0], 'mode': self.mode, 'node': node, 'plt': self.run['plate-forme'], 'vers': self.prof.get_version_path(), 'tpsjob': self.prof['tpsjob'][0], 'vmem': float(self.prof['memjob'][0] or 0.) / 1024., 'ncpus': self.prof['ncpus'][0] or 'auto', 'mpi_nbnoeud': self.prof['mpi_nbnoeud'][0], 'mpi_nbcpu': self.prof['mpi_nbcpu'][0], 'dbg': self.prof['debug'][0], 'prof_content': self.prof.get_content(), 'nomjob': self.prof['nomjob'][0], 'nomjob_': self.flash('', ''), 'nomjob_p': self.flash('export', '$num_job'), 'as_run_cmd': " ".join(self.run.get_as_run_cmd(with_args=False)), 'who': self.run.system.getuser_host()[0], 'opts': opts, 'remote_args': " ".join(self.run.get_as_run_args()), } if self.prof['srv_dbg'][0] in YES_VALUES: self.dict_info['opts'] += ' --debug' if self.prof['srv_verb'][0] in YES_VALUES: self.dict_info['opts'] += ' --verbose' # rep_trav from profile or config(_nodename) / keep consistancy with job.py rep_trav = self.prof['rep_trav'][0] if rep_trav == '': rep_trav = get_nodepara(node, 'rep_trav', self.run['rep_trav']) self.dict_info['rep_trav'] = rep_trav # set message using previous content self.dict_info['message'] = self.message() def message(self): """Format information message. """ # No "' in ASTK_MESSAGE ! ASTK_MESSAGE = [] # check client and server versions serv_vers = self.run.__version__ try: client_vers = self.prof['origine'][0].split()[1] except Exception, msg: self.run.DBG('Error : unexpected "origine" value :', self.prof['origine'][0]) client_vers = '' if client_vers == '': ASTK_MESSAGE.append(msg_cli_version % serv_vers) elif serv_vers != client_vers: ASTK_MESSAGE.append(msg_pb_version % (serv_vers, client_vers)) ASTK_MESSAGE.append(msg_info % self.dict_info) if self.prof['classe'][0]: ASTK_MESSAGE.append(msg_classe % self.prof['classe'][0]) if self.prof['depart'][0]: ASTK_MESSAGE.append(msg_depart % self.prof['depart'][0]) ASTK_MESSAGE.append(self.dict_info['sep']) if self.prof['consbtc'][0] not in NO_VALUES: msg = "generated" else: msg = "provided by user" ASTK_MESSAGE.append(msg_consbtc % msg) ASTK_MESSAGE.append(self.dict_info['sep']) ASTK_MESSAGE.append(msg_vers % (serv_vers, client_vers)) return convert(os.linesep.join(ASTK_MESSAGE))
run.Mess(_(u'Code_Aster product not found in database !'), '<F>_DB_ERROR') emis = STATUS({'_name' : 'emis'}, c) try: emis.read() except ReadDBError: run.Mess(_(u"Status 'emis' not found in database !"), '<F>_DB_ERROR') # 4.2. get version item d_vers = prod.GetLinks() vers = d_vers.get(d['VERSION']) if vers == None: run.Mess(_(u"Version %s not found in database !") % d['VERSION']) # 4.3. fill fields date_now = now(datefmt=mysql_date_fmt, timefmt="") txtmsg = convert(d['TEXTE'], db_encoding) txtmsg = cut_long_lines(txtmsg, maxlen=100) issue = ISSUE({ '_creator' : user, '_produit' : prod, '_status' : emis, '_title' : convert(d['TITRE'], db_encoding), '_type' : d_typ[d['TYPFIC']], '_version' : vers, '_fichetude' : fichetude, }, c) descr = MSG({'_author' : user, '_creation' : date_now, '_creator' : user, '_date' : date_now, '_summary' : txtmsg[:255], } ,c)