def SendMail(self, dest, author=None, subject='no subject', text=''):
"""Send a message by mail.
Use ", " in `dest` to separate multiple recipients.
"""
sign = _(u'email sent at %s by as_run from %s') % (now(), local_host)
sign = os.linesep*2 + '-'*len(sign) + os.linesep \
+ sign \
+ os.linesep + '-'*len(sign) + os.linesep
try:
import smtplib
from asrun.common.utils import MIMETextClass
if MIMETextClass is None:
raise ImportError
except ImportError:
self._mess(_(u'Can not send mail from this machine'),
'<A>_NO_MAILER')
return
dest = [s.strip() for s in dest.split(',')]
mail_encoding = get_encoding()
content = convert(to_unicode(text) + sign, mail_encoding)
msg = MIMETextClass(content, _charset=mail_encoding)
msg['Subject'] = convert(subject, mail_encoding)
if author == None:
author = '%s@%s' % (local_user, local_full_host)
msg['From'] = author
msg['To'] = ', '.join(dest)
s = smtplib.SMTP()
s.connect()
s.sendmail(msg['From'], dest, msg.as_string())
s.close()
def Histor(run, *args):
"""Extract the content of some issues from REX database.
"""
if len(args) != 2:
run.parser.error(_(u"'--%s' requires two arguments") % run.current_action)
iret = 0
on_machref = run.get('rep_agla', 'local') != 'local'
if not on_machref:
run.Mess(_(u'Only available on the AGLA machine'), '<F>_AGLA_ERROR')
# 0. check imports
if not imports_succeed:
run.Mess(_(u'Imports of REX interface failed.'), '<F>_IMPORT_ERROR')
# 1. copy input file
jn = run['num_job']
ffich = get_tmpname(run, run['tmp_user'], basename='hist_input')
kret = run.Copy(ffich, args[0], niverr='<F>_COPYFILE')
# 2. read input file
hist_content = open(ffich, 'r').read()
expr = re.compile('([0-9]+)', re.MULTILINE)
l_nf = [int(n) for n in expr.findall(hist_content)]
# 3. open database connection
try:
c = db.CONNECT('REX', rcdir=confdir, verbose=run['debug'])
except Exception, msg:
run.Mess(convert(msg), '<F>_DB_ERROR')
def AddToEnv(self, profile):
"""Read 'profile' file (with sh/bash/ksh syntax) and add updated
variables to os.environ.
"""
if not os.path.isfile(profile):
self._mess(ufmt(_(u'file not found : %s'), profile),
'<A>_FILE_NOT_FOUND')
return
# read initial environment
iret, out = self.Shell('%s env' % shell_cmd)
self._dbg("env_init", out, all=True)
env_init = env2dict(out)
if iret != 0:
self._mess(_(u'error getting environment'), '<E>_ABNORMAL_ABORT')
return
# read profile and dump modified environment
iret, out = self.Shell('%s ". %s ; env"' % (shell_cmd, profile))
self._dbg("env_prof", out, all=True)
env_prof = env2dict(out)
if iret != 0:
self._mess(ufmt(_(u'error reading profile : %s'), profile),
'<E>_ABNORMAL_ABORT')
return
# "diff"
for k, v in env_prof.items():
if env_init.get(k, None) != v:
self._dbg('AddToEnv set : %s=%s' % (k, v))
os.environ[k] = convert(v)
for k in [k for k in env_init.keys() if env_prof.get(k) is None]:
self._dbg('unset %s ' % k, DBG=True)
try:
del os.environ[k]
except:
pass
def get_exec_command(self, cmd_in, add_tee=False, env=None):
"""Return command to run Code_Aster.
"""
run = magic.run
# add source of environments files
if env is not None and self.really():
envstr = [". %s" % f for f in env]
envstr.append(cmd_in)
cmd_in = " ; ".join(envstr)
dict_val = {
'cmd_in' : cmd_in,
'var' : 'EXECUTION_CODE_ASTER_EXIT_%s' % run['num_job'],
}
if add_tee:
cmd_in = """( %(cmd_in)s ; echo %(var)s=$? ) | tee fort.6""" % dict_val
if not self.really():
if add_tee:
cmd_in += """ ; exit `grep -a %(var)s fort.6 | head -1 """ \
"""| sed -e 's/%(var)s=//'`""" % dict_val
return cmd_in
mpi_script = osp.join(self.global_reptrav, 'mpi_script.sh')
if run.get('use_parallel_cp') in YES_VALUES:
cp_cmd = '%s --with-as_run %s %s' \
% (osp.join(aster_root, 'bin', 'parallel_cp'),
" ".join(run.get_remote_args()),
self.global_reptrav)
elif self.nbnode() > 1:
cp_cmd = 'scp -r'
else:
cp_cmd = 'cp -r'
dict_mpi_args = {
'cmd_to_run' : cmd_in,
'program' : mpi_script,
'cp_cmd' : cp_cmd,
}
dict_mpi_args.update(self.build_dict_mpi_args())
template = osp.join(datadir, 'mpirun_template')
if not run.Exists(template):
run.Mess(ufmt(_(u'file not found : %s'), template), '<F>_FILE_NOT_FOUND')
content = open(template, 'r').read() % dict_mpi_args
run.DBG(content, all=True)
open(mpi_script, 'w').write(convert(content))
os.chmod(mpi_script, 0755)
# add comment because cpu/system times are not counted by the timer
self._add_timer_comment()
# mpirun/mpiexec
command = dict_mpi_args['mpirun_cmd'] % dict_mpi_args
# need to initialize MPI session ?
if dict_mpi_args['mpi_ini']:
command = dict_mpi_args['mpi_ini'] % dict_mpi_args + " ; " + command
# need to close MPI session ?
if dict_mpi_args['mpi_end']:
command = command + " ; " + dict_mpi_args['mpi_end'] % dict_mpi_args
return command
def Tail(run, *args):
"""Output the last part of fort.6 file or filter lines matching a pattern.
"""
if len(args) < 5:
run.parser.error(_(u"'--%s' takes at least %d arguments (%d given)") % \
(run.current_action, 5, len(args)))
elif len(args) > 6:
run.parser.error(_(u"'--%s' takes at most %d arguments (%d given)") % \
(run.current_action, 6, len(args)))
# arguments
njob, nomjob, mode, fdest, nbline = args[:5]
expression = None
if len(args) > 5:
expression = args[5]
# allow to customize of the executed function
worker = Func_tail
if run.get('schema_tail_exec'):
worker = get_plugin(run['schema_tail_exec'])
run.DBG("calling plugin : %s" % run['schema_tail_exec'])
etat, diag, s_out = worker(run, njob, nomjob, mode, nbline, expression)
print_tail_result(nomjob, njob, etat, diag)
if s_out == "":
s_out = _(u'the output is empty (job ended ?)')
# exit if job isn't running
run.PrintExitCode = False
if run.get('result_to_output') or fdest == 'None':
run.DBG(_(u'tail put to output'))
print3(s_out)
else:
# send output file
if run.IsRemote(fdest):
ftmp = get_tmpname(run, run['tmp_user'], basename='tail')
open(ftmp, 'w').write(convert(s_out))
jret = run.Copy(fdest, ftmp)
else:
fdest = run.PathOnly(fdest)
open(fdest, 'w').write(convert(s_out))
run.DBG(ufmt(u'output written to : %s', fdest))
def AddComment(self, txt, init=False):
"""Add a comment to the result table.
"""
sign = sha1(convert(txt)).hexdigest()
if self._comment_seen.get(sign):
return
self._comment_seen[sign] = True
if init:
self.comment = ''
lines = self.comment.splitlines()
lines.append(txt)
self.comment = os.linesep.join([" %s" % line for line in lines])
def __convert_string(self, out_encoding, copy):
"""Convert all strings to `out_encoding`.
If 'copy' is True a copy of `.values` is returned,
if 'copy' is False `.values` is modified in place.
"""
if copy:
dico = self.values.copy()
else:
dico = self.values
dico = self.values
for k, v in dico.items():
if type(v) in (str, unicode):
dico[k] = convert(v, out_encoding)
if copy:
return dico
def exit(self, status=0, msg=None):
"""Call 'run.Sortie' method instead of 'sys.exit'."""
if msg:
magic.get_stderr().write(convert(msg))
self.run.PrintExitCode = False
self.run.Sortie(status)
class ITEM:
"""Enregistrement d'une table avec éventuellement des références à
des enregistrements d'autres tables.
(Je pense que hyperdb de Roundup pourrait être utilisée, mais je n'ai pas
trouvé comment...!).
Attributs :
table : nom de la table associée à ce type d'enregistrement
c : connexion MySQL
refs : dictionnaire indiquant la sous-classe des ITEMs référencés
idkey : vaut 'id' sauf pour les tables multi-link où il vaut 'linkid'
primkeys : liste des clés "primaires" (permettant de retrouver un item)
values : dictionnaire des champs
default : contenu d'item par défaut
inDB : True si on a vérifié que l'enregistrement est dans la base
lu : True si on a déjà lu l'enregistrement dans la base
cols : (cache) ordre des colonnes dans la table (pour SELECT * par ex)
link_class : table où sont stockés les "multi-link"
parent_class : la classe "parent"
A priori, tant que lu=False, on n'est pas assuré que les champs qui font
référence à un autre enregistrement contienne l'ITEM référencé, ils peuvent
contenir que l'id.
"""
table = None
link_class = None
parent_class = None
default = {}
refs = {}
primkeys = ['id']
idkey = 'id'
db_encoding = 'utf-8'
# this is used for Roundup schema where max ids are store in 'ids' table.
tab_idmax = 'ids'
def __init__(self, dini=None, c=None):
"""Initialisation avec dini qui est :
- soit le dictionnaire des valeurs des champs de l'enregistrement,
- soit la valeur du champ "primaire".
Sont équivalents (en supposant que 'id' est la clé primaire):
ITEM({'id' : num}, c) et ITEM(num, c)
"""
toread = False
if dini == None:
dini = {}
elif not type(dini) is dict:
dini = { self.idkey : dini }
toread = True
self.c = c
self.values = self.default.copy()
self.values.update(dini)
self.inDB = False
self.lu = False
self.cols = None
if toread:
self.read()
def __setitem__(self, key, value):
"""Positionne un champ de l'enregistrement
"""
self.values[key] = value
def __getitem__(self, key):
"""Retourne la valeur d'un champ de l'enregistrement
"""
return self.values[key]
def __setid(self, value):
"""Positionne le champ idkey de l'objet
"""
if not self.idkey in self.refs:
self.values[self.idkey] = value
elif __verbose__:
print3('Warning <setid>: %s is an ITEM object.' % repr(self.idkey))
def __getid(self):
"""Retourne la valeur du champ idkey de l'objet
"""
value = self.values[self.idkey]
if not self.idkey in self.refs or not isinstance(value, ITEM):
return value
else:
if __verbose__:
print3('Warning <getid>: %s is an ITEM object.' % repr(self.idkey))
return value.get(value.idkey)
def get(self, key, default=None):
"""Retourne la valeur d'un champ de l'enregistrement ou default
"""
return self.values.get(key, default)
def getrepr(self, refs=True):
"""Affichage. Si 'refs'=False, on n'imprime pas les sous-items.
"""
txt = []
for k, v in self.values.items():
if k in self.refs.keys():
item = None
if isinstance(self[k], ITEM):
item = v
idref = v.get(v.idkey)
else:
item = eval('%s(%s)' % (self.refs[k], 'c=self.c'))
try:
item.read({item.idkey:v})
idref = v
except (ConnectDBError, ReadDBError), msg:
item = None
if __verbose__:
print3(msg)
if item != None and (item.__class__.__name__ != self.__class__.__name__\
or idref != self.get(self.idkey)):
if not refs:
txt.append(fmt_vr % (k, idref))
else:
txt.append(fmt_id % (k, idref))
txt.append(os.linesep.join([' > %s' \
% l for l in repr(item).split(os.linesep)]))
txt.append(fmt_fin)
else:
txt.append(fmt_vr % (k, v))
else:
if type(v) == str:
v = to_unicode(v)
txt.append(fmt_val % (k, v))
return convert(os.linesep.join(txt))
def local_shell(self,
cmd,
bg=False,
verbose=False,
follow_output=False,
alt_comment=None,
interact=False,
separated_stderr=False,
stack_id=3,
**ignore_args):
"""Execute a command shell
cmd : command
bg : put command in background if True
verbose : print status messages during execution if True
follow_output : follow interactively output of command
alt_comment : print this "alternative comment" instead of "cmd"
Return :
iret : exit code if bg = False,
0 if bg = True
output : output lines (as string)
"""
if not alt_comment:
alt_comment = cmd
if len(cmd) > self.MaxCmdLen:
self._mess((_(u'length of command shell greater '\
'than %d characters.') % self.MaxCmdLen), '<A>_ALARM')
self._dbg('cmd :',
cmd,
'background : %s' % bg,
'follow_output : %s' % follow_output,
all=True,
stack_id=stack_id)
self.VerbStart(alt_comment, verbose=verbose)
if follow_output and verbose:
print3(os.linesep + _(u'Command output :'))
var_id = "EXIT_COMMAND_%s" % get_command_id()
fout_name = get_tmpname_base(self._tmpdir,
'local_shell_output',
user=local_user,
node=local_host,
pid="auto")
ferr_name = get_tmpname_base(self._tmpdir,
'local_shell_error',
user=local_user,
node=local_host,
pid="auto")
new_cmd = get_command_line(cmd, bg, follow_output, separated_stderr,
fout_name, ferr_name, var_id)
# execution
iret = os.system(convert(new_cmd))
output, error = "", ""
try:
output = to_unicode(open(fout_name, "r").read())
os.remove(fout_name)
except:
pass
try:
error = to_unicode(open(ferr_name, "r").read())
os.remove(ferr_name)
except:
pass
if follow_output:
# repeat header message
self.VerbStart(alt_comment, verbose=verbose)
mat = re.search('EXIT_CODE=([0-9]+)', output)
if mat:
iret = int(mat.group(1))
elif follow_output:
# os.system returns exit code of tee
mat = re.search("%s=([0-9]+)" % var_id, output)
if mat:
iret = int(mat.group(1))
self.VerbEnd(iret, output, verbose=verbose)
if iret != 0:
print('ERROR : iret = %s' % iret, '+++ STANDARD OUTPUT:', output,
'+++ STANDARD ERROR:')
if bg:
iret = 0
if not separated_stderr:
result = iret, output
else:
result = iret, output, error
return result
class AsterCalcul(BaseCalcul):
"""This class read user's profile and (if needed) call as_run through or not
a terminal, or just write a btc file...
"""
_supported_services = (
'study',
'parametric_study',
'testcase',
'meshtool',
'stanley',
'convbase',
'distribution',
'exectool',
'multiple',
)
def __init__(self,
run,
filename=None,
prof=None,
pid=None,
differ_init=False):
"""Initializations
"""
BaseCalcul.__init__(self)
assert filename or prof, 'none of (filename, prof) provided!'
self.run = run
if pid is None:
self.pid = self.run['num_job']
else:
self.pid = pid
self.studyid = self.pid
if prof is not None:
self.prof = prof
else:
# ----- profile filename
fprof = get_tmpname(self.run,
self.run['tmp_user'],
basename='profil_astk')
self.run.ToDelete(fprof)
kret = self.run.Copy(fprof, filename, niverr='<F>_PROFILE_COPY')
self.prof = AsterProfil(fprof, self.run)
if self.prof['nomjob'][0] == '':
self.prof['nomjob'] = 'unnamed'
# attributes
self.dict_info = None
self.as_exec_ref = self.run.get('as_exec_ref')
self.diag = '?'
self.__initialized = False
if not differ_init:
self.finalize_init()
def finalize_init(self):
"""Finalize initialization.
Allow to adapt prof object before customization."""
# decode service called
self.decode_special_service()
# add memory
self.add_memory()
# allow customization of calcul object
if self.run['schema_calcul']:
schem = get_plugin(self.run['schema_calcul'])
self.run.DBG("calling plugin : %s" % self.run['schema_calcul'])
self.prof = schem(self)
self.__initialized = True
def decode_special_service(self):
"""Return the profile modified for the "special" service.
"""
self.serv, self.prof = apply_special_service(self.prof, self.run)
if self.serv == '':
if self.prof['parent'][0] == 'parametric':
self.serv = 'parametric_study'
elif self.prof['parent'][0] == 'astout':
self.serv = 'testcase'
else:
self.serv = 'study'
self.prof['service'] = self.serv
self.run.DBG("service name : %s" % self.serv)
if self.serv not in self._supported_services:
self.error(_(u'Unknown service : %s') % self.serv)
def add_memory(self):
"""Add an amount of memory (MB) to the export parameters"""
if self.serv in ('parametric_study', ):
return
conf = build_config_from_export(self.run, self.prof)
self.run.DBG("memory to add: %s" % conf['ADDMEM'][0])
try:
addmem = float(conf['ADDMEM'][0])
except ValueError:
addmem = 0.
if not addmem:
return
memory = float(self.prof['memjob'][0] or 0.) / 1024. + addmem
self.prof.set_param_memory(memory)
self.run.DBG("new memory parameters: memjob=%s memjeveux=%s" % \
(self.prof['memjob'][0], self.prof.args['memjeveux']))
def build_dict_info(self, opts):
"""Build a dictionnary grouping all parameters.
"""
sep = "-----------------------------------------------"
self.mode = self.prof['mode'][0]
if not self.mode or self.run.get(self.mode) not in YES_VALUES:
self.mode = self.prof['mode'] = "interactif"
if self.mode == 'batch':
self.scheduler = BatchSystemFactory(self.run, self.prof)
node = self.prof['noeud'][0] or self.prof['serveur'][0]
self.dict_info = {
'sep': sep,
'export': self.prof.get_filename(),
'mcli': self.prof['mclient'][0],
'ucli': self.prof['uclient'][0],
'serv': self.prof['serveur'][0],
'user': self.prof['username'][0],
'mode': self.mode,
'node': node,
'plt': self.run['plate-forme'],
'vers': self.prof.get_version_path(),
'tpsjob': self.prof['tpsjob'][0],
'vmem': float(self.prof['memjob'][0] or 0.) / 1024.,
'ncpus': self.prof['ncpus'][0] or 'auto',
'mpi_nbnoeud': self.prof['mpi_nbnoeud'][0],
'mpi_nbcpu': self.prof['mpi_nbcpu'][0],
'dbg': self.prof['debug'][0],
'prof_content': self.prof.get_content(),
'nomjob': self.prof['nomjob'][0],
'nomjob_': self.flash('', ''),
'nomjob_p': self.flash('export', '$num_job'),
'as_run_cmd': " ".join(self.run.get_as_run_cmd(with_args=False)),
'who': self.run.system.getuser_host()[0],
'opts': opts,
'remote_args': " ".join(self.run.get_as_run_args()),
}
if self.prof['srv_dbg'][0] in YES_VALUES:
self.dict_info['opts'] += ' --debug'
if self.prof['srv_verb'][0] in YES_VALUES:
self.dict_info['opts'] += ' --verbose'
# rep_trav from profile or config(_nodename) / keep consistancy with job.py
rep_trav = self.prof['rep_trav'][0]
if rep_trav == '':
rep_trav = get_nodepara(node, 'rep_trav', self.run['rep_trav'])
self.dict_info['rep_trav'] = rep_trav
# set message using previous content
self.dict_info['message'] = self.message()
def message(self):
"""Format information message.
"""
# No "' in ASTK_MESSAGE !
ASTK_MESSAGE = []
# check client and server versions
serv_vers = self.run.__version__
try:
client_vers = self.prof['origine'][0].split()[1]
except Exception, msg:
self.run.DBG('Error : unexpected "origine" value :',
self.prof['origine'][0])
client_vers = ''
if client_vers == '':
ASTK_MESSAGE.append(msg_cli_version % serv_vers)
elif serv_vers != client_vers:
ASTK_MESSAGE.append(msg_pb_version % (serv_vers, client_vers))
ASTK_MESSAGE.append(msg_info % self.dict_info)
if self.prof['classe'][0]:
ASTK_MESSAGE.append(msg_classe % self.prof['classe'][0])
if self.prof['depart'][0]:
ASTK_MESSAGE.append(msg_depart % self.prof['depart'][0])
ASTK_MESSAGE.append(self.dict_info['sep'])
if self.prof['consbtc'][0] not in NO_VALUES:
msg = "generated"
else:
msg = "provided by user"
ASTK_MESSAGE.append(msg_consbtc % msg)
ASTK_MESSAGE.append(self.dict_info['sep'])
ASTK_MESSAGE.append(msg_vers % (serv_vers, client_vers))
return convert(os.linesep.join(ASTK_MESSAGE))
run.Mess(_(u'Code_Aster product not found in database !'), '<F>_DB_ERROR')
emis = STATUS({'_name' : 'emis'}, c)
try:
emis.read()
except ReadDBError:
run.Mess(_(u"Status 'emis' not found in database !"), '<F>_DB_ERROR')
# 4.2. get version item
d_vers = prod.GetLinks()
vers = d_vers.get(d['VERSION'])
if vers == None:
run.Mess(_(u"Version %s not found in database !") % d['VERSION'])
# 4.3. fill fields
date_now = now(datefmt=mysql_date_fmt, timefmt="")
txtmsg = convert(d['TEXTE'], db_encoding)
txtmsg = cut_long_lines(txtmsg, maxlen=100)
issue = ISSUE({
'_creator' : user,
'_produit' : prod,
'_status' : emis,
'_title' : convert(d['TITRE'], db_encoding),
'_type' : d_typ[d['TYPFIC']],
'_version' : vers,
'_fichetude' : fichetude,
}, c)
descr = MSG({'_author' : user,
'_creation' : date_now,
'_creator' : user,
'_date' : date_now,
'_summary' : txtmsg[:255], } ,c)
