def SetParser(run):
"""Configure the command-line parser, add options name to store to the list,
set actions informations.
run : AsterRun object which manages the execution
"""
acts_descr = {
'run': {
'method':
RunAster,
'syntax':
'[options] profile',
'help':
_(u'Execute the execution described by the profile (default action)'
)
},
'quick': {
'method':
StartNow,
'syntax':
'[options] file1 [file2 [...]] [--surch_pyt=...] [--surch_fort=...]',
'help':
_(u'Start quickly an interactive execution using the files given in arguments'
)
},
'test': {
'method': StartTestNow,
'syntax': '[options] testcase [results_directory]',
'help': _(u'Start an interactive execution of a testcase')
},
'make_shared': {
'method':
MakeShared,
'syntax':
'--output=FILE [src1 [...]] srcN',
'help':
_(u'Produce a shared library named FILE by compiling the '
'source files src1... srcN. Typically used to build a UMAT library.'
),
},
}
opts_descr = {
'copy_all_results': {
'args': ('--copy_all_results', ),
'kwargs': {
'action':
'store_true',
'default':
False,
'dest':
'copy_all_results',
'help':
_(u'copy all results in the current directory (for --quick action)'
)
}
},
'debugger': {
'args': ('--debugger', ),
'kwargs': {
'action': 'store_true',
'default': False,
'dest': 'debugger',
'help': _(u'run in the debugger (for --quick/--test action)')
}
},
'exectool': {
'args': ('--exectool', ),
'kwargs': {
'action':
'store',
'default':
False,
'dest':
'exectool',
'help':
_(u'run using the specified tool (for --quick/--test action)')
}
},
'run_params': {
'args': ('--run_params', ),
'kwargs': {
'action':
'append',
'default': [],
'dest':
'run_params',
'help':
_(u'list of parameters added in the export file '
'(for --quick/--test action). '
'Example: --run_params=actions=make_env will set "P actions make_env"'
' in the export file')
}
},
}
run.SetActions(actions_descr=acts_descr,
actions_order=['run', 'quick', 'test', 'make_shared'],
group_options=False,
options_descr=opts_descr)
dico['name'] = dico['name'].replace('/cygdrive/', '')
if dico['name'].find(':/') < 0:
dico['name'] = dico['name'].replace('/', ':/', 1)
os.makedirs(dico['name'])
except OSError, s:
# maybe simultaneously created by another process
if not os.path.isdir(dico['name']):
iret = 4
try:
os.chmod(dico['name'], chmod)
except OSError, s:
self._mess(ufmt(_(u'can not change permissions on %s'), rep),
'<A>_ALARM')
self.VerbEnd(iret, s, verbose)
if iret != 0:
self._mess(ufmt(_(u'can not create directory %s'), rep), niverr)
return iret
def Delete(self, rep, remove_dirs=True, verbose=None):
"""Delete a file or a directory tree (rm -rf ...).
Set 'remove_dirs' to False to be sure to delete 'rep' only if it's a file.
"""
if verbose == None:
verbose = self.verbose
iret = 0
dico = self.filename2dict(rep)
# preventing to delete first level directories (as /, /home, /usr...)
if dico['name'][0] == '/' and len(dico['name'][:-1].split(
os.sep)) <= 2:
self._mess(ufmt(_(u'deleting this directory seems too dangerous. ' \
'%s has not been removed'), dico['name']), '<A>_ALARM')
def Copy(self, dest, *src, **opts):
"""Copy files/directories (from 'src' list) to 'dest'
Remote names format :
[[user[:passwd]@]machine:]name
where name is an absolute or a relative path.
optional arguments : verbose, follow_output passed to local_shell,
niverr : default is set to <F>_COPY_ERROR, but may be '<A>' or less !
protocol : to use a different protocol just for this copy.
Note : if src[i] is a directory and dest doesn't exist
dest is created and the content of src[i] is copied into dest.
This is not a default behavior of rsync which would create
`basename src[i]` in dest even if dest doesn't exist.
"""
iret = 0
kargs = {}
# take values from opts if exist
kargs['verbose'] = opts.get('verbose', self.verbose)
kargs['follow_output'] = opts.get('follow_output', False)
kargs['separated_stderr'] = False
if opts.get('niverr'):
niverr = opts['niverr']
else:
niverr = '<F>_COPY_ERROR'
ddest = self.filename2dict(dest)
if self.IsRemote(dest) \
or (ddest['user'] != '' and ddest['user'] != local_user):
fdest = ddest['user'] + '@' + ddest['mach'] + ':' + ddest['name']
else:
fdest = ddest['name']
self._dbg('source list : %s' % (src, ),
'destination : %s' % fdest,
stack_id=2)
if len(src) < 1:
self._mess(_(u'no source file to copy'), '<A>_ALARM')
for f in src:
# here because we can change 'proto' if necessary
proto = opts.get('protocol', self.param['remote_copy_protocol'])
jret = 0
df = self.filename2dict(f)
if self.IsRemote(f):
fsrc = df['user'] + '@' + df['mach'] + ':' + df['name']
else:
fsrc = df['name']
df['user'] = df['mach'] = ''
cmd = ''
tail = '.../' + '/'.join(f.split('/')[-2:])
if not opts.has_key('alt_comment'):
kargs['alt_comment'] = ufmt(_(u"copying %s..."), tail)
else:
kargs['alt_comment'] = opts['alt_comment']
if df['mach'] == '' and ddest['mach'] == '':
cmd = command['copy'] % {"args": fsrc + ' ' + fdest}
else:
if proto == 'RSYNC' and df['mach'] != '' and ddest[
'mach'] != '':
proto = 'RCP'
self._mess(_(u"copying a remote file to another remote server " \
"isn't allowed through RSYNC, trying with RCP."))
if proto == 'RCP':
cmd = 'rcp -r ' + fsrc + ' ' + fdest
elif proto == 'SCP':
cmd = 'scp -rBCq -o StrictHostKeyChecking=no ' + fsrc + ' ' + fdest
elif proto == 'RSYNC':
if self.IsDir(f) and not self.Exists(dest):
self.MkDir(dest)
cmd = 'rsync -rz ' + os.path.join(fsrc,
'*') + ' ' + fdest
else:
cmd = 'rsync -rz ' + fsrc + ' ' + fdest
elif proto == 'HTTP':
str_user = ''
if not df['user'] in ('', 'anonymous'):
str_user = df['user'] + '@'
# dest must be local
if ddest['mach'] == '':
cmd = 'wget http://' + str_user + df['mach'] + df[
'name'] + ' -O ' + fdest
else:
cmd = ''
self._mess(ufmt(_(u'remote destination not allowed through %s' \
' : %s'), proto, fdest), niverr)
if cmd != '':
jret, out = self.local_shell(cmd, **kargs)
if jret != 0 and niverr != 'SILENT':
self._mess(ufmt(_(u'error during copying %s to %s'), f, fdest) \
+ os.linesep + ufmt(_(u'message : %s'), out), niverr)
else:
self._mess(_(u'unexpected error or unknown copy protocol : %s') \
% proto, niverr)
iret = max(jret, iret)
return iret
def GetMessageInfo(run, *args):
"""Return info about Code_Aster messages.
"""
if not run.get('aster_vers'):
run.parser.error(
_(u"You must define 'default_vers' in 'aster' configuration file or use '--vers' option."
))
REPREF = run.get_version_path(run['aster_vers'])
fconf = run.get('config')
if fconf:
fconf = osp.abspath(fconf)
conf = build_config_of_version(run, run['aster_vers'], fconf)
if run['nolocal']:
run.Mess(
_(u'This operation only works on local source files. "--nolocal" option ignored'
))
bibfor = [
conf['SRCFOR'][0],
]
if conf['SRCF90'][0] != '':
bibfor.append(conf['SRCF90'][0])
args = list(args)
named_actions = (
'check',
'move',
)
action = None
if len(args) > 0 and args[0] in named_actions:
action = args.pop(0)
elif len(args) == 0:
run.Mess(_(u'You must choose an operation from %s or give a subroutine name or a message ID') \
% repr(named_actions), '<F>_INVALID_ARGUMENT')
# surcharge
surch_fort = run.get('surch_fort', [])
if surch_fort:
surch_fort = surch_fort.split(',')
surch_pyt = run.get('surch_pyt', [])
if surch_pyt:
surch_pyt = surch_pyt.split(',')
pick_cache = 'messages_dicts.%s.pick' % (run['aster_vers'].replace(
os.sep, '_'))
msgman = MESSAGE_MANAGER(
repref=REPREF,
fort=bibfor,
pyt=conf['SRCPY'][0],
capy=conf['SRCCAPY'][0],
cache_dict=osp.join(run['cache_dir'], pick_cache),
force=run['force'],
verbose=run['verbose'],
debug=run['debug'],
surch_fort=surch_fort,
surch_pyt=surch_pyt,
unig=run.get('unigest', None),
)
msgman.read_cata()
if action == 'check':
try:
unused, not_found = msgman.check_cata()
except CataMessageError, msg:
run.Sortie(4)
if not run['silent']:
print3('Messages inutilises :')
print3(' '.join(unused))
print3('Messages inexistants :')
print3(' '.join(not_found))
if len(unused) + len(not_found) > 0:
run.Sortie(4)
def start(self, options=''):
"""Go !
"""
self.is_starting()
self.run.DBG('Profile to run', self.prof.get_content(), all=True)
# ----- copy and read .export, build dict for formatting
self.build_dict_info(options)
if self.run.get('log_usage_version') and self.serv != 'testcase':
from asrun.contrib.log_usage import log_usage_version_unfail
log_usage_version_unfail(self.run['log_usage_version'], self.prof)
jn = self.pid
self.name = self.prof['nomjob'][0]
if self.run.get(self.mode) not in YES_VALUES:
print3(
ufmt(
_(u"the configuration file (%s) does not allow mode='%s'"),
osp.join(confdir, "asrun"), self.mode))
return 4, ""
# export file is not necessary in interactive mode
if self.mode == 'batch':
self.prof.WriteExportTo(self.prof.get_filename())
self.run.DBG('profile written into :', self.prof.get_filename())
# ----- consbtc ?
fbtc = osp.join(self.run['flasheur'], 'btc.%s' % jn)
if self.prof['consbtc'][
0] in NO_VALUES and not 'make_env' in self.prof['actions']:
fbtc0 = self.prof.Get('D', 'btc')[0]['path']
iret = self.run.Copy(fbtc, fbtc0)
self.change_btc(fbtc)
else:
self.consbtc(fbtc)
# ----- soumbtc ?
iret = 0
if self.prof['soumbtc'][0] not in NO_VALUES:
if self.mode == 'interactif':
iret, jobid, self.queue = self.soumbtc_interactif(fbtc)
else:
iret, jobid, self.queue = self.soumbtc_batch(fbtc)
if iret == 0:
self.jobid = jobid
# copy fbtc into flasheur/ (already removed if run in foreground)
if osp.exists(fbtc):
jret = self.run.Copy(self.flash('script'),
fbtc,
niverr='<A>_ALARM')
# faut-il recopier le btc vers le client
res_fbtc = self.prof.Get('R', 'btc')
if len(res_fbtc) > 0:
res_fbtc = res_fbtc[0]['path']
self.run.Copy(res_fbtc, fbtc)
print3("BTCFILE=%s" % res_fbtc)
return iret, ''
class MESSAGE_MANAGER(object):
"""Classe pour récupérer des informations sur le catalogue de messages.
"""
def __init__(self,
repref,
fort=('bibfor', 'bibf90'),
pyt='bibpyt',
capy='catapy',
cache_dict=None,
verbose=True,
force=False,
ignore_comments=True,
debug=False,
surch_fort=[],
surch_pyt=[],
unig=None):
"""Initialisations."""
if type(fort) not in (list, tuple):
fort = [
fort,
]
if type(surch_fort) not in (list, tuple):
surch_fort = [
surch_fort,
]
if type(surch_pyt) not in (list, tuple):
surch_pyt = [
surch_pyt,
]
self.repref = repref
self.fort = [osp.join(repref, rep) for rep in fort]
self.pyt = osp.join(repref, pyt)
self.capy = osp.join(repref, capy)
self.surch_fort = [osp.abspath(d) for d in surch_fort]
self.surch_pyt = [osp.abspath(d) for d in surch_pyt]
self.verbose = verbose
self.debug = debug
self.force = force
self.message_dir = 'Messages'
self.cache_dict = cache_dict or osp.join('/tmp/',
'messages_dicts.pick')
self.ignore_comments = ignore_comments
self.wrk_fort = osp.join(os.getcwdu(), 'F')
self.wrk_pyt = osp.join(os.getcwdu(), 'PY')
self.unig = unig
# init AsterConfig, AsterBuild objects
fconf = osp.join(self.repref, 'config.txt')
self.conf_obj = AsterConfig(fconf, run=None)
self.build_obj = AsterBuild(run=None, conf=self.conf_obj)
if unig:
# use with_fordepla=False not to mark moved files as removed
self.unig = unigest2dict(unig, self.conf_obj, with_fordepla=True)
if self.verbose:
print3('Repertoires :\n REF=%s\n FORTRAN=%s\n PYTHON=%s\n CAPY=%s' \
% (self.repref, self.fort, self.pyt, self.capy))
self.l_cata = self._get_list_cata()
self.l_src, self.l_pyt = self._get_list_src()
# self._build_regexp()
def read_cata(self):
"""Read all catalogues"""
self._build_dict()
self.print_stats()
def _filename_cata(self, cata):
"""Retourne le nom du fichier associé à un catalogue."""
name = osp.join(self.pyt, self.message_dir, cata + '.py')
return name
def _id2filename(self, msgid):
"""Retourne modelisa5 à partir de MODELISA5_46"""
mat = re.search('([A-Z]+[0-9]*)_([0-9]+)', msgid)
if mat == None:
raise MessageError(msgid, _(u'invalid message id'))
cata, num = mat.groups()
return self._filename_cata(cata.lower()), int(num)
def _filename2id(self, fcata, num):
"""Retourne MODELISA5_46 à partir de modelisa5"""
return '%s_%d' % (osp.splitext(osp.basename(fcata))[0].upper(), num)
def makedirs(self):
"""Crée les répertoires pour les fichiers modifiés
"""
for d in (self.wrk_fort, self.wrk_pyt):
if not osp.isdir(d):
#peter.zhang, for cygwin
d = d.replace('\\', '/')
d = d.replace('/cygdrive/', '')
if d.find(':/') < 0:
d = d.replace('/', ':/', 1)
os.makedirs(d)
def _get_list_cata(self):
"""Retourne la liste des catalogues de messages.
"""
re_mess = re.compile('#@.*Messages')
s_cata = set(glob(osp.join(self.pyt, self.message_dir, '*.py')))
s_suppr = set()
l_surch = []
# ajouter la surcharge et retirer le catalogue d'origine
for dsrc in self.surch_pyt:
for f in glob(osp.join(dsrc, '*.py')) + glob(
osp.join(dsrc, '*', '*.py')):
txt = open(f, 'r').read()
if re_mess.search(txt):
l_surch.append(osp.abspath(f))
fsup = osp.join(self.pyt, self.message_dir,
osp.basename(f))
if osp.exists(fsup):
s_suppr.add(fsup)
s_cata.update(l_surch)
if len(l_surch) > 0:
print3('%5d catalogues en surcharge : ' % len(l_surch))
print3(os.linesep.join(l_surch))
s_cata = set([osp.abspath(f) for f in s_cata \
if not osp.basename(f) in ['__init__.py', 'context_info.py', 'fermetur.py']])
l_suppr = []
if self.unig and len(self.unig['py']) > 0:
l_suppr = [osp.abspath(osp.join(self.repref, f)) \
for f in self.unig['py'] if f.find(self.message_dir) > -1]
s_suppr.update(l_suppr)
l_suppr = list(s_suppr)
l_suppr.sort()
if len(l_suppr) > 0:
print3('%5d catalogue(s) supprime(s) :' % len(l_suppr))
if self.verbose: print3(os.linesep.join(l_suppr))
l_cata = list(s_cata.difference(s_suppr))
l_cata.sort()
return l_cata
def check_cata(self):
"""Vérifie les catalogues.
"""
if self.verbose:
print3('%5d catalogues dans %s + %s' \
% (len(self.l_cata), self.pyt, self.surch_pyt))
all_msg = []
error_occurs = False
for f in self.l_cata:
try:
cata = CATA(f)
cata.check()
all_msg.extend([self._filename2id(f, i) for i in cata])
if self.verbose:
print3(cata)
except CataMessageError, msg:
error_occurs = True
print3(msg)
self.l_cata.remove(f)
if error_occurs:
raise CataMessageError(_(u"global"), _(u"errors occurred!"))
all_msg = set(all_msg)
# messages jamais appelés
used = set(self.d_msg_call.keys())
unused = list(all_msg.difference(used))
unused.sort()
union = used.union(all_msg)
not_found = list(used.difference(all_msg))
not_found.sort()
if self.verbose:
print3('%6d messages dans les catalogues' % len(all_msg))
print3('dont %6d messages appeles presents dans les catalogues' %
(len(used) - len(not_found)))
print3(' et %6d messages inutilises' % len(unused))
print3(' + %6d messages appeles absents des catalogues' %
len(not_found))
#print '%6d messages total (union pour verif)' % len(union)
return unused, not_found
def __delitem__(self, key):
"""Supprime le message d'indice 'key'.
"""
if self[key] == None:
raise MessageError(key, _(u'this message does not exist !'))
del self.cata_msg[key]
from asrun.common.i18n import _
from asrun.mystring import print3, ufmt, add_to_tail, indent
from asrun.thread import Dispatcher
from asrun.repart import ResourceManager
from asrun.common.utils import now, version2tuple, YES_VALUES
from asrun.common.sysutils import local_host, same_hosts, short_hostname
from asrun.distrib import DistributionTask
from asrun.client import (
ClientConfig,
AsterCalcHdlrMulti,
SERVER_CONF,
MULTIDIR,
)
fmt_head = '%s %s %s %s %s %s' % (
_(u"job").center(12), _(u"result").center(18), _(u"cpu").rjust(8),
_(u"sys").rjust(8), _(u"cpu+sys").rjust(8), _(u"elapsed").rjust(8))
fmt_resu = '%-12s %-18s %8.2f %8.2f %8.2f %8.2f'
fmt_res2 = '%%4d %s %%4d %s %%8.2f %%8.2f %%8.2f %%8.2f' \
% (_("jobs").ljust(7), _("errors").ljust(10))
fmt_tot = '-' * 12 + ' ' + '-' * 18 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8
class DistribMultipleTask(DistributionTask):
"""Manage several Code_Aster executions.
items are couples (username, hostname)
attributes (init during instanciation) :
IN :
run : AsterRun object
hostrc : ResourceManager object
prof : AsterProfil object
def Multiple(run, prof, runner, numthread='auto'):
"""Run a multiple execution.
"""
run.print_timer = True
# 1. ----- initializations
jn = run['num_job']
# 1.2. rep_trav from profile or from run[...]
reptrav = runner.reptrav()
run.Mess(_(u'Code_Aster multiple execution'), 'TITLE')
runner.set_cpuinfo(1, 1)
# ----- how many threads ?
try:
numthread = int(numthread)
except (TypeError, ValueError):
numthread = run.GetCpuInfo('numthread')
# 1.3. content of the profile
serv_list = prof['multiple_server_list'][0]
if not serv_list:
run.Mess(
_(u'List of servers ("multiple_server_list" parameter) not found'),
'<F>_ERROR')
serv_list = serv_list.split()
nbval = len(serv_list)
# tell if results have to be transfered on the client server
# or let on each host.
result_on_client = prof['multiple_result_on_client'][0] in YES_VALUES
# this hostrc object is only used to check the hosts availability
tit = _(u"Checking hosts")
run.timer.Start(tit)
client = ClientConfig(run.rcdir)
client.init_server_config()
avail_servers = client.get_server_list()
dhost = {}
couples = []
for sname in serv_list:
found = False
for label in avail_servers:
cfg = client.get_server_config(label)
if same_hosts(sname, cfg['nom_complet']):
found = True
client.refresh_server_config([label])
cfg = client.get_server_config(label)
if version2tuple(cfg['asrun_vers']) >= (1, 9, 2):
couples.append((short_hostname(sname), cfg))
dhost[sname] = {'user': cfg['login']}
else:
run.Mess(ufmt(_(u"Version 1.9.2 or newer is required to run " \
"multiple executions. It is %s on %s (installed in %s)."),
cfg['asrun_vers'], label, cfg['rep_serv']), '<E>_ERROR')
break
if not found:
run.Mess(
ufmt(_(u"Host '%s' is not available in %s."), sname,
client.rcfile(SERVER_CONF)), "<E>_ERROR")
run.DBG("couples :", couples, all=True)
hostrc = ResourceManager(dhost)
n_avail, n_tot = hostrc.CheckHosts(run, numthread=numthread)
run.timer.Stop(tit)
run.Mess(
_(u'Number of available hosts : %d/%d') % (n_avail, n_tot), "SILENT")
if n_avail < 1:
run.Mess(_(u"No available host. Run cancelled."), "<F>_ERROR")
if n_avail < len(serv_list):
run.Mess(_(u"All the hosts are not available. Run cancelled."),
"<F>_ERROR")
# define a hostrc object to allow a lot of simultaneous executions
hostinfo = {local_host: {'mem': 999999, 'cpu': 999999}}
hostrc = ResourceManager(hostinfo)
#XXX tpsjob : max elapsed time ?
try:
timeout = prof.get_timeout() * 2.
except Exception, reason:
run.Mess(
_(u"incorrect value for tpsjob : %s") % reason,
'<F>_INVALID_PARAMETER')
def TestList(run, *args):
"""Build a list of testcases.
"""
if not run.get('aster_vers'):
run.parser.error(
_(u"You must define 'default_vers' in 'aster' configuration file or use '--vers' option."
))
REPREF = osp.join(aster_root, run['aster_vers'])
fconf = run.get('config')
if fconf:
fconf = osp.abspath(fconf)
conf = build_config_of_version(run, run['aster_vers'], fconf)
l_ct = list(args)
if len(l_ct) < 1 and not (run['all_test'] or run.get('test_list')):
run.parser.error(
_(u"'--%s' : you must give testcase names or use one of --all/--test_list option") \
% run.current_action)
if run.get('test_list'):
iret, l_lue = get_list(run['test_list'])
if iret == 0:
l_ct.extend(l_lue)
else:
run.Mess(
ufmt(_(u'error during reading file : %s'), run['test_list']),
'<F>_ERROR')
l_dirs = run.get('astest_dir', ','.join(conf['SRCTEST']))
l_dirs = [
osp.abspath(osp.join(REPREF, p.strip())) for p in l_dirs.split(',')
]
run.PrintExitCode = False
if run['debug']:
run.DBG('astest_dir', l_dirs)
run.DBG('filter', run.get('filter'))
run.DBG('command', run.get('command'))
run.DBG('search', run.get('search'))
filtre = LISTE_CT(astest_dir=l_dirs,
all=run['all_test'],
verbose=not run['silent'])
filtre.add_ct(l_ct)
res = []
# add filters on parameters
if run.get('filter'):
for expr in run['filter']:
filtre.add_filter('para', expr)
if run['debug']:
res.append('%% filter on parameter : %s' % expr)
# add filters based on command used
for expr in run.get('command', []):
filtre.add_filter('command', expr)
if run['debug']:
res.append('%% filter on command : %s' % expr)
# add filters based on regular expression
for expr in run.get('search', []):
filtre.add_filter('regexp', expr)
if run['debug']:
res.append('%% filter on regexp : %s' % expr)
# add user filters
if run.get('user_filter'):
for f in run['user_filter']:
filtre.add_filter('user', f)
if run['debug']:
res.append('%% user filter, file : %s' % f)
res.extend(filtre.build_list())
result = os.linesep.join(res)
if run.get('output'):
open(run['output'], 'w').write(result)
print3(
ufmt(_(u'The results have been written into the file : %s'),
run['output']))
else:
print3(result)
def __init__(self, s_regexp, flags=re.MULTILINE | re.DOTALL):
"""s_regexp : chaine de l'expression qui doit être vérifiée.
"""
print3(ufmt(_(u" - regular expression : %s"), repr(s_regexp)))
self.regexp = re.compile(s_regexp, flags)
def RunAstout(run, conf, prof, runner, numthread='auto', **kargs):
"""Run a list of test cases...
"""
run.print_timer = True
# 1. ----- initializations
REPREF = prof.get_version_path()
# 1.2. rep_trav from profile or from run[...]
reptrav = runner.reptrav()
run.Mess(_(u'Code_Aster tests execution'), 'TITLE')
runner.set_cpuinfo(1, 1)
# ----- how many threads ?
try:
numthread = int(numthread)
except (TypeError, ValueError):
numthread = run.GetCpuInfo('numthread')
# 1.3. content of the profile
ltest = osp.join(reptrav, 'tests.list')
if not prof.Get('D', typ='list'):
run.Mess(_(u'no list of tests found'), '<E>_NO_TEST_LIST')
else:
for dli in prof.Get('D', typ='list'):
if run.IsRemote(dli['path']):
tmplist = get_tmpname(run, run['tmp_user'], basename='list')
run.ToDelete(tmplist)
kret = run.Copy(tmplist, dli['path'])
run.FileCat(tmplist, ltest)
else:
tmplist = run.PathOnly(dli['path'])
run.FileCat(tmplist, ltest)
if not prof.Get('D', typ='rep_test'):
reptest = []
else:
reptest = [r['path'] for r in prof.Get('D', typ='rep_test')]
if not prof.Get('R', typ='resu_test'):
run.Mess(_(u'no result directory found'), '<E>_NO_RESU_DIR')
else:
resutest = prof.Get('R', typ='resu_test')[0]['path']
if run.IsRemote(resutest):
run.Mess(_(u'the result directory must not be on a remote host'),
'<F>_ERROR')
resutest = run.PathOnly(resutest)
#peter.zhang, for cygwin
resutest = resutest.replace('\\', '/')
resutest = resutest.replace('/cygdrive/', '')
if resutest.find(':/') < 0:
resutest = resutest.replace('/', ':/', 1)
run.MkDir(resutest)
run.Delete(osp.join(resutest, 'RESULTAT'))
run.Delete(osp.join(resutest, 'NOOK'))
flashdir = osp.join(resutest, 'flash')
prfl = prof.Get('R', typ='flash')
if prfl:
if prfl[0]['path'] == "None":
flashdir = None
else:
flashdir = prfl[0]['path']
facmtps = 1.
nbmaxnook = 5
cpresok = 'RESNOOK'
try:
if prof['facmtps'][0] != '':
facmtps = float(prof['facmtps'][0])
except ValueError:
run.Mess(
_(u'incorrect value for %s : %s') %
('facmtps', prof['facmtps'][0]))
try:
if prof['nbmaxnook'][0] != '':
nbmaxnook = int(prof['nbmaxnook'][0])
except ValueError:
run.Mess(
_(u'incorrect value for %s : %s') %
('nbmaxnook', prof['nbmaxnook'][0]))
if prof['cpresok'][0] != '':
cpresok = prof['cpresok'][0]
run.CheckOK()
# get the list of the tests to execute
iret, list_tests = get_list(ltest, unique=True)
if iret != 0:
run.Mess(_(u'error during reading file : %s') % ltest, '<F>_ERROR')
nbtest = len(list_tests)
# should we run only nook tests ?
if run['only_nook'] or prof['only_nook'][0] in YES_VALUES:
l_dirs = reptest + [resutest]
old_run_result = getDiagnostic(run, l_dirs, list_tests)
run.DBG('getDiagnostic result', old_run_result, all=True)
nbini = len(list_tests)
list_tests = []
for test, dres in old_run_result.items():
if test.startswith('__'):
continue
if run.GetGrav(dres['diag']) >= run.GetGrav('NOOK'):
list_tests.append(test)
nbtest = len(list_tests)
print3(
_(u"""
--- %d test-cases to run initially
%d test-cases previously failed (or their results have not been found) and will be run again
""") % (nbini, nbtest))
random.shuffle(list_tests)
# suivi des calculs partagé entre 'numthread' threads
tit = _(u"Checking hosts")
hostrc = get_hostrc(run, prof)
run.timer.Start(tit)
n_avail, n_tot = hostrc.CheckHosts(run, numthread=numthread)
run.timer.Stop(tit)
run.Mess(
_(u'Number of available hosts : %d/%d') % (n_avail, n_tot), "SILENT")
if n_avail < 1:
run.Mess(_(u"No available host. Run cancelled."),
"<F>_INVALID_PARAMETER")
# timeout before rejected a job = tpsjob
try:
timeout = prof.get_timeout()
except Exception, reason:
run.Mess(
_(u"incorrect value for tpsjob : %s") % reason,
'<F>_INVALID_PARAMETER')
import os
import os.path as osp
import random
import cPickle
from asrun.common.i18n import _
from asrun.mystring import print3, ufmt
from asrun.thread import Dispatcher
from asrun.distrib import DistribTestTask
from asrun.repart import get_hostrc
from asrun.common_func import get_tmpname
from asrun.maintenance import get_aster_version, repr_vers, getDiagnostic
from asrun.common.utils import get_list, now, YES_VALUES
fmt_head = '%s %s %s %s %s %s' % (
_(u"testcase").center(12), _(u"result").center(18), _(u"cpu").rjust(8),
_(u"sys").rjust(8), _(u"cpu+sys").rjust(8), _(u"elapsed").rjust(8))
fmt_resu = '%-12s %-18s %8.2f %8.2f %8.2f %8.2f'
fmt_res2 = '%%4d %s %%4d %s %%8.2f %%8.2f %%8.2f %%8.2f' \
% (_("tests").ljust(7), _("errors").ljust(10))
fmt_tot = '-' * 12 + ' ' + '-' * 18 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8
def RunAstout(run, conf, prof, runner, numthread='auto', **kargs):
"""Run a list of test cases...
"""
run.print_timer = True
# 1. ----- initializations
REPREF = prof.get_version_path()
def StartNow(run, *args):
"""Start quickly a simple execution using files in arguments.
"""
if not run["silent"]:
run.Mess(
_(u"This functionnality is still in a beta state of development "
u"and may be removed a future release, or may never be improved. "
u"Please use --silent option to ignore this message the next time."
))
# ----- check argument
if not run.get('aster_vers'):
run.parser.error(
_(u"You must define 'default_vers' in 'aster' configuration file or use '--vers' option."
))
if len(args) < 1:
run.parser.error(
_(u"'--%s' requires at least one argument") % run.current_action)
# build export
lf = [osp.abspath(f) for f in args]
prof = build_export_from_files(run, lf, with_results=True)
prof = use_options(run, prof)
# development files
surch_pyt = run.get('surch_pyt', [])
if surch_pyt:
for obj in surch_pyt.split(','):
prof.Set(
'D', {
'path': osp.abspath(obj),
'type': 'py',
'ul': 0,
'isrep': osp.isdir(osp.abspath(obj)),
'compr': False,
})
exetmp = None
surch_fort = run.get('surch_fort', [])
if surch_fort:
for obj in surch_fort.split(','):
prof.Set(
'D', {
'path': osp.abspath(obj),
'type': 'f',
'ul': 0,
'isrep': osp.isdir(osp.abspath(obj)),
'compr': False,
})
exetmp = get_tmpname(run, run['tmp_user'], basename='executable')
prof.Set(
'DR', {
'path': exetmp,
'type': 'exec',
'ul': 0,
'isrep': False,
'compr': False,
})
if exetmp is not None:
prof['actions'] = prof['actions'] + ['make_exec']
# try to grab all results files
if run["copy_all_results"]:
resudir = os.getcwdu()
lcomm = prof.get_type('comm')
if len(lcomm) > 0:
jobname = osp.splitext(osp.basename(lcomm[0].path))[0]
else:
jobname = 'unamed'
add_all_results(prof, resudir, jobname)
run.Mess(_(u"Profile used :"), 'TITLE')
run.Mess(os.linesep + prof.get_content(), 'SILENT')
# execution
RunAster(run, prof)
def __str__(self):
return ufmt(_(u"%s: %s, error: %s"), self.__class__.__name__,
self.fcata, self.reason)
def _mess_timeout(self, dt, timeout, job, refused):
"""Emit a message when submission timeout occurs."""
self.run.Mess(ufmt(_(u"no submission for last %.0f s " \
"(timeout %.0f s, equal to 2 * the time requested by the main job), " \
"job '%s' cancelled after %d attempts."), dt, timeout, job, refused))
def move(self, oldid, dest, reuse_hole=True):
"""Déplace le message 'oldid' dans le catalogue 'dest'.
"""
if self.verbose:
print3('--- moving "%s" into "%s"' % (oldid, dest))
# catalogue objects
old_f, old_num = self._id2filename(oldid)
new_f = self._filename_cata(dest.lower())
# have these catalogues been already changed ?
fmod = osp.join(self.wrk_pyt, osp.basename(old_f))
if osp.isfile(fmod):
print3('from %s' % fmod)
old_f = fmod
fmod = osp.join(self.wrk_pyt, osp.basename(new_f))
if osp.isfile(fmod):
print3('from %s' % fmod)
new_f = fmod
old_cata = CATA(old_f)
new_cata = CATA(new_f)
if self.verbose:
print3(old_cata)
print3(new_cata)
new_num = new_cata.get_new_id(reuse_hole)
if new_num < 0:
raise CataMessageError(
new_f, _(u'no message id available in this catalog'))
newid = self._filename2id(new_f, new_num)
# check message existence
if old_cata[old_num] == None:
raise MessageError(oldid, _(u'unknown message'))
new_cata[new_num] = old_cata[old_num]
del old_cata[old_num]
# write modified catalogues
self.makedirs()
fout = osp.join(self.wrk_pyt, osp.basename(old_f))
content = old_cata.get_content()
open(fout, 'w').write(content)
fout = osp.join(self.wrk_pyt, osp.basename(new_f))
content = new_cata.get_content()
open(fout, 'w').write(content)
print3('Nouveau fichier : %s' % fout)
# modify source using 'oldid' message
l_src = self.d_msg_call.get(oldid, [])
for f in l_src:
ext = osp.splitext(f)[-1]
rdest = self.wrk_fort
if ext == '.py':
rdest = self.wrk_pyt
# already changed ?
fmod = osp.join(rdest, osp.basename(f))
if osp.isfile(fmod):
print3('from %s' % fmod)
f = fmod
txt = open(osp.join(self.repref, f), 'r').read()
new = re.sub('%s([\'\"]+)' % oldid, newid + r'\1', txt)
fout = osp.join(rdest, osp.basename(f))
open(fout, 'w').write(new)
print3('Nouveau fichier : %s' % fout)
def _mess_running_timeout(self, dt, timeout, job):
"""Emit a message when running timeout occured."""
self.run.Mess(ufmt(_(u"The job '%s' has been submitted since %.0f s and is "\
"not ended (timeout %.0f s, equal to 4 * the time asked for the main job). " \
"You can kill it and get other results or wait again..."),
job, dt, timeout))
def __setitem__(self, key, msg):
"""Ajoute le message 'msg' à l'indice 'key'.
"""
if self[key] != None:
raise MessageError(key, _(u'message already exists !'))
self.cata_msg[key] = msg
def start():
# ----- divers
os.stat_float_times(True)
# ----- backward compatibility
from asrun.backward_compatibility import change_argv
sys.argv = change_argv(sys.argv)
# ----- initialisation
run = AsterRun()
magic.run = run
# ----- retrieve options and arguments
opts, args = run.ParseArgs()
# init magic
magic.set_stdout(run['stdout'])
magic.set_stderr(run['stderr'])
magic.init_logger(filename=run['log_progress'], debug=run['debug'])
run.current_action = opts.action
if run.current_action == None:
# if symbolic link "action" -> "as_run --action"
alias = os.path.basename(sys.argv[0])
if alias in run.actions_info.keys():
run.current_action = alias
else:
# default to 'run'
run.current_action = 'run'
#run.parser.error(_(u'you must specify an action'))
# ----- get system commands
run.DBG("Command line run on '%s'" % local_full_host,
"using python executable '%s' :" % sys.executable, sys.argv)
run.system = AsterSystem(run)
run.PostConf()
# ----- debug information
if run['debug']:
run.PrintConfig()
print3(_(u'Arguments :'), repr(args))
print3()
# ----- start 'current_action'
try:
act = run.current_action
if run.options['proxy'] is True:
act = 'call_proxy'
meth = run.actions_info[act]['method']
except KeyError:
run.Mess(
_(u'dictionnary bad defined :') + ' actions_info',
'<F>_PROGRAM_ERROR')
else:
# trap <Control+C>
try:
meth(run, *args)
except KeyboardInterrupt:
run.Mess(
_(u"'--%s' stopped by user") % run.current_action,
'<F>_INTERRUPT')
run.Sortie(0)
def __repr__(self):
"""Résumé du catalogue.
"""
return ufmt(_(u'%3d messages (last=%3d, next=%3d) in %s'),
self.get_nb_msg(), self.get_last_id(), self.get_new_id(),
self.fcata)
try:
c = db.CONNECT('REX', rcdir=confdir, verbose=run['debug'])
except Exception, msg:
run.Mess(msg, '<F>_DB_ERROR')
typ = ReadDB(TYPE, c)
d_typ = {
'AL' : typ['anomalie'],
'EL' : typ['evolution'],
'AOM' : typ['aide utilisation'],
'AO' : typ['anomalie'],
'EO' : typ['evolution'],
'ED' : typ['evolution'],
}
if not d['TYPFIC'] in d_typ.keys():
run.Mess(_(u'Unknown issue type : %s') % d['TYPFIC'], '<F>_PARSE_ERROR')
# 4. create new issue
# 4.1. get fields from db
login = run.system.getuser_host()[0]
loginrex = login
try:
res = c.exe("""SELECT id, _username FROM _user WHERE _loginaster='%s' AND __retired__=0;""" % login)
loginrex = res[0][1]
if len(res) > 1:
run.Mess(_(u"More than one user has '%s' as login in REX database, " \
"'%s' is taken") % (login, loginrex),
'<A>_MULTIPLE_USER')
except (IndexError, db.MySQLError), msg:
run.Mess(str(msg))
run.Mess(_(u'User %s unknown in REX database') % login,
def soumbtc_interactif(self, fbtc):
"""Run btc in interactive mode.
"""
self.jobid = self.pid
# commandes
cmd_batch = '%(fbtc)s 1> %(output)s 2> %(error)s'
cmd_interactif = '%(fbtc)s 2> %(error)s | tee %(output)s'
node = self.dict_info['node']
dico = {
'fbtc': fbtc,
'output': self.flash('output'),
'error': self.flash('error'),
}
# follow output or not
xterm = ''
if self.prof['follow_output'][0] in YES_VALUES:
xterm = self.run['terminal']
if self.prof['depart'][0] != '' or xterm == '':
cmd = cmd_batch % dico
else:
cmd = cmd_interactif % dico
# delayed start
if self.prof['depart'][0] != '':
cmd = "echo '%s' | at %s" % (cmd, self.prof['depart'][0])
elif xterm != '':
if re.search('@E', xterm) == None:
xterm = xterm + ' -e @E'
cmd = xterm.replace('@E', '%s "%s"' % (shell_cmd, cmd))
# run on another node
distant = not is_localhost2(node)
if not distant:
# check xterm command
if xterm != '':
term = xterm.split()[0]
if not os.access(term, os.X_OK):
print3(_(u"Not an executable : %s") % term)
return 7, '', 'unknown'
else:
# check node connection
iret, output = self.run.Shell('echo hello', mach=node)
if output.find('hello') < 0:
print3(output)
print3(
_(u"Connection failure to %s (from %s)") %
(node, self.prof['serveur'][0]))
return 6, '', 'unknown'
# background is not possible if display forwarding is required
# (the connection must stay alive)
need_display = xterm != ''
background = (not need_display) \
or same_hosts2(self.prof['mclient'][0], node,
self.prof['uclient'][0], self.prof['username'][0])
kret, output = self.run.Shell(cmd,
mach=node,
bg=background,
display_forwarding=need_display)
return 0, self.jobid, 'interactif'
def mark_as_closed(run, l_nf, cnx, tagv, expl):
"""Mark a list of issues as closed
l_nf: list of issues numbers
cnx: connection object to database
tagv: tag inserted in issues
expl: True if it concerns a stable version (exploitation)"""
etat = _read_table(run, cnx, STATUS)
prod = _read_table(run, cnx, PRODUIT)
typv = 'dev'
autr = 'expl'
if expl:
typv = 'expl'
autr = 'dev'
d_champ = {
'a_corrige' : '_corrV' + typv,
'v_correct' : '_verCorrV' + typv,
'a_tester' : '_corrV' + autr,
'v_tester' : '_verCorrV' + autr,
}
req_status = 'valide_EDA'
for numf in l_nf:
# 4.1. read the issue
try:
issue = ISSUE(numf, cnx)
except ReadDBError:
run.Mess(_(u'Unable to read issue %s') % numf, '<E>_UNKNOWN_ISSUE')
continue
# 4.2. check issue values
status = issue['_status'].GetPrimValue()
if status != req_status:
run.Mess(_(u"Status of issue %s is not '%s' (%s)") \
% (numf, req_status, status), '<A>_UNEXPECTED_VALUE')
issue['_status'] = etat[req_status]
if issue['_produit'].GetPrimValue() == 'Code_Aster':
if not issue[d_champ['a_corrige']] \
and issue['_type'].GetPrimValue() != 'aide utilisation':
issue[d_champ['a_corrige']] = YES
run.Mess(_(u"issue %s should not been solved in version %s") \
% (numf, typv), '<A>_UNEXPECTED_VALUE')
else:
issue[d_champ['a_corrige']] = NO
issue[d_champ['a_tester']] = NO
# 4.3. fill issue fields
issue[d_champ['v_correct']] = tagv
# 4.4. close issue ?
if not issue[d_champ['a_tester']] or not issue[d_champ['v_tester']] in ('', None):
new_status = etat['attente_doc']
if not issue['_impactDoc']:
new_status = etat['ferme']
issue['_status'] = new_status
run.Mess(_(u'issue %s is closed') % numf)
else:
run.Mess(_(u'issue %s must be solved in V%s too') % (numf, autr))
# 4.5. write issue in database
try:
if not run['debug']:
issue.write(force=True)
else:
issue.repr()
except WriteDBError, msg:
run.Mess(_(u'error occurs during writing issue'), '<F>_DB_ERROR')
"""
import os
import os.path as osp
from math import log10
from asrun.common.i18n import _
from asrun.mystring import print3, ufmt, cleanCR
from asrun.thread import Dispatcher
from asrun.distrib import DistribParametricTask
from asrun.common_func import get_tmpname
from asrun.repart import get_hostrc
from asrun.common.utils import now
fmt_head = '%s %s %s %s %s %s' % (
_(u"job").center(12), _(u"result").center(18), _(u"cpu").rjust(8),
_(u"sys").rjust(8), _(u"cpu+sys").rjust(8), _(u"elapsed").rjust(8))
fmt_resu = '%-12s %-18s %8.2f %8.2f %8.2f %8.2f'
fmt_res2 = '%%4d %s %%4d %s %%8.2f %%8.2f %%8.2f %%8.2f' \
% (_("jobs").ljust(7), _("errors").ljust(10))
fmt_tot = '-' * 12 + ' ' + '-' * 18 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8 + ' ' + '-' * 8
def Parametric(run, prof, runner, numthread='auto', **kargs):
"""Run a parametric study.
"""
run.print_timer = True
# 1. ----- initializations
jn = run['num_job']
def SetParser(run):
"""Configure the command-line parser, add options name to store to the list,
set actions informations.
run : AsterRun object which manages the execution
"""
acts_descr = {
'create_issue' : {
'method' : Creer,
'syntax' : 'issue_file export_file',
'help' : _(u'Insert a new entry in the issue tracking system and '
'copy attached files if an export file is provided')
},
'extract_histor' : {
'method' : Histor,
'syntax' : '[--status=STAT] [--format=FORM] [--all_msg] input_file histor',
'help' : _(u'Extract the content of issues listed in `input_file` to `histor`')
},
'close_issue' : {
'method' : Solder,
'syntax' : '--vers=VERS histor',
'help' : _(u'Fill "corrVdev" or "corrVexpl" field (depends on VERS) in issues found in `histor` and eventually close them')
},
}
opts_descr = {
'status' : {
'args' : ('--status', ),
'kwargs' : {
'action' : 'store',
'default' : 'all',
'metavar' : 'STAT',
'choices' : ('all', 'resolu', 'valide_EDA', 'attente_doc', 'ferme'),
'dest' : 'status',
'help' : _(u'raise an error if issues are not in this status')
}
},
'format' : {
'args' : ('--format', ),
'kwargs' : {
'action' : 'store',
'default' : 'text',
'metavar' : 'FORM',
'choices' : ('text', 'html', 'fsq'),
'dest' : 'format',
'help' : _(u'format of generated histor file (text or html)')
}
},
'all_msg' : {
'args' : ('--all_msg', ),
'kwargs' : {
'action' : 'store_true',
'default' : False,
'dest' : 'all_msg',
'help' : _(u'retrieve all the messages of issues')
}
},
}
title = _(u'Options for issue tracker interface')
run.SetActions(
actions_descr=acts_descr,
actions_order=['create_issue', 'close_issue', 'extract_histor'],
group_options=True, group_title=title, actions_group_title=False,
options_descr=opts_descr,
)
def local_shell(self,
cmd,
bg=False,
verbose=False,
follow_output=False,
alt_comment=None,
interact=False,
separated_stderr=False,
stack_id=3,
**ignore_args):
"""Execute a command shell
cmd : command
bg : put command in background if True
verbose : print status messages during execution if True
follow_output : follow interactively output of command
alt_comment : print this "alternative comment" instead of "cmd"
Return :
iret : exit code if bg = False,
0 if bg = True
output : output lines (as string)
"""
if not alt_comment:
alt_comment = cmd
if len(cmd) > self.MaxCmdLen:
self._mess((_(u'length of command shell greater '\
'than %d characters.') % self.MaxCmdLen), '<A>_ALARM')
self._dbg('cmd :',
cmd,
'background : %s' % bg,
'follow_output : %s' % follow_output,
all=True,
stack_id=stack_id)
self.VerbStart(alt_comment, verbose=verbose)
if follow_output and verbose:
print3(os.linesep + _(u'Command output :'))
var_id = "EXIT_COMMAND_%s" % get_command_id()
fout_name = get_tmpname_base(self._tmpdir,
'local_shell_output',
user=local_user,
node=local_host,
pid="auto")
ferr_name = get_tmpname_base(self._tmpdir,
'local_shell_error',
user=local_user,
node=local_host,
pid="auto")
new_cmd = get_command_line(cmd, bg, follow_output, separated_stderr,
fout_name, ferr_name, var_id)
# execution
iret = os.system(convert(new_cmd))
output, error = "", ""
try:
output = to_unicode(open(fout_name, "r").read())
os.remove(fout_name)
except:
pass
try:
error = to_unicode(open(ferr_name, "r").read())
os.remove(ferr_name)
except:
pass
if follow_output:
# repeat header message
self.VerbStart(alt_comment, verbose=verbose)
mat = re.search('EXIT_CODE=([0-9]+)', output)
if mat:
iret = int(mat.group(1))
elif follow_output:
# os.system returns exit code of tee
mat = re.search("%s=([0-9]+)" % var_id, output)
if mat:
iret = int(mat.group(1))
self.VerbEnd(iret, output, verbose=verbose)
if iret != 0:
print('ERROR : iret = %s' % iret, '+++ STANDARD OUTPUT:', output,
'+++ STANDARD ERROR:')
if bg:
iret = 0
if not separated_stderr:
result = iret, output
else:
result = iret, output, error
return result
try:
unused, not_found = msgman.check_cata()
except CataMessageError, msg:
run.Sortie(4)
if not run['silent']:
print3('Messages inutilises :')
print3(' '.join(unused))
print3('Messages inexistants :')
print3(' '.join(not_found))
if len(unused) + len(not_found) > 0:
run.Sortie(4)
elif action == 'move':
if len(args) != 2:
run.parser.error(
_(u"'--%s %s' requires 2 arguments") %
(run.current_action, action))
msgman.move(*args)
else:
print3()
fmt = '%12s : %s'
for obj in args:
l_msg = list(set(msgman.which_msg(obj)))
if len(l_msg) > 0:
l_msg.sort()
print3(fmt % (obj, l_msg))
l_sub = list(set(msgman.who_use(obj)))
if len(l_sub) > 0:
l_sub.sort()
print3(fmt % (obj, l_sub))
def _gzip_gunzip(self, mode, src, **opts):
"""Called by Gzip (mode=gzip) and Gunzip (mode=gunzip) methods.
Return status and filename of src after de/compression.
Only local files/directories supported !
"""
para = {
'gzip': {
'cmd': 'gzip ',
'niverr': '<F>_COMPRES_ERROR',
'msg0': _(u'no file/directory to compress'),
'comment': _(u'compressing %s'),
'msgerr': _(u'error during compressing %s'),
},
'gunzip': {
# sometimes gunzip doesn't exist, gzip seems safer
'cmd': 'gzip -d ',
'niverr': '<F>_DECOMPRESSION',
'msg0': _(u'no file/directory to decompress'),
'comment': _(u'decompressing %s'),
'msgerr': _(u'error during decompressing %s'),
},
}
if not mode in para.keys():
self._mess(_(u'unknown mode : %s') % mode, '<F>_PROGRAM_ERROR')
iret = 0
if opts.has_key('cmd'):
cmd = opts['cmd']
else:
cmd = para[mode]['cmd']
if opts.has_key('verbose'):
verbose = opts['verbose']
else:
verbose = self.verbose
if opts.has_key('niverr'):
niverr = opts['niverr']
else:
niverr = para[mode]['niverr']
if len(src) < 1:
self._mess(para[mode]['msg0'], '<A>_ALARM')
return iret
if not os.path.isdir(src):
if mode == 'gzip':
name = src + '.gz'
else:
name = re.sub('\.gz$', '', src)
lf = [src]
else:
name = src
lf = glob.glob(os.path.join(src, '*'))
for f in lf:
jret = 0
comment = para[mode]['comment'] % f
if os.path.isdir(f):
self.VerbStart(comment, verbose=verbose)
self.VerbIgnore(verbose=verbose)
else:
jret = self.local_shell(cmd + f,
verbose=verbose,
alt_comment=comment)[0]
if jret != 0:
self._mess(para[mode]['msgerr'] % f, niverr)
iret = max(iret, jret)
return iret, name
def RunAster(run, *args):
"""Allow to run Code_Aster with or without compiling additional source files,
check a development, run a list of test cases...
"""
run.print_timer = True
prev = os.getcwdu()
# ----- check argument
if len(args) != 1:
run.parser.error(
_(u"'--%s' requires one argument") % run.current_action)
# 1. ----- initializations
jn = run['num_job']
fprof = get_tmpname(run, run['tmp_user'], basename='profil_astk')
run.ToDelete(fprof)
# 1.1. ----- check argument type
# 1.1.1. ----- use profile from args
if isinstance(args[0], AsterProfil):
prof = args[0].copy()
prof.WriteExportTo(fprof)
forig = fprof
else:
# 1.1.2. ----- read profile from args
forig = args[0]
kret = run.Copy(fprof, forig, niverr='<F>_PROFILE_COPY')
prof = AsterProfil(fprof, run)
if not run.IsRemote(forig):
export_fname = run.PathOnly(get_absolute_path(forig))
prof.absolutize_filename(export_fname)
if not prof['mode'][0] in ('batch', 'interactif'):
run.Mess(_(u"Unknown mode (%s), use 'interactif' instead") % \
repr(prof['mode'][0]), 'UNEXPECTED_VALUE')
prof['mode'] = ['interactif']
run.DBG("Input export : %s" % fprof, prof)
# 1.2. get AsterConfig and AsterBuild objects
REPREF = prof.get_version_path()
conf = build_config_from_export(run, prof)
build = AsterBuild(run, conf)
DbgPara = {
'debug': {
'exe': conf['BIN_DBG'][0],
'suff': conf['BINOBJ_DBG'][0],
'libast': conf['BINLIB_DBG'][0],
'libfer': conf['BINLIBF_DBG'][0]
},
'nodebug': {
'exe': conf['BIN_NODBG'][0],
'suff': conf['BINOBJ_NODBG'][0],
'libast': conf['BINLIB_NODBG'][0],
'libfer': conf['BINLIBF_NODBG'][0]
},
}
# 1.3. set environment depending on version
for f in conf.get_with_absolute_path('ENV_SH'):
run.AddToEnv(f)
# 1.4. set runner parameters
klass = Runner
# allow to customize of the execution objects
if run.get('schema_execute'):
schem = get_plugin(run['schema_execute'])
run.DBG("calling plugin : %s" % run['schema_execute'])
klass = schem(prof)
runner = klass(conf.get_defines())
iret = runner.set_cpuinfo(prof['mpi_nbnoeud'][0], prof['mpi_nbcpu'][0])
if iret == 1:
run.Mess(
ufmt(
_(u"%s is not a MPI version of Code_Aster. "
"The number of nodes/processors must be equal to 1."),
REPREF), "<F>_INVALID_PARAMETER")
elif iret != 0:
run.Mess(_(u"incorrect value for mpi_nbnoeud (%s) or mpi_nbcpu (%s)") \
% (prof['mpi_nbnoeud'][0], prof['mpi_nbcpu'][0]), '<F>_INVALID_PARAMETER')
# 1.5. rep_trav from profile or from run[...]
reptrav = runner.set_rep_trav(prof['rep_trav'][0], prof['mode'][0])
# write reptrav in the export
prof['rep_trav'] = reptrav
prof.WriteExportTo(prof.get_filename())
#XXX overrides the original export
if forig != prof.get_filename():
run.Copy(forig, prof.get_filename(), niverr='<A>_ALARM')
# add reptrav to LD_LIBRARY_PATH (to find dynamic libs provided by user)
old = os.environ.get("LD_LIBRARY_PATH", "")
os.environ["LD_LIBRARY_PATH"] = (reptrav + os.pathsep + old).strip(
os.pathsep)
# do not reinitialize rep_trav if
if prof['prep_env'][0] not in NO_VALUES:
run.MkDir(reptrav, chmod=0700)
if prof['detr_rep_trav'][0] not in NO_VALUES:
run.ToDelete(reptrav)
# 1.6. copy profile in rep_trav
kret = run.Copy(osp.join(reptrav, jn + '.export'), fprof)
# ... and config file as ./config.txt
conf.WriteConfigTo(osp.join(reptrav, 'config.txt'))
# 1.7. debug/nodebug
dbg = prof['debug'][0]
if dbg == '':
dbg = 'nodebug'
# 1.8. default values
exetmp = osp.join(REPREF, DbgPara[dbg]['exe'])
cmdetmp = osp.join(REPREF, conf['BINCMDE'][0])
eletmp = osp.join(REPREF, conf['BINELE'][0])
# 2. ----- read profile values
# it's valid because exec, cmde and ele must appear only once
# these values will be overidden if they are available in reptrav
# after an occurence of 'make_...'
if prof.Get('DR', 'exec'):
exetmp = prof.Get('DR', 'exec')[0]['path']
if prof.Get('DR', 'cmde'):
cmdetmp = prof.Get('DR', 'cmde')[0]['path']
if prof.Get('DR', 'ele'):
eletmp = prof.Get('DR', 'ele')[0]['path']
# order of actions :
list_actions = [
'make_exec', 'make_cmde', 'make_ele', 'make_etude', 'make_dbg',
'make_env', 'astout', 'distribution', 'multiple', 'exec_crs',
'exec_crp'
]
# 3. ==> Let's go !
# 3.0. check if we know what to do
for act in prof['actions']:
if act == '':
run.Mess(_(u'nothing to do'), 'OK')
elif not act in list_actions:
run.Mess(_(u'unknown action : %s') % act, '<A>_ALARM')
# check if the version allows developments
if conf['DEVEL'][0] in NO_VALUES and \
( 'make_exec' in prof['actions'] or \
'make_cmde' in prof['actions'] or \
'make_ele' in prof['actions'] ):
run.Mess(
_(u'The configuration of this version does not allow '
'user developments.'), '<F>_ERROR')
#
# 3.1. ----- make_exec
#
iret = 0
if 'make_exec' in prof['actions']:
run.DBG(u'Start make_exec action')
exetmp = osp.join(reptrav, 'aster.exe')
tit = _(u'Compilation of source files')
run.Mess(tit, 'TITLE')
run.timer.Start(tit)
repact = osp.join(reptrav, 'make_exec')
repobj = osp.join(repact, 'repobj')
run.MkDir(repact)
lf = []
for typ in ('c', 'f', 'f90'):
for rep in [l['path'] for l in prof.Get('D', typ=typ)]:
jret, lbi = build.Compil(typ.upper(),
rep,
repobj,
dbg,
rep_trav=repact,
error_if_empty=True,
numthread='auto')
iret = max(iret, jret)
lf.extend(lbi)
# liste des fichiers surchargés
vers = get_aster_version(REPREF)
vers = '.'.join(vers[:3])
fsurch = osp.join(repact, 'surchg.f')
listsurcharge(vers, fsurch, lf)
jret, lbi = build.Compil('F', fsurch, repobj, dbg, repact)
run.timer.Stop(tit)
run.CheckOK()
tit = _(u'Build executable')
run.Mess(tit, 'TITLE')
run.timer.Start(tit)
libaster = osp.join(REPREF, DbgPara[dbg]['libast'])
libferm = osp.join(REPREF, DbgPara[dbg]['libfer'])
# for backward compatibility
if run.IsDir(libaster):
libaster = osp.join(libaster, 'lib_aster.lib')
if run.IsDir(libferm):
libferm = osp.join(libferm, 'ferm.lib')
lobj = glob(osp.join(repobj, '*.o'))
# build an archive if there are more than NNN object files
if len(lobj) > 500:
run.timer.Stop(tit)
tit2 = _(u'Add object files to library')
run.timer.Start(tit2)
libtmp = osp.join(repobj, 'libsurch.a')
run.Copy(libtmp, libaster)
kret = build.Archive(repobj, libtmp, force=True)
lobj = []
libaster = libtmp
run.timer.Stop(tit2)
run.timer.Start(tit)
kret = build.Link(exetmp, lobj, libaster, libferm, repact)
run.timer.Stop(tit)
run.CheckOK()
tit = _(u'Copying results')
run.timer.Start(tit, num=999)
if prof.Get('R', typ='exec'):
exe = prof.Get('R', typ='exec')[0]
run.Delete(exe['path'], remove_dirs=False)
iret = run.MkDir(osp.dirname(exe['path']))
iret = run.Copy(exe['path'], exetmp)
exedata = prof.Get('D', typ='exec')
if exedata and exedata[0]['path'] != exe['path']:
exetmp = exedata[0]['path']
run.timer.Stop(tit)
run.Mess(_(u'Code_Aster executable successfully created'), 'OK')
#
# 3.2. ----- make_cmde
#
if 'make_cmde' in prof['actions']:
run.DBG(u'Start make_cmde action')
tit = _(u"Compilation of commands catalogue")
cmdetmp = osp.join(reptrav, 'cata_commande')
run.timer.Start(tit)
repact = osp.join(reptrav, 'make_cmde')
run.MkDir(repact)
kargs = {
'exe': exetmp,
'cmde': cmdetmp,
}
kargs['capy'] = [l['path'] for l in prof.Get('D', typ='capy')]
lfun = prof.Get('D', typ='unig')
if lfun:
kargs['unigest'] = build.GetUnigest(lfun[0]['path'])
if prof.Get('D', typ='py'):
kargs['py'] = [l['path'] for l in prof.Get('D', typ='py')]
jret = build.CompilCapy(REPREF, repact, **kargs) #i18n=True,
run.timer.Stop(tit)
run.CheckOK()
tit = _(u'Copying results')
run.timer.Start(tit)
if prof.Get('R', typ='cmde'):
cmde = prof.Get('R', typ='cmde')[0]
iret = run.MkDir(cmde['path'])
iret = run.Copy(cmde['path'], osp.join(cmdetmp, 'cata*.py*'))
run.timer.Stop(tit)
#
# 3.3. ----- make_ele
#
if 'make_ele' in prof['actions']:
run.DBG(u'Start make_ele action')
tit = _(u"Compilation of elements")
eletmp = osp.join(reptrav, 'elem.1')
run.timer.Start(tit)
repact = osp.join(reptrav, 'make_ele')
run.MkDir(repact)
kargs = {
'exe': exetmp,
'cmde': cmdetmp,
'ele': eletmp,
}
kargs['cata'] = [l['path'] for l in prof.Get('D', typ='cata')]
lfun = prof.Get('D', typ='unig')
if lfun:
kargs['unigest'] = build.GetUnigest(lfun[0]['path'])
if prof.Get('D', typ='py'):
kargs['py'] = [l['path'] for l in prof.Get('D', typ='py')]
jret = build.CompilEle(REPREF, repact, **kargs)
run.timer.Stop(tit)
run.CheckOK()
tit = _(u'Copying results')
run.timer.Start(tit)
if prof.Get('R', typ='ele'):
ele = prof.Get('R', typ='ele')[0]
iret = run.MkDir(osp.dirname(ele['path']))
iret = run.Copy(ele['path'], eletmp)
run.timer.Stop(tit)
#
# 3.4. ----- make_env / make_etude / make_dbg
#
if 'make_env' in prof['actions'] or 'make_etude' in prof['actions'] or \
'make_dbg' in prof['actions']:
run.DBG(u'Start make_etude/make_env/make_dbg action')
os.chdir(reptrav)
run.Mess(_(u'Code_Aster execution'), 'TITLE')
# 3.4.1. prepare reptrav to run Code_Aster (proc# = 0)
only_env = 'make_env' in prof['actions']
kargs = {
'exe': exetmp,
'cmde': cmdetmp,
'ele': eletmp,
'lang': prof['lang'][0],
'only_env': only_env,
}
lfun = prof.Get('D', typ='unig')
if lfun:
kargs['unigest'] = build.GetUnigest(lfun[0]['path'])
if prof.Get('D', typ='py'):
kargs['py'] = [l['path'] for l in prof.Get('D', typ='py')]
tit = _(u'Preparation of environment')
run.timer.Start(tit)
run.Mess(ufmt(_(u'prepare environment in %s'), reptrav))
if prof['prep_env'][0] != 'no':
build.PrepEnv(REPREF, reptrav, dbg=dbg, **kargs)
else:
run.Mess(_(u'... skipped (%s = no) !') % 'prep_env', 'SILENT')
run.timer.Stop(tit)
# 3.4.2. copy datas (raise <E> errors if failed)
tit = _(u'Copying datas')
run.Mess(tit, 'TITLE')
run.timer.Start(tit)
if prof['copy_data'][0] not in NO_VALUES:
copyfiles(run, 'DATA', prof)
else:
run.Mess(_(u'... skipped (%s = no) !') % 'copy_data', 'SILENT')
print3(os.getcwdu())
run.timer.Stop(tit)
# 3.4.3. execution
diag, tcpu, tsys, ttot, copybase = execute(reptrav,
multiple=False,
with_dbg='make_dbg'
in prof['actions'],
only_env=only_env,
runner=runner,
run=run,
conf=conf,
prof=prof,
build=build,
exe=exetmp)
if not 'make_env' in prof['actions']:
# 3.4.4. copy results
tit = _(u'Copying results')
run.Mess(tit, 'TITLE')
run.timer.Start(tit)
if prof['copy_result'][0] not in NO_VALUES:
emit_alarm = prof['copy_result_alarm'][0] not in NO_VALUES
copyfiles(run, 'RESU', prof, copybase, emit_alarm)
else:
run.Mess(
_(u'... skipped (%s = no) !') % 'copy_result', 'SILENT')
run.timer.Stop(tit)
run.Mess(_(u'Code_Aster run ended'), diag)
# 3.4.5. add .resu/.erre to output for testcases
ctest = prof['parent'][0] == "astout"
if ctest:
run.Mess(_(u'Content of RESU file'), 'TITLE')
run.FileCat('fort.8', magic.get_stdout())
run.Mess(_(u'Content of ERROR file'), 'TITLE')
run.FileCat('fort.9', magic.get_stdout())
# 3.4.6. notify the user
if prof['notify'][0]:
content = _(
'[Code_Aster] job %(job)s on %(server)s ended: %(diag)s')
content = content % {
'job': prof.get_jobname(),
'diag': diag,
'server': prof['serveur'][0],
}
dest = ','.join(prof['notify'])
run.SendMail(dest=dest,
text=content,
subject=content.splitlines()[0])
run.Mess(_(u'Email notification sent to %s') % dest)
run.CheckOK()
os.chdir(prev)
# 3.5. ----- astout
if 'astout' in prof['actions']:
run.DBG(u'Start astout action')
kargs = {
'exe': exetmp,
'cmde': cmdetmp,
'ele': eletmp,
'numthread': prof['numthread'][0],
}
os.chdir(reptrav)
RunAstout(run, conf, prof, runner=runner, **kargs)
os.chdir(prev)
# 3.6. ----- distribution
if 'distribution' in prof['actions']:
run.DBG(u'Start distribution action')
kargs = {
'exe': exetmp,
'cmde': cmdetmp,
'ele': eletmp,
'numthread': prof['numthread'][0],
}
Parametric(run, prof, runner=runner, **kargs)
# 3.7. ----- multiple
if 'multiple' in prof['actions']:
run.DBG(u'Start multiple action')
Multiple(run, prof, runner=runner, numthread=prof['numthread'][0])
# 4. ----- clean up
if 'make_env' in prof['actions'] and prof['detr_rep_trav'][
0] not in NO_VALUES:
run.DoNotDelete(reptrav)
