def Request(self, run=None, **kwjob):
"""Ask for an available host and block resources.
"""
info = self.action('available_host', **kwjob)
host = info.get(self.d_crit['host'][0], None)
if _DBG_ALLOC:
print3('job allocated on %s : %s' % (host, pformat(kwjob)))
if host is not None:
status = ALLOCATED
for key in self.job_keys:
if isnum(kwjob.get(key)):
self.action('add', host, key + 'run', kwjob[key])
else:
status = NORESOURCE
for key, lim in self.limit.items():
if kwjob.get(key) and (\
(self.d_crit[key][2] and kwjob[key] > lim) or \
(not self.d_crit[key][2] and kwjob[key] < lim) ):
status = OVERLIMIT
if run:
run.DBG("OVERLIMIT %s : requested = %s limit = %s (reverse = %s)" \
% (key, kwjob[key], lim, self.d_crit[key][2]))
break
self.action('store_job', host, **kwjob)
if _DBG_ALLOC:
print3(self.Load())
return host, status
def __setid(self, value):
"""Positionne le champ idkey de l'objet
"""
if not self.idkey in self.refs:
self.values[self.idkey] = value
elif __verbose__:
print3('Warning <setid>: %s is an ITEM object.' % repr(self.idkey))
def __init__(self, config_file, run=None, version_path=None):
"""config_file : filename of the 'config.txt' file to read
run : AsterRun object (optional)
version_path : directory of the version (if it is not the dirname of 'config_file').
"""
# ----- initialisations
self.config = {}
self.filename = config_file
self.dirn = version_path or get_absolute_dirname(config_file)
self.verbose = False
self.debug = False
# ----- reference to AsterRun object which manages the execution
self.run = run
if run != None:
self.verbose = run['verbose']
self.debug = run['debug']
# ----- set optional/defaults values (ALWAYS AS LIST !)
self.config = DEFAULTS.copy()
# ----- read config file
if not osp.isfile(config_file):
self._mess(ufmt(_(u'file not found : %s'), config_file),
'<F>_FILE_NOT_FOUND')
f = open(config_file, 'r')
content = f.read()
f.close()
self.config.update(self._parse(content))
if self.debug:
print3('<DBG> <init> AsterConfig :')
print3(self)
def ended(self, job, opts, itemid, calcul, res):
"""Call when a job is ended."""
line = self.summary_line(job, opts, res)
# printing line is not thread safe but it's only printing!
print3(line)
add_to_tail(self.reptrav, line)
# count nook for each thread
gravity = self.run.GetGrav(calcul.diag)
error = gravity == -9 or gravity >= self.run.GetGrav('NOOK')
if error:
self.nbnook[opts['threadid']] += 1
add_to_tail(self.resudir, line, filename='NOOK')
add_to_tail(self.resudir, line, filename='RESULTAT')
output_filename = _(u'no error or flashdir not defined')
# copy output/error to flashdir
if self.flashdir != None:
try:
if not osp.isdir(self.flashdir):
self.run.MkDir(self.flashdir, niverr='SILENT')
except OSError:
pass
self.run.Copy(self.flashdir,
calcul.flash('output'), calcul.flash('error'), calcul.flash('export'),
niverr='<A>_ALARM')
output_filename = osp.join(self.flashdir, osp.basename(calcul.flash('output')))
result = [job, opts]
result.extend(res)
result.append(output_filename)
# clean flasheur
calcul.kill()
return result
def add_filter(self, typ, *args):
"""Ajoute un ou plusieurs filtres.
typ = 'regexp', 'para' ou 'user'.
"""
if typ == 'command':
assert len(args) == 1
expr = self.decode_cmde(args[0])
if expr is not None:
self.filter.append(FILTER_REGEXP(expr))
elif typ == 'regexp':
assert len(args) == 1
expr = args[0]
if expr is not None:
self.filter.append(FILTER_REGEXP(expr))
elif typ == 'para':
assert len(args) == 1
self.filter.append(FILTER_PARA(args[0]))
elif typ == 'user':
assert len(args) == 1
d = globals().copy()
execfile(args[0], d)
if d.has_key('user_filter'):
if type(d['user_filter']) in (list, tuple):
l_filter = d['user_filter']
else:
l_filter = [
d['user_filter'],
]
for ff in l_filter:
assert isinstance(ff, FILTER)
self.filter.extend(l_filter)
else:
print3('>>> unknown filter type (%s) : ignored.' % typ)
def getrepr(self, refs=True):
"""Affichage. Si 'refs'=False, on n'imprime pas les sous-items.
"""
txt = []
for k, v in self.values.items():
if k in self.refs.keys():
item = None
if isinstance(self[k], ITEM):
item = v
idref = v.get(v.idkey)
else:
item = eval('%s(%s)' % (self.refs[k], 'c=self.c'))
try:
item.read({item.idkey:v})
idref = v
except (ConnectDBError, ReadDBError), msg:
item = None
if __verbose__:
print3(msg)
if item != None and (item.__class__.__name__ != self.__class__.__name__\
or idref != self.get(self.idkey)):
if not refs:
txt.append(fmt_vr % (k, idref))
else:
txt.append(fmt_id % (k, idref))
txt.append(os.linesep.join([' > %s' \
% l for l in repr(item).split(os.linesep)]))
txt.append(fmt_fin)
else:
txt.append(fmt_vr % (k, v))
else:
if type(v) == str:
v = to_unicode(v)
txt.append(fmt_val % (k, v))
def AsDict(self):
"""Retourne un dictionnaire avec toutes les valeurs, valeurs du champ
"primaire" pour les sous-items.
"""
def repr_val(val):
v = val
if type(val) in (str, unicode):
mat = re.search(re_date, val)
if mat != None:
v = mat.group(1)
return v
dico = {}
for k, v in self.values.items():
if k.startswith('__'):
continue
if k in self.refs.keys():
item = None
if isinstance(self[k], ITEM):
item = v
idref = v.get(v.idkey)
else:
item = eval('%s(%s)' % (self.refs[k], 'c=self.c'))
try:
item.read({item.idkey:v})
idref = v
except (ConnectDBError, ReadDBError), msg:
item = None
if __verbose__:
print3(msg)
if item != None and (item.__class__.__name__ != self.__class__.__name__\
or idref != self.get(self.idkey)):
dico[k] = repr_val(item.GetPrimValue())
elif not v is None:
dico[k] = repr_val(v)
def Sortie(self, exit_code):
"""Exit by printing diagnostic if defined or exit code if not null"""
# print important messages
msg = self.get_important_messages()
if len(msg) > 0:
print3(msg)
# raise an exception to be catched by the Worker object
if not is_main_thread():
raise TaskAbort(msg)
# helps to locate error in PROGRAM_ERROR case or in verbose mode
if self.diag == '<F>_PROGRAM_ERROR' \
or (exit_code != 0 and (self['debug'] or self['verbose'] or not self.ExitOnFatalError)):
self.DBG(u'### Raise RunAsterError exception because of program error or debug mode ###')
raise RunAsterError, exit_code
# stop all timers
if self.print_timer and hasattr(self, 'timer'):
print3(os.linesep, self.timer)
if self.print_timer: # related to non silent actions
print3(self.program + ' ' + self.__version__)
if self.diag:
# change <E> to <F>
self.diag = self.diag.replace('<E>', '<F>')
print3(self.fmt['diag'] % self.diag)
if exit_code == 0 and \
("NOOK" in self.diag or self.diag == "NO_TEST_RESU"):
exit_code = 1
if self.PrintExitCode or exit_code != 0:
print3(self.fmt['exit'] % exit_code)
self._clear_tmp()
self.DBG("exit %s" % exit_code)
sys.exit(exit_code)
def VerbIgnore(self, verbose=None):
"""End message in verbose mode
"""
if verbose == None:
verbose = self.verbose
if verbose:
print3(label['SKIP'])
def update(self, table, line, cond):
"""Update 'line' (as a list of dict "column:value") into the 'table'
where 'cond' is respected.
Return the number of updated lines.
"""
if not hasattr(self.cursor, 'fetchone'):
print3(_(u'Connection not opened.'))
return 0
query = ['UPDATE %s SET' % table]
lset = []
for k, v in line.items():
if v != None:
if type(v) is LongType:
v = '%d' % v
lset.append('%s=%s' % (k, repr(v)))
query.append(', '.join(lset))
query.append('WHERE')
lwh = []
for k, v in cond.items():
if v != None:
if type(v) is LongType:
v = '%d' % v
lwh.append('%s=%s' % (k, repr(v)))
query.append(' AND '.join(lwh))
query.append(';')
if self.verbose:
print3('### Query : %s' % (' '.join(query)))
try:
nb = self.cursor.execute(' '.join(query))
except:
traceback.print_exc()
raise MySQLError, 'ERROR query = %s' % (' '.join(query))
return nb
def get_first(self, values=None):
"""Return the most available host.
"""
if values is None:
values = self.infos[:]
if len(values) == 0:
return {}
for crit, rev in self.l_crit:
values.sort(reverse=rev)
if _DBG_TRI:
print3(crit)
pprint(values)
val0 = values[0][crit]
new = [
values[0],
]
for info in values[1:]:
if info[crit] != val0:
break
new.append(info)
values = new
if len(values) == 1:
break
if _DBG_TRI:
print3('--- FIN ---')
pprint(values)
return values[0]
def _read_rc(self, ficrc, destdict, optional=False, mcsimp=None):
"""Read a ressource file and store variables to 'destdict'."""
if osp.isfile(ficrc):
read_rcfile(ficrc, destdict, mcsimp=mcsimp)
elif not optional:
print3(ufmt(_(u'file not found : %s'), ficrc))
self.Sortie(4)
def _mess(self, msg, cod='', store=False):
"""Just print a message
"""
if hasattr(self.run, 'Mess'):
self.run.Mess(msg, cod, store)
else:
print3('%-18s %s' % (cod, msg))
def Insert(run, *args):
"""Insert an execution to the database.
"""
if len(args) != 1:
run.parser.error(_(u"'--%s' takes exactly %d arguments (%d given)") % \
(run.current_action, 1, len(args)))
iret = 0
if not run.config.get('astketud'):
run.Mess(_(u"'astketud' is not defined in 'agla' configuration file."),
'<F>_AGLA_ERROR')
# 1. copy export file
jn = run['num_job']
fprof = get_tmpname(run, run['tmp_user'], basename='etude_prof')
kret = run.Copy(fprof, args[0], niverr='<F>_PROFILE_COPY')
# 2. insert study in database
cmd = '%(astketud)s %(profile)s' % {
'astketud': run['astketud'],
'profile': fprof,
}
iret, output = run.Shell(cmd)
if iret != 0:
run.Mess(output, '<F>_DB_ERROR')
# astketud always returns exit=0 !
else:
print3(output)
def Mess(self, msg, cod='', store=False):
"""Print a message sur stdout,
'cod' is an error code (format "<.>_...")
<E> : continue,
<F> : stop,
<A> : alarm, continue,
'' or INFO : for an info,
SILENT : like an info without <INFO> string,
TITLE : add a separator.
If cod='<F>' (without a description), if exists last <E> error is
transformed into <F>, else <F>_ABNORMAL_ABORT.
If store is True, 'msg' is stored in print_on_exit dictionnary."""
# ----- gravite
g0 = self.GetGrav(self.diag)
g1 = self.GetGrav(cod)
coderr = cod
if cod == '<F>':
if g0 < self.GetGrav('<E>'):
coderr = '<F>_ABNORMAL_ABORT'
else:
coderr = self.diag.replace('<E>', '<F>')
if g1 >= self.GetGrav('<A>'):
self.DBG(u'Warning or error raised :', '%s %s' % (cod, msg),
print_traceback=True)
# ----- message
if cod == '' or cod == 'INFO':
fmt = self.fmt['msg_info']
coderr = ''
elif cod == 'SILENT':
fmt = self.fmt['silent']
coderr = ''
elif cod == 'TITLE':
fmt = self.fmt['title']
coderr = ''
else:
fmt = self.fmt['msg+cod']
# unknown flag
if g1 == -9:
coderr = '<I> '+coderr
print3(ufmt(fmt, coderr, msg))
magic.get_stdout().flush()
# ----- store in print_on_exit
if store or (not self.ExitOnFatalError and g1 >= self.GetGrav('<S>')):
k = '?'
msg2 = msg
mat = re.search('<(.)>', cod)
if mat != None and mat.group(1) in ('A', 'S', 'E', 'F'):
k = mat.group(1)
msg2 = self.fmt['msg+cod'] % (coderr, msg)
elif cod in ('OK', 'NOOK'):
k = cod
self.print_on_exit[k] = self.print_on_exit.get(k, []) + [msg2, ]
# ----- diagnostic le plus défavorable
if g1 > g0:
self.diag = coderr
if g1 == self.GetGrav('<F>'):
self.Sortie(4)
def print_stats(self):
"""Affiche les stats sur les données lues/construites
"""
if self.verbose:
print3('%6d messages appelés dans les sources' %
len(self.d_msg_call.keys()))
print3('%6d fichiers sources appellent le catalogue de messages' %
len(self.d_msg_used.keys()))
def search_message(self, fich):
"""Retourne les messages utilisés dans 'fich'.
"""
try:
txt = open(fich, 'r').read()
except IOError, msg:
print3(_(u'Error with file %s : %s') % (fich, msg))
return []
def _dbg(self, *args, **kargs):
"""Print debug informations
"""
if hasattr(self.run, 'DBG'):
kargs['stack_id'] = kargs.get('stack_id', 1)
self.run.DBG(*args, **kargs)
elif self.debug:
print3('<DBG> %s // %s' % (str(args), str(kargs)))
def _get_list_python(self):
"""Retourne la liste des fichiers python
"""
s_pyt = set()
s_pyt.update(glob(osp.join(self.pyt, '*.py')))
s_pyt.update(glob(osp.join(self.pyt, '*', '*.py')))
s_pyt.update(glob(osp.join(self.capy, '*.capy')))
s_pyt.update(glob(osp.join(self.capy, '*', '*.capy')))
if self.build_obj.support('waf'):
l_pyt = [osp.abspath(f) for f in s_pyt]
l_pyt.sort()
return l_pyt
d_py = {}
for f in s_pyt:
typ = osp.splitext(f)[-1][1:]
key = self.build_obj.GetCModif(typ, f)
assert d_py.get(key) is None, 'ERROR : %s (old : %s)' % (
key, d_py[key])
d_py[key] = f
# surcharge
s_surch = set()
for dsrc in self.surch_pyt:
s_surch.update(glob(osp.join(dsrc, '*.py')))
s_surch.update(glob(osp.join(dsrc, '*', '*.py')))
s_surch.update(glob(osp.join(dsrc, '*.capy')))
s_surch.update(glob(osp.join(dsrc, '*', '*.capy')))
if len(s_surch) > 0:
l_surch = list(s_surch)
l_surch.sort()
print3('%5d module(s) python en surcharge : ' % len(l_surch))
print3(os.linesep.join(l_surch))
# retirer des sources originaux ceux de la surcharge...
s_suppr = set()
for f in s_surch:
typ = osp.splitext(f)[-1][1:]
key = self.build_obj.GetCModif(typ, f)
fexist = d_py.get(key)
if fexist:
s_suppr.add(fexist)
if self.verbose: print3('suppression :', fexist)
# ... et ceux de l'unigest
if self.unig:
iunig = 0
for typ in ('py', 'capy'):
for f in self.unig[typ]:
iunig += 1
fabs = osp.abspath(osp.join(self.repref, f))
key = self.build_obj.GetCModif(typ, fabs)
s_suppr.add(d_py.get(key, ''))
if self.verbose: print3('suppression :', fabs)
if iunig > 0:
print3('%5d module(s) python supprime(s).' % iunig)
s_pyt.difference_update(s_suppr)
s_pyt.update(s_surch)
l_pyt = [osp.abspath(f) for f in s_pyt]
l_pyt.sort()
return l_pyt
def __init__(self,
repref,
fort=('bibfor', 'bibf90'),
pyt='bibpyt',
capy='catapy',
cache_dict=None,
verbose=True,
force=False,
ignore_comments=True,
debug=False,
surch_fort=[],
surch_pyt=[],
unig=None):
"""Initialisations."""
if type(fort) not in (list, tuple):
fort = [
fort,
]
if type(surch_fort) not in (list, tuple):
surch_fort = [
surch_fort,
]
if type(surch_pyt) not in (list, tuple):
surch_pyt = [
surch_pyt,
]
self.repref = repref
self.fort = [osp.join(repref, rep) for rep in fort]
self.pyt = osp.join(repref, pyt)
self.capy = osp.join(repref, capy)
self.surch_fort = [osp.abspath(d) for d in surch_fort]
self.surch_pyt = [osp.abspath(d) for d in surch_pyt]
self.verbose = verbose
self.debug = debug
self.force = force
self.message_dir = 'Messages'
self.cache_dict = cache_dict or osp.join('/tmp/',
'messages_dicts.pick')
self.ignore_comments = ignore_comments
self.wrk_fort = osp.join(os.getcwdu(), 'F')
self.wrk_pyt = osp.join(os.getcwdu(), 'PY')
self.unig = unig
# init AsterConfig, AsterBuild objects
fconf = osp.join(self.repref, 'config.txt')
self.conf_obj = AsterConfig(fconf, run=None)
self.build_obj = AsterBuild(run=None, conf=self.conf_obj)
if unig:
# use with_fordepla=False not to mark moved files as removed
self.unig = unigest2dict(unig, self.conf_obj, with_fordepla=True)
if self.verbose:
print3('Repertoires :\n REF=%s\n FORTRAN=%s\n PYTHON=%s\n CAPY=%s' \
% (self.repref, self.fort, self.pyt, self.capy))
self.l_cata = self._get_list_cata()
self.l_src, self.l_pyt = self._get_list_src()
def __getid(self):
"""Retourne la valeur du champ idkey de l'objet
"""
value = self.values[self.idkey]
if not self.idkey in self.refs or not isinstance(value, ITEM):
return value
else:
if __verbose__:
print3('Warning <getid>: %s is an ITEM object.' % repr(self.idkey))
return value.get(value.idkey)
def copyfileD(run, df, icomm, ncomm):
"""Copy datas from `df` into current directory.
Raise only <E> if an error occurs run CheckOK() after.
"""
dest = None
# 1. ----- if logical unit is set : fort.*
if df['ul'] != 0 or df['type'] in ('nom',):
dest = 'fort.%d' % df['ul']
if df['type'] == 'nom':
dest = osp.basename(df['path'])
# exception for multiple command files (adding _N)
if df['ul'] == 1:
icomm += 1
format = '%%0%dd' % (int(log10(max(1, ncomm))) + 1)
dest = dest + '.' + format % icomm
# warning if file already exists
if run.Exists(dest):
run.Mess(ufmt(_(u"'%s' overwrites '%s'"), df['path'], dest), '<A>_COPY_DATA')
if df['compr']:
dest = dest + '.gz'
# 2. ----- bases and directories (ul=0)
else:
# base
if df['type'] in ('base', 'bhdf'):
dest = osp.basename(df['path'])
# ensi
elif df['type'] == 'ensi':
dest = 'DONNEES_ENSIGHT'
# repe
elif df['type'] == 'repe':
dest = 'REPE_IN'
if dest is not None:
# 3. --- copy
kret = run.Copy(dest, df['path'], niverr='<E>_COPY_ERROR', verbose=True)
# 4. --- decompression
if kret == 0 and df['compr']:
kret, dest = run.Gunzip(dest, niverr='<E>_DECOMPRESSION', verbose=True)
# 5. --- move the bases in main directory
if df['type'] in ('base', 'bhdf'):
for f in glob(osp.join(dest, '*')):
run.Rename(f, osp.basename(f))
# force the file to be writable
make_writable(dest)
# clean text files if necessary
if df['ul'] != 0 and run.IsTextFileWithCR(dest):
file_cleanCR(dest)
print3(ufmt(' ` ' + _(u'line terminators have been removed from %s'), dest))
return icomm
def get_export(prof, args, print_output=True, **kwargs):
"""Call --get_export action on a server."""
# read server informations
serv = build_server_from_profile(prof, TYPES.EXEC)
vers = kwargs.get('vers', magic.run['aster_vers'])
iret, output, err = call_generic_service("get_export", serv, prof, args,
{'vers': vers})
if print_output:
print3(output)
return iret, output
def release(self):
"""Release current lock.
"""
if self.lock is None:
raise CopyError, _('No lock to release')
flock = self.lock_filename()
if DEBUG:
print3(u' RELEASED by', self.leech)
if os.path.exists(flock):
os.remove(flock)
self.lock = None
def _get_list_src(self):
"""Retourne la liste des routines fortran et python.
"""
l_f = self._get_list_fort()
l_pyt = self._get_list_python()
if self.verbose:
print3('%5d routines fortran dans %s + %s' %
(len(l_f), self.fort, self.surch_fort))
print3('%5d modules python dans %s + %s' %
(len(l_pyt), self.pyt, self.surch_pyt))
return l_f, l_pyt
def _get_list_fort(self):
"""Retourne la liste des fichiers sources fortran/fortran90."""
s_f = set()
for dsrc in self.fort:
s_f.update(glob(osp.join(dsrc, '*.f')))
s_f.update(glob(osp.join(dsrc, '*', '*.f')))
s_f.update(glob(osp.join(dsrc, '*.F')))
s_f.update(glob(osp.join(dsrc, '*', '*.F')))
s_f.update(glob(osp.join(dsrc, '*.F90')))
s_f.update(glob(osp.join(dsrc, '*', '*.F90')))
if self.build_obj.support('waf'):
l_f = [osp.abspath(f) for f in s_f]
l_f.sort()
return l_f
d_f = {}
for f in s_f:
assert d_f.get(
osp.basename(f)) is None, 'ERROR : %s (old : %s)' % (
f, d_f[osp.basename(f)])
d_f[osp.basename(f)] = f
# surcharge
s_surch = set()
for dsrc in self.surch_fort:
s_surch.update(glob(osp.join(dsrc, '*.f')))
s_surch.update(glob(osp.join(dsrc, '*', '*.f')))
s_surch.update(glob(osp.join(dsrc, '*.F')))
s_surch.update(glob(osp.join(dsrc, '*', '*.F')))
if len(s_surch) > 0:
l_surch = list(s_surch)
l_surch.sort()
print3('%5d sources en surcharge : ' % len(l_surch))
print3(os.linesep.join(l_surch))
# retirer des sources originaux ceux de la surcharge...
s_suppr = set()
for f in s_surch:
fexist = d_f.get(osp.basename(f))
if fexist:
s_suppr.add(fexist)
if self.verbose: print3('suppression :', fexist)
# ... et ceux de l'unigest
if self.unig:
iunig = 0
for f in self.unig['f'] + self.unig['f90']:
iunig += 1
s_suppr.add(d_f.get(osp.basename(f), ''))
if self.verbose: print3('suppression :', f)
if iunig > 0:
print3('%5d source(s) supprime(s).' % iunig)
s_f.difference_update(s_suppr)
s_f.update(s_surch)
l_f = [osp.abspath(f) for f in s_f]
l_f.sort()
return l_f
def __init__(self, astest_dir, all=False, verbose=True):
"""Initialisations
"""
self.astest_dir = force_list(astest_dir)
self.liste_ct = []
self.alltest = all
self.filter = []
self.liste_crit = []
self.list_para_test = list_para_test
self.verbose = verbose
if self.verbose:
print3(ufmt(_(u'Directory of testcases : %s'), self.astest_dir))
def VerbStart(self, cmd, verbose=None):
"""Start message in verbose mode
"""
Lm = self._LineLen
if verbose == None:
verbose = self.verbose
if verbose:
pcmd = cmd
if len(cmd) > Lm - 2 or cmd.count(os.linesep) > 0:
pcmd = pcmd + os.linesep + ' ' * Lm
print3(('%-' + str(Lm) + 's') % (pcmd, ), end=' ')
magic.get_stdout().flush()
def soumbtc_batch(self, fbtc):
"""Run btc script in batch mode.
"""
self.scheduler.set_subpara(fbtc, change_input_script=True)
iret, jobid, queue = self.scheduler.start()
if iret != 0:
print3(_(u"Error during submitting job !"))
iret = 5
if jobid == '':
print3(_(u"Empty jobid"))
iret = 5
return iret, jobid, queue
def available_host(self, **kwjob):
"""Return the most available host accepting job parameters.
"""
if _DBG_CRIT:
print3('job parameters : ', end=' ')
pprint(kwjob)
values = self.suitable_host(**kwjob)
if _DBG_CRIT:
print '%d suitable hosts : %s' \
% (len(values), [info[self.d_crit['host'][0]] for info in values])
avail = self.get_first(values)
return avail
