def build_export_from_files(run, lf, root="", with_default=True, with_results=False):
"""Build an export file from a list of files.
"""
prof = AsterProfil(run=run)
if with_default:
prof.add_default_parameters()
ddat = build_dict_file(run, prof, dict_typ_test(root), lf, ['com?', '[0-9]*'])
dres = {}
if with_results:
dres = build_dict_file(run, prof, dict_typ_result(), lf)
for dicf, dr in ((ddat, 'D'), (dres, 'R')):
lcom_i = []
for f, dico in dicf.items():
if dico['type'] != 'comm' or osp.splitext(f)[-1] == '.comm':
prof.Set(dr, dico)
else:
lcom_i.append([f, dico])
lcom_i.sort()
for f, dico in lcom_i:
prof.Set(dr, dico)
return prof
def run_distrib(self, list_val):
""" Module permettant de lancer N+1 calculs avec le module de calculs distribues d'asrun
"""
# ----------------------------------------------------------------------------
# Parametres
# ----------------------------------------------------------------------------
# Code_Aster installation
ASTER_ROOT = self.ASTER_ROOT
as_run = self.as_run
# General
resudir = self.resudir
clean = self.clean
info = self.info
# Study
export = self.export
# MACR_RECAL inputs
parametres = self.parametres
calcul = self.calcul
experience = self.experience
parametres = self.parametres
calcul = self.calcul
experience = self.experience
CalcGradient = self.CalcGradient
NMAX_SIMULT = self.NMAX_SIMULT
# ----------------------------------------------------------------------------
# Import des modules python d'ASTK
# ----------------------------------------------------------------------------
if not ASTER_ROOT:
try:
ASTER_ROOT = os.environ['ASTER_ROOT']
except:
pass
try:
sys.path.append(
os.path.join(ASTER_ROOT, 'ASTK', 'ASTK_SERV', 'lib'))
sys.path.append(
os.path.join(
ASTER_ROOT, 'lib',
'python%s.%s' % (sys.version_info[0], sys.version_info[1]),
'site-packages'))
except:
pass
assert is_list_of_dict(list_val)
nbval = len(list_val)
# ----------------------------------------------------------------------------
# Generation des etudes esclaves
# ----------------------------------------------------------------------------
sys.argv = ['']
run = AsRunFactory()
# if info<=2: run.options['debug_stderr'] = False # pas d'output d'executions des esclaves dans l'output maitre
if self.unity_follow and info == 2:
run.options['debug_stderr'] = True
else:
run.options[
'debug_stderr'] = False # pas d'output d'executions des esclaves dans l'output maitre
# Master profile
prof = AsterProfil(run=run, filename=export)
tmp_param = tempfile.mkdtemp()
try:
username = prof.param['username'][0]
except:
username = os.environ['LOGNAME']
try:
noeud = prof.param['noeud'][0]
except:
noeud = platform.uname()[1]
tmp_param = "%s@%s:%s" % (username, noeud, tmp_param)
prof.Set(
'R', {
'type': 'repe',
'isrep': True,
'ul': 0,
'compr': False,
'path': tmp_param
})
if info >= 2:
print(prof)
# Si batch n'est pas possible, on bascule en interactif
if not prof['mode'][0] or (prof['mode'][0] == 'batch'
and run.get('batch') == 'non'):
UTMESS('I', 'RECAL0_28', valk=noeud)
prof['mode'] = 'interactif'
prof['version'] = ExecutionParameter().get_option("rcdir")
# result directories
if resudir:
if not os.path.isdir(resudir):
try:
os.mkdir(resudir)
except:
if info >= 1:
UTMESS('A', 'RECAL0_82', valk=resudir)
resudir = None
if not resudir:
# Par defaut, dans un sous-repertoire du repertoire d'execution
pref = 'tmp_macr_recal_'
# On cherche s'il y a un fichier hostfile pour placer les fichiers dans un repertoire partage
l_fr = getattr(prof, 'data')
l_tmp = l_fr[:]
for dico in l_tmp:
if dico['type'] == 'hostfile':
pref = get_shared_tmpdir('tmp_macr_recal_')
break
# Si batch alors on place les fichiers dans un repertoire partage
if prof['mode'][0] == 'batch':
pref = get_shared_tmpdir('tmp_macr_recal1_')
resudir = tempfile.mkdtemp(prefix=pref)
flashdir = os.path.join(resudir, 'flash')
if info >= 1:
UTMESS('I', 'RECAL0_81', valk=resudir)
prof.WriteExportTo(os.path.join(resudir, 'master.export'))
# get hostrc object
hostrc = get_hostrc(run, prof)
# timeout before rejected a job
timeout = prof.get_timeout()
# Ajout des impressions de tables a la fin du .comm
t = []
reponses = calcul
for i in range(len(reponses)):
_ul = str(int(100 + i))
num_ul = '99'
# Pour la dynamique la table avec la matrice MAC a un traitement different
if self.DYNAMIQUE:
if ('MAC' in reponses[i][2]):
t.append(self.ajout_post_mac(reponses[i]))
try:
os.remove('tmp_macr_recal' + os.sep + "REPE_TABLE" + os.sep +
"fort." + _ul)
except:
pass
t.append("\n# Recuperation de la table : " + str(reponses[i][0]) +
"\n")
t.append("DEFI_FICHIER(UNITE=" + num_ul + ", FICHIER='" +
os.path.join('.', 'REPE_OUT', 'fort.' + _ul) + "',);\n")
t.append("IMPR_TABLE(TABLE=" + str(reponses[i][0]) +
", FORMAT='ASTER', UNITE=" + num_ul +
", INFO=1, FORMAT_R='E30.20',);\n")
t.append("DEFI_FICHIER(ACTION='LIBERER', UNITE=" + num_ul +
",);\n")
# Pour la dynamique uniquement
if self.DYNAMIQUE:
if (
self.DYNAMIQUE['APPARIEMENT_MANUEL'] == 'OUI'
and self.graph_mac and
(True
in ['MAC' in reponses[ii][2] for ii in range(len(reponses))])
): ## on cherche a inverser la liste de frequences potentiellement changee par la fenetre MAC
for ind_rep in range(len(reponses)):
if reponses[ind_rep][2] == 'FREQ':
t.append("data1 = " + reponses[ind_rep][0] +
".EXTR_TABLE().Array('" +
reponses[ind_rep][1] + "','FREQ')\n"
) ## on recupere la table des frequences
t.append("nume_freq=data1[:,0].tolist()\n")
t.append("val_freq=data1[:,1].tolist()\n")
t.append(
"val_freq_permute=[]\n"
) ## val_freq_permute contient la liste de frequences permutee
t.append("for ii in range(len(list_num_pour_freq)):\n")
t.append(
" if list_num_pour_freq[ii]==list_exp_pour_freq[ii]:\n"
)
t.append(
" val_freq_permute.append(val_freq[ii])\n")
t.append(" else:\n")
t.append(
" ii_p= list_exp_pour_freq.index(list_num_pour_freq[ii])\n"
)
t.append(
" val_freq_permute.append(val_freq[ii_p])\n")
t.append("DETRUIRE(CONCEPT=_F(NOM=" +
str(reponses[ind_rep][0]) + "),)\n")
t.append(
reponses[ind_rep][0] +
"=CREA_TABLE(LISTE=(_F(PARA='" +
reponses[ind_rep][1] +
"',LISTE_I=nume_freq,),_F(PARA='FREQ',LISTE_R=val_freq_permute,),),)\n"
)
# number of threads to follow execution
numthread = 1
# ----------------------------------------------------------------------------
# Executions des etudes esclaves
# ----------------------------------------------------------------------------
# ----- Execute calcutions in parallel using a Dispatcher object
# elementary task...
task = DistribParametricTask(
run=run,
prof=prof, # IN
hostrc=hostrc,
nbmaxitem=self.NMAX_SIMULT,
timeout=timeout,
resudir=resudir,
flashdir=flashdir,
keywords={'POST_CALCUL': '\n'.join(t)},
info=info,
nbnook=[
0,
] * numthread,
exec_result=[]) # OUT
# ... and dispatch task on 'list_tests'
etiq = 'calc_%%0%dd' % (int(log10(nbval)) + 1)
labels = [etiq % (i + 1) for i in range(nbval)]
couples = list(zip(labels, list_val))
if info >= 2:
print(couples)
execution = Dispatcher(couples, task, numthread=numthread)
# ----------------------------------------------------------------------------
# Liste des diagnostics
# ----------------------------------------------------------------------------
d_diag = {}
for result in task.exec_result:
label = result[0]
diag = result[2]
if len(result) >= 8:
output_filename = os.path.join('~', 'flasheur', str(result[7]))
else:
output_filename = ''
d_diag[label] = diag
# Affichage de l'output de l'esclave dans l'output du maitre
if self.unity_follow:
affiche(unity=self.unity_follow,
filename=output_filename,
label=label,
filetype='stdout')
# Calcul esclave NOOK
if not diag[0:2] in ['OK', '<A']:
# Affichage de l'output et/ou de l'error de l'esclave dans l'output du maitre
try:
affiche(unity=None,
filename=output_filename,
label=label,
filetype='stdout')
error_filename = '.'.join(
output_filename.split('.')
[0:-1]) + '.e' + output_filename.split('.')[-1][1:]
affiche(unity=None,
filename=error_filename,
label=label,
filetype='stderr')
except Exception as e:
print(e)
if diag in ['<F>_NOT_RUN', '<A>_NOT_SUBMITTED']:
UTMESS('F', 'RECAL0_86', valk=(label, diag))
else:
UTMESS('A', 'RECAL0_83', valk=(label, output_filename))
if not d_diag:
UTMESS('F', 'RECAL0_84', valk=resudir)
self.list_diag = [d_diag[label] for label in labels]
# ----------------------------------------------------------------------------
# Arret si tous les jobs ne se sont pas deroules correctement
# ----------------------------------------------------------------------------
if sum(task.nbnook) > 0:
UTMESS('F', 'RECAL0_84', valk=resudir)
# ----------------------------------------------------------------------------
# Recuperation des tables calculees
# ----------------------------------------------------------------------------
Lcalc = []
i = 0
for c in labels:
tbl = get_tables(tables_calc=calcul,
tmp_repe_table=os.path.join(
resudir, c, 'REPE_OUT'),
prof=prof)
Lcalc.append(tbl) # On stocke sous la forme d'une liste de numpy
i += 1
# ----------------------------------------------------------------------------
# Calcul de la fonctionnelle et du gradient
# ----------------------------------------------------------------------------
if debug:
print("AA4/Lcalc=", Lcalc)
fonctionnelle, gradient = self.calc2fonc_gradient(Lcalc)
# ----------------------------------------------------------------------------
# Clean result directories
# ----------------------------------------------------------------------------
if clean:
shutil.rmtree(resudir, ignore_errors=True)
# ----------------------------------------------------------------------------
# Save all calculated responses
# ----------------------------------------------------------------------------
self.Lcalc = Lcalc
return fonctionnelle, gradient