def check_events(self):
"""Check some basic property of the events file"""
sum_of_weight = 0
sum_of_abs_weight = 0
negative_event = 0
positive_event = 0
start = time.time()
for event_nb,event in enumerate(self.lhe_input):
#control logger
if (event_nb % max(int(10**int(math.log10(float(event_nb)+1))),10)==0):
running_time = misc.format_timer(time.time()-start)
logger.info('Event nb %s %s' % (event_nb, running_time))
if (event_nb==10001): logger.info('reducing number of print status. Next status update in 10000 events')
event.check() #check 4 momenta/...
sum_of_weight += event.wgt
sum_of_abs_weight += abs(event.wgt)
if event.wgt < 0 :
negative_event +=1
else:
positive_event +=1
logger.info("total cross-section: %s" % sum_of_weight)
logger.info("total abs cross-section: %s" % sum_of_abs_weight)
logger.info("fraction of negative event %s", negative_event/(negative_event+positive_event))
logger.info("total number of events %s", (negative_event+positive_event))
logger.info("negative event %s", negative_event)
def do_launch(self, line):
"""end of the configuration launched the code"""
args = self.split_arg(line)
self.check_launch(args)
model_line = self.banner.get('proc_card', 'full_model_line')
if not self.has_standalone_dir:
self.create_standalone_directory()
ff = open(pjoin(self.me_dir, 'rw_me','Cards', 'param_card.dat'), 'w')
ff.write(self.banner['slha'])
ff.close()
ff = open(pjoin(self.me_dir, 'rw_me','Cards', 'param_card_orig.dat'), 'w')
ff.write(self.banner['slha'])
ff.close()
cmd = common_run_interface.CommonRunCmd.ask_edit_card_static(cards=['param_card.dat'],
ask=self.ask, pwd=pjoin(self.me_dir,'rw_me'))
new_card = open(pjoin(self.me_dir, 'rw_me', 'Cards', 'param_card.dat')).read()
# Find new tag in the banner and add information if needed
if 'initrwgt' in self.banner:
if 'type=\'mg_reweighting\'' in self.banner['initrwgt']:
blockpat = re.compile(r'''<weightgroup type=\'mg_reweighting\'\s*>(?P<text>.*?)</weightgroup>''', re.I+re.M+re.S)
before, content, after = blockpat.split(self.banner['initrwgt'])
header_rwgt_other = before + after
pattern = re.compile('<weight id=\'mg_reweight_(?P<id>\d+)\'>(?P<info>[^<>]*)</weight>', re.S+re.I+re.M)
mg_rwgt_info = pattern.findall(content)
maxid = 0
for i, diff in mg_rwgt_info:
if int(i) > maxid:
maxid = int(i)
maxid += 1
rewgtid = maxid
else:
header_rwgt_other = self.banner['initrwgt']
mg_rwgt_info = []
rewgtid = 1
else:
self.banner['initrwgt'] = ''
header_rwgt_other = ''
mg_rwgt_info = []
rewgtid = 1
# add the reweighting in the banner information:
#starts by computing the difference in the cards.
s_orig = self.banner['slha']
s_new = new_card
old_param = check_param_card.ParamCard(s_orig.splitlines())
new_param = check_param_card.ParamCard(s_new.splitlines())
card_diff = old_param.create_diff(new_param)
if card_diff == '':
raise self.InvalidCmd, 'original card and new card are identical'
mg_rwgt_info.append((str(rewgtid), card_diff))
# re-create the banner.
self.banner['initrwgt'] = header_rwgt_other
self.banner['initrwgt'] += '\n<weightgroup type=\'mg_reweighting\'>\n'
for tag, diff in mg_rwgt_info:
self.banner['initrwgt'] += '<weight id=\'mg_reweight_%s\'>%s</weight>\n' % \
(tag, diff)
self.banner['initrwgt'] += '\n</weightgroup>\n'
self.banner['initrwgt'] = self.banner['initrwgt'].replace('\n\n', '\n')
output = open( self.lhe_input.name +'rw', 'w')
logger.info('starts to compute weight for events with the following modification to the param_card:')
logger.info(card_diff)
#write the banner to the output file
self.banner.write(output, close_tag=False)
# prepare the output file for the weight plot
if self.mother:
out_path = pjoin(self.mother.me_dir, 'Events', 'reweight.lhe')
output2 = open(out_path, 'w')
self.banner.write(output2, close_tag=False)
new_banner = banner.Banner(self.banner)
if not hasattr(self, 'run_card'):
self.run_card = new_banner.charge_card('run_card')
self.run_card['run_tag'] = 'reweight_%s' % rewgtid
new_banner['slha'] = s_new
del new_banner['initrwgt']
#ensure that original banner is kept untouched
assert new_banner['slha'] != self.banner['slha']
assert 'initrwgt' in self.banner
ff = open(pjoin(self.mother.me_dir,'Events',self.mother.run_name, '%s_%s_banner.txt' % \
(self.mother.run_name, self.run_card['run_tag'])),'w')
new_banner.write(ff)
ff.close()
# Loop over all events
tag_name = 'mg_reweight_%s' % rewgtid
start = time.time()
cross = 0
os.environ['GFORTRAN_UNBUFFERED_ALL'] = 'y'
if self.lhe_input.closed:
self.lhe_input = lhe_parser.EventFile(self.lhe_input.name)
for event_nb,event in enumerate(self.lhe_input):
#control logger
if (event_nb % max(int(10**int(math.log10(float(event_nb)+1))),1000)==0):
running_time = misc.format_timer(time.time()-start)
logger.info('Event nb %s %s' % (event_nb, running_time))
if (event_nb==10001): logger.info('reducing number of print status. Next status update in 10000 events')
weight = self.calculate_weight(event)
cross += weight
event.reweight_data[tag_name] = weight
#write this event with weight
output.write(str(event))
if self.mother:
event.wgt = weight
event.reweight_data = {}
output2.write(str(event))
running_time = misc.format_timer(time.time()-start)
logger.info('All event done (nb_event: %s) %s' % (event_nb+1, running_time))
output.write('</LesHouchesEvents>\n')
output.close()
os.environ['GFORTRAN_UNBUFFERED_ALL'] = 'n'
if self.mother:
output2.write('</LesHouchesEvents>\n')
output2.close()
# add output information
if hasattr(self.mother, 'results'):
run_name = self.mother.run_name
results = self.mother.results
results.add_run(run_name, self.run_card, current=True)
results.add_detail('nb_event', event_nb+1)
results.add_detail('cross', cross)
results.add_detail('error', 'nan')
self.mother.create_plot(mode='reweight', event_path=output2.name,
tag=self.run_card['run_tag'])
#modify the html output to add the original run
if 'plot' in results.current.reweight:
html_dir = pjoin(self.mother.me_dir, 'HTML', run_name)
td = pjoin(self.mother.options['td_path'], 'td')
MA = pjoin(self.mother.options['madanalysis_path'])
path1 = pjoin(html_dir, 'plots_parton')
path2 = pjoin(html_dir, 'plots_%s' % self.run_card['run_tag'])
outputplot = path2
combine_plots.merge_all_plots(path2, path1, outputplot, td, MA)
#results.update_status(level='reweight')
#results.update(status, level, makehtml=True, error=False)
#old_name = self.mother.results.current['run_name']
#new_run = '%s_rw_%s' % (old_name, rewgtid)
#self.mother.results.add_run( new_run, self.run_card)
#self.mother.results.add_detail('nb_event', event_nb+1)
#self.mother.results.add_detail('cross', cross)
#self.mother.results.add_detail('error', 'nan')
#self.mother.do_plot('%s -f' % new_run)
#self.mother.update_status('Reweight %s done' % rewgtid, 'madspin')
#self.mother.results.def_current(old_name)
#self.run_card['run_tag'] = self.run_card['run_tag'][9:]
#self.mother.run_name = old_name
self.lhe_input.close()
files.mv(output.name, self.lhe_input.name)
logger.info('Event %s have now the additional weight' % self.lhe_input.name)
logger.info('new cross-section is : %g pb' % cross)
self.terminate_fortran_executables(new_card_only=True)
#store result
self.all_cross_section[rewgtid] = cross
def runIOTests(arg=[''],update=True,force=0,synchronize=False):
""" running the IOtests associated to expression. By default, this launch all
the tests created in classes inheriting IOTests.
"""
# Update the tarball, while removing the .backups.
def noBackUps(tarinfo):
if tarinfo.name.endswith('.BackUp'):
return None
else:
return tarinfo
if synchronize:
print "Please, prefer updating the reference file automatically "+\
"rather than by hand."
tar = tarfile.open(_hc_comparison_tarball, "w:bz2")
tar.add(_hc_comparison_files, \
arcname=path.basename(_hc_comparison_files), filter=noBackUps)
tar.close()
# I am too lazy to work out the difference with the existing tarball and
# put it in the log. So this is why one should refrain from editing the
# reference files by hand.
text = " \nModifications performed by hand on %s at %s in"%(\
str(datetime.date.today()),misc.format_timer(0.0)[14:])
text += '\n MadGraph5_aMC@NLO v. %(version)s, %(date)s\n'%misc.get_pkg_info()
log = open(_hc_comparison_modif_log,mode='a')
log.write(text)
log.close()
print "INFO:: Ref. tarball %s updated"%str(_hc_comparison_tarball)
return
if len(arg)!=1 or not isinstance(arg[0],str):
print "Exactly one argument, and in must be a string, not %s."%arg
return
arg=arg[0]
# Extract the tarball for hardcoded comparison if necessary
if not path.isdir(_hc_comparison_files):
if path.isfile(_hc_comparison_tarball):
tar = tarfile.open(_hc_comparison_tarball,mode='r:bz2')
tar.extractall(path.dirname(_hc_comparison_files))
tar.close()
else:
os.makedirs(_hc_comparison_files)
# Make a backup of the comparison file directory in order to revert it if
# the user wants to ignore the changes detected (only when updating the refs)
hc_comparison_files_BackUp = _hc_comparison_files+'_BackUp'
if update and path.isdir(_hc_comparison_files):
if path.isdir(hc_comparison_files_BackUp):
shutil.rmtree(hc_comparison_files_BackUp)
shutil.copytree(_hc_comparison_files,hc_comparison_files_BackUp)
IOTestManager.testFolders_filter = arg.split('/')[0].split('&')
IOTestManager.testNames_filter = arg.split('/')[1].split('&')
IOTestManager.filesChecked_filter = '/'.join(arg.split('/')[2:]).split('&')
#print "INFO:: Using folders %s"%str(IOTestManager.testFolders_filter)
#print "INFO:: Using test names %s"%str(IOTestManager.testNames_filter)
#print "INFO:: Using file paths %s"%str(IOTestManager.filesChecked_filter)
# Initiate all the IOTests from all the setUp()
IOTestsInstances = []
start = time.time()
for IOTestsClass in IOTestFinder():
# Instantiate the class
IOTestsInstances.append(IOTestsClass())
# Run the setUp
IOTestsInstances[-1].setUp()
# Find the testIO defined and use them in load mode only, we will run
# them later here.
IOTestsFunctions = IOTestFinder()
IOTestsFunctions.collect_function(IOTestsClass,prefix='testIO')
for IOTestFunction in IOTestsFunctions:
start = time.time()
eval('IOTestsInstances[-1].'+IOTestFunction.split('.')[-1]+\
'(load_only=True)')
setUp_time = time.time() - start
if setUp_time > 0.5:
print colored%(34,"Loading IOtest %s is slow (%s)"%
(colored%(32,'.'.join(IOTestFunction.split('.')[-3:])),
colored%(34,'%.2fs'%setUp_time)))
if len(IOTestsInstances)==0:
print "No IOTest found."
return
# runIOTests cannot be made a classmethod, so I use an instance, but it does
# not matter which one as no instance attribute will be used.
try:
modifications = IOTestsInstances[-1].runIOTests( update = update,
force = force, verbose=True, testKeys=IOTestManager.all_tests.keys())
except KeyboardInterrupt:
if update:
# Remove the BackUp of the reference files.
if not path.isdir(hc_comparison_files_BackUp):
print "\nWARNING:: Update interrupted and modifications already "+\
"performed could not be reverted."
else:
shutil.rmtree(_hc_comparison_files)
mv(hc_comparison_files_BackUp,_hc_comparison_files)
print colored%(34,
"\nINFO:: Update interrupted, existing modifications reverted.")
sys.exit(0)
else:
print "\nINFO:: IOTest runs interrupted."
sys.exit(0)
tot_time = time.time() - start
if modifications == 'test_over':
print colored%(32,"\n%d IOTests "%len(IOTestManager.all_tests.keys()))+\
"successfully tested in %s."%(colored%(34,'%.2fs'%tot_time))
sys.exit(0)
elif not isinstance(modifications,dict):
print "Error during the files update."
sys.exit(0)
if sum(len(v) for v in modifications.values())>0:
# Display the modifications
text = colored%(34, " \nModifications performed on %s at %s in"%(\
str(datetime.date.today()),misc.format_timer(0.0)[14:]))
text += colored%(34,
'\n MadGraph5_aMC@NLO v. %(version)s, %(date)s\n'%misc.get_pkg_info())
for key in modifications.keys():
if len(modifications[key])==0:
continue
text += colored%(32,"The following reference files have been %s :"%key)
text += '\n'+'\n'.join([" %s"%mod for mod in modifications[key]])
text += '\n'
print text
try:
answer = Cmd.timed_input(question=
"Do you want to apply the modifications listed above? [y/n] >",default="y")
except KeyboardInterrupt:
answer = 'n'
if answer == 'y':
log = open(_hc_comparison_modif_log,mode='a')
log.write(text)
log.close()
if IOTestManager._compress_ref_fodler:
tar = tarfile.open(_hc_comparison_tarball, "w:bz2")
tar.add(_hc_comparison_files, \
arcname=path.basename(_hc_comparison_files), filter=noBackUps)
tar.close()
print colored%(32,"INFO:: tarball %s updated"%str(_hc_comparison_tarball))
else:
print colored%(32,"INFO:: Reference %s updated"%\
str(os.path.basename(_hc_comparison_files)))
if len(modifications['created'])>0:
print colored%(31,"Some ref. files have been created; add "+\
"them to the revision with\n "+
"bzr add tests/input_files/IOTestsComparison")
# Make sure to remove the BackUp files
filelist = glob.glob(os.path.join(_hc_comparison_files,
'*','*','*.BackUp'))
for f in filelist:
os.remove(f)
else:
if path.isdir(hc_comparison_files_BackUp):
shutil.rmtree(_hc_comparison_files)
shutil.copytree(hc_comparison_files_BackUp,_hc_comparison_files)
print colored%(32,"INFO:: No modifications applied.")
else:
print colored%(31,
"ERROR:: Could not revert the modifications. No backup found.")
else:
print colored%(32,"\nNo modifications performed. No update necessary.")
# Remove the BackUp of the reference files.
if path.isdir(hc_comparison_files_BackUp):
shutil.rmtree(hc_comparison_files_BackUp)
def runIOTests(arg=[''], update=True, force=0, synchronize=False):
""" running the IOtests associated to expression. By default, this launch all
the tests created in classes inheriting IOTests.
"""
# Update the tarball, while removing the .backups.
def noBackUps(tarinfo):
if tarinfo.name.endswith('.BackUp'):
return None
else:
return tarinfo
if synchronize:
print "Please, prefer updating the reference file automatically "+\
"rather than by hand."
tar = tarfile.open(_hc_comparison_tarball, "w:bz2")
tar.add(_hc_comparison_files, \
arcname=path.basename(_hc_comparison_files), filter=noBackUps)
tar.close()
# I am too lazy to work out the difference with the existing tarball and
# put it in the log. So this is why one should refrain from editing the
# reference files by hand.
text = " \nModifications performed by hand on %s at %s in"%(\
str(datetime.date.today()),misc.format_timer(0.0)[14:])
text += '\n MadGraph5_aMC@NLO v. %(version)s, %(date)s\n' % misc.get_pkg_info(
)
log = open(_hc_comparison_modif_log, mode='a')
log.write(text)
log.close()
print "INFO:: Ref. tarball %s updated" % str(_hc_comparison_tarball)
return
if len(arg) != 1 or not isinstance(arg[0], str):
print "Exactly one argument, and in must be a string, not %s." % arg
return
arg = arg[0]
# Extract the tarball for hardcoded comparison if necessary
if not path.isdir(_hc_comparison_files):
if path.isfile(_hc_comparison_tarball):
tar = tarfile.open(_hc_comparison_tarball, mode='r:bz2')
tar.extractall(path.dirname(_hc_comparison_files))
tar.close()
else:
os.makedirs(_hc_comparison_files)
# Make a backup of the comparison file directory in order to revert it if
# the user wants to ignore the changes detected (only when updating the refs)
hc_comparison_files_BackUp = _hc_comparison_files + '_BackUp'
if update and path.isdir(_hc_comparison_files):
if path.isdir(hc_comparison_files_BackUp):
shutil.rmtree(hc_comparison_files_BackUp)
shutil.copytree(_hc_comparison_files, hc_comparison_files_BackUp)
IOTestManager.testFolders_filter = arg.split('/')[0].split('&')
IOTestManager.testNames_filter = arg.split('/')[1].split('&')
IOTestManager.filesChecked_filter = '/'.join(arg.split('/')[2:]).split('&')
#print "INFO:: Using folders %s"%str(IOTestManager.testFolders_filter)
#print "INFO:: Using test names %s"%str(IOTestManager.testNames_filter)
#print "INFO:: Using file paths %s"%str(IOTestManager.filesChecked_filter)
# Initiate all the IOTests from all the setUp()
IOTestsInstances = []
start = time.time()
for IOTestsClass in IOTestFinder():
# Instantiate the class
IOTestsInstances.append(IOTestsClass())
# Run the setUp
IOTestsInstances[-1].setUp()
# Find the testIO defined and use them in load mode only, we will run
# them later here.
IOTestsFunctions = IOTestFinder()
IOTestsFunctions.collect_function(IOTestsClass, prefix='testIO')
for IOTestFunction in IOTestsFunctions:
start = time.time()
# Add all the tests automatically (i.e. bypass filters) if the
# specified test is the name of the IOtest. the [7:] is to
# skip the testIO prefix
name_filer_bu = None
if IOTestFunction.split('.')[-1][7:] in \
IOTestManager.testNames_filter:
name_filer_bu = IOTestManager.testNames_filter
IOTestManager.testNames_filter = ['ALL']
existing_tests = IOTestManager.all_tests.keys()
eval('IOTestsInstances[-1].'+IOTestFunction.split('.')[-1]+\
'(load_only=True)')
if name_filer_bu:
new_tests = [test[0] for test in IOTestManager.all_tests.keys() \
if test not in existing_tests]
IOTestManager.testNames_filter = name_filer_bu + new_tests
name_filer_bu = None
setUp_time = time.time() - start
if setUp_time > 0.5:
print colored % (
34, "Loading IOtest %s is slow (%s)" %
(colored %
(32, '.'.join(IOTestFunction.split('.')[-3:])), colored %
(34, '%.2fs' % setUp_time)))
if len(IOTestsInstances) == 0:
print "No IOTest found."
return
# runIOTests cannot be made a classmethod, so I use an instance, but it does
# not matter which one as no instance attribute will be used.
try:
modifications = IOTestsInstances[-1].runIOTests(
update=update,
force=force,
verbose=True,
testKeys=IOTestManager.all_tests.keys())
except KeyboardInterrupt:
if update:
# Remove the BackUp of the reference files.
if not path.isdir(hc_comparison_files_BackUp):
print "\nWARNING:: Update interrupted and modifications already "+\
"performed could not be reverted."
else:
shutil.rmtree(_hc_comparison_files)
mv(hc_comparison_files_BackUp, _hc_comparison_files)
print colored % (
34,
"\nINFO:: Update interrupted, existing modifications reverted."
)
sys.exit(0)
else:
print "\nINFO:: IOTest runs interrupted."
sys.exit(0)
tot_time = time.time() - start
if modifications == 'test_over':
print colored%(32,"\n%d IOTests "%len(IOTestManager.all_tests.keys()))+\
"successfully tested in %s."%(colored%(34,'%.2fs'%tot_time))
sys.exit(0)
elif not isinstance(modifications, dict):
print "Error during the files update."
sys.exit(0)
if len(modifications['missing']) > 0:
text = '\n'
text += colored % (
31,
"The following files were not generated by the tests, fix this!")
text += '\n' + '\n'.join(
[" %s" % mod for mod in modifications['missing']])
print text
modifications['missing'] = []
if sum(len(v) for v in modifications.values()) > 0:
# Display the modifications
text = colored%(34, " \nModifications performed on %s at %s in"%(\
str(datetime.date.today()),misc.format_timer(0.0)[14:]))
text += colored % (
34, '\n MadGraph5_aMC@NLO v. %(version)s, %(date)s\n' %
misc.get_pkg_info())
for key in modifications.keys():
if len(modifications[key]) == 0:
continue
text += colored % (
32, "The following reference files have been %s :" % key)
text += '\n' + '\n'.join(
[" %s" % mod for mod in modifications[key]])
text += '\n'
print text
try:
answer = Cmd.timed_input(
question=
"Do you want to apply the modifications listed above? [y/n] >",
default="y")
except KeyboardInterrupt:
answer = 'n'
if answer == 'y':
log = open(_hc_comparison_modif_log, mode='a')
log.write(text)
log.close()
if IOTestManager._compress_ref_fodler:
tar = tarfile.open(_hc_comparison_tarball, "w:bz2")
tar.add(_hc_comparison_files, \
arcname=path.basename(_hc_comparison_files), filter=noBackUps)
tar.close()
print colored % (32, "INFO:: tarball %s updated" %
str(_hc_comparison_tarball))
else:
print colored%(32,"INFO:: Reference %s updated"%\
str(os.path.basename(_hc_comparison_files)))
if len(modifications['created']) > 0:
print colored%(31,"Some ref. files have been created; add "+\
"them to the revision with\n "+
"bzr add tests/input_files/IOTestsComparison")
# Make sure to remove the BackUp files
filelist = glob.glob(
os.path.join(_hc_comparison_files, '*', '*', '*.BackUp'))
for f in filelist:
os.remove(f)
else:
if path.isdir(hc_comparison_files_BackUp):
shutil.rmtree(_hc_comparison_files)
shutil.copytree(hc_comparison_files_BackUp,
_hc_comparison_files)
print colored % (32, "INFO:: No modifications applied.")
else:
print colored % (
31,
"ERROR:: Could not revert the modifications. No backup found."
)
else:
print colored % (32,
"\nNo modifications performed. No update necessary.")
# Remove the BackUp of the reference files.
if path.isdir(hc_comparison_files_BackUp):
shutil.rmtree(hc_comparison_files_BackUp)