def test_madspin_gridpack(self): self.out_dir = self.run_dir self.generate('g g > t t~', 'sm') #put the MadSpin card ff = open(pjoin(self.out_dir, 'Cards/madspin_card.dat'), 'w') orig_card = open(pjoin(self.out_dir, 'Cards/madspin_card_default.dat')).read() ff.write('set ms_dir %s' % pjoin(self.out_dir, 'MSDIR1')) ff.write(orig_card) ff.close() #reduce the number of events files.cp(pjoin(_file_path, 'input_files', 'run_card_matching.dat'), pjoin(self.out_dir, 'Cards/run_card.dat')) #create the gridpack self.do('launch -f') #move the MS gridpack self.assertTrue(os.path.exists(pjoin(self.out_dir, 'MSDIR1'))) files.mv(pjoin(self.out_dir, 'MSDIR1'), pjoin(self.out_dir, 'MSDIR2')) #put the MadSpin card ff = open(pjoin(self.out_dir, 'Cards/madspin_card.dat'), 'w') ff.write('set ms_dir %s' % pjoin(self.out_dir, 'MSDIR2')) ff.write(orig_card) ff.close() #create the gridpack self.do('launch -f') self.check_parton_output('run_02_decayed_1', 100)
def finalize(self, matrix_elements, history, mg5options, flaglist): """call the mother class, and do a couple of other things relevant to OS subtraction """ super(MadSTRExporter, self).finalize(matrix_elements, history, mg5options, flaglist) os_ids = self.get_os_ids_from_file(pjoin(self.dir_path, 'SubProcesses', 'os_ids.mg')) # add the widths corresponding to the os_ids to coupl.inc particle_dict = self.model.get('particle_dict') # list(set( is needed for particle/antiparticle with the same width keep_widths = list(set([particle_dict[idd].get('width') for idd in os_ids])) self.update_couplinc(keep_widths,pjoin(self.dir_path, 'Source', 'coupl.inc')) # also, add a function that returns these widths to MODEL filename=pjoin(self.dir_path, 'Source', 'MODEL', 'get_mass_width_fcts.f') width_particles = [particle_dict[idd] for idd in os_ids] self.update_get_mass_width(width_particles, filename) # replace the common_run_interface with the one from madstr_plugin internal = pjoin(self.dir_path, 'bin', 'internal') files.mv(pjoin(internal, 'common_run_interface.py'), \ pjoin(internal, 'common_run_interface_MG.py')) files.cp(pjoin(plugin_path, 'common_run_interface.py'), internal) # finally patch fks_singular so that it won't complain about negative # weights for the real emission subprocess.call('patch -p3 < %s' % pjoin(self.template_path, 'fks_singular_patch.txt'), cwd=self.dir_path, shell=True)
def copy_model_resources(self): """Make the copy/symbolic links""" model_path = pjoin(self.export_dir, 'Source', 'MODEL') if os.path.exists(pjoin(model_path, 'ident_card.dat')): mv(pjoin(model_path, 'ident_card.dat'), pjoin(self.export_dir, 'Cards')) cp(pjoin(model_path, 'param_card.dat'), pjoin(self.export_dir, 'Cards')) mv(pjoin(model_path, 'param_card.dat'), pjoin(self.export_dir, 'Cards', 'param_card_default.dat'))
def test_madspin_gridpack(self): self.out_dir = self.run_dir self.generate('g g > t t~', 'sm') #put the MadSpin card ff = open(pjoin(self.out_dir, 'Cards/madspin_card.dat'), 'w') orig_card = open(pjoin(self.out_dir, 'Cards/madspin_card_default.dat')).read() ff.write('set ms_dir %s' % pjoin(self.out_dir, 'MSDIR1')) ff.write(orig_card) ff.close() run_card = banner.RunCardLO( pjoin(self.run_dir, 'Cards', 'run_card.dat')) self.assertFalse('ptj' in run_card.user_set) self.assertFalse('drjj' in run_card.user_set) self.assertFalse('ptj2min' in run_card.user_set) self.assertFalse('ptj3min' in run_card.user_set) self.assertFalse('mmjj' in run_card.user_set) self.assertTrue('ptheavy' in run_card.user_set) self.assertFalse('el' in run_card.user_set) self.assertFalse('ej' in run_card.user_set) self.assertFalse('polbeam1' in run_card.user_set) self.assertFalse('ptl' in run_card.user_set) #reduce the number of events files.cp(pjoin(_file_path, 'input_files', 'run_card_matching.dat'), pjoin(self.out_dir, 'Cards/run_card.dat')) #create the gridpack self.do('launch -f') self.check_parton_output('run_01', 100) self.check_parton_output('run_01_decayed_1', 100) #move the MS gridpack self.assertTrue(os.path.exists(pjoin(self.out_dir, 'MSDIR1'))) files.mv(pjoin(self.out_dir, 'MSDIR1'), pjoin(self.out_dir, 'MSDIR2')) #put the MadSpin card ff = open(pjoin(self.out_dir, 'Cards/madspin_card.dat'), 'w') ff.write('set ms_dir %s' % pjoin(self.out_dir, 'MSDIR2')) ff.write(orig_card) ff.close() #create the gridpack self.do('launch -f') self.check_parton_output('run_02_decayed_1', 100) self.assertEqual(self.debugging, False)
def add_to_file(self, path, seed=None): """Add the banner to a file and change the associate seed in the banner""" if seed is not None: self.set("run_card", "iseed", seed) ff = self.write("%s.tmp" % path, close_tag=False, exclude=['MGGenerationInfo', '/header', 'init']) ff.write("## END BANNER##\n") if self.lhe_version >= 3: #add the original content [ff.write(line) if not line.startswith("<generator name='MadGraph5_aMC@NLO'") else ff.write("<generator name='MadGraph5_aMC@NLO' version='%s'>" % self['mgversion'][:-1]) for line in open(path)] else: [ff.write(line) for line in open(path)] ff.write("</LesHouchesEvents>\n") ff.close() files.mv("%s.tmp" % path, path)
def do_collect(self, line): """MadWeight Function: making the collect of the results""" self.configure() args = self.split_arg(line) self.check_collect(args) xml_reader = MWParserXML() name = self.MWparam.name # 1. Concatanate the file. ############################################# out_dir = pjoin(self.me_dir, 'Events', name) if '-refine' in args: out_path = pjoin(out_dir, 'refine.xml') else: out_path = pjoin(out_dir, 'output.xml') if os.path.exists(out_path): logger.warning( 'Output file already exists. Current one will be tagged with _old suffix' ) logger.warning( 'Run "collect -refine to instead update your current results."' ) files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) files.mv(pjoin(out_dir, 'weights.out'), pjoin(out_dir, 'weights.out')) for MWdir in self.MWparam.MW_listdir: out_dir = pjoin(self.me_dir, 'Events', name, MWdir) files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) out_dir = pjoin(self.me_dir, 'Events', name) fsock = open(out_path, 'w') fsock.write('<madweight>\n<banner>\n') # BANNER for card in [ 'proc_card_mg5.dat', 'MadWeight_card.dat', 'transfer_card.dat', 'param_card.dat', 'run_card.dat' ]: cname = card[:-4] fsock.write('<%s>\n' % cname) fsock.write( open(pjoin(self.me_dir, 'Cards', card)).read().replace('<', '!>')) fsock.write('</%s>\n' % cname) fsock.write('</banner>\n') at_least_one = False for MWdir in self.MWparam.MW_listdir: out_dir = pjoin(self.me_dir, 'Events', name, MWdir) input_dir = pjoin(self.me_dir, 'SubProcesses', MWdir, name) if not os.path.exists(out_dir): os.mkdir(out_dir) if '-refine' in args: out_path = pjoin(out_dir, 'refine.xml') else: out_path = pjoin(out_dir, 'output.xml') fsock2 = open(out_path, 'w') fsock.write('<subprocess id=\'%s\'>\n' % MWdir) fsock2.write('<subprocess id=\'%s\'>\n' % MWdir) for output in glob.glob(pjoin(input_dir, 'output_*_*.xml')): at_least_one = True text = open(output).read() fsock2.write(text) fsock.write(text) os.remove(output) fsock.write('</subprocess>\n') fsock2.write('</subprocess>\n') fsock2.close() fsock.write('\n</madweight>\n') fsock.close() # 2. Special treatment for refine mode if '-refine' in args: xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level']) for MWdir in self.MWparam.MW_listdir: out_dir = pjoin(self.me_dir, 'Events', name, MWdir) ref_output = xml_reader2.read_file(pjoin( out_dir, 'refine.xml')) xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level']) base_output = xml_reader2.read_file( pjoin(out_dir, 'output.xml')) base_output.refine(ref_output) files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) base_output.write(pjoin(out_dir, 'output.xml'), MWdir) elif not at_least_one: logger.warning("Nothing to collect restore _old file as current.") out_dir = pjoin(self.me_dir, 'Events', name) files.mv(pjoin(out_dir, 'output_old.xml'), pjoin(out_dir, 'output.xml')) files.mv(pjoin(out_dir, 'weights_old.out'), pjoin(out_dir, 'weights.out')) for MWdir in self.MWparam.MW_listdir: out_dir = pjoin(self.me_dir, 'Events', name, MWdir) files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) # 3. Read the (final) log file for extracting data total = {} likelihood = {} err_likelihood = {} cards = set() events = set() tf_sets = set() for MW_dir in self.MWparam.MW_listdir: out_dir = pjoin(self.me_dir, 'Events', name, MW_dir) xml_reader = MWParserXML() data = xml_reader.read_file(pjoin(out_dir, 'output.xml')) # log_level = self.MWparam['mw_run']['log_level'] generator = ((int(i), j, int(k), data[i][j][k]) for i in data for j in data[i] for k in data[i][j]) for card, event, tf_set, obj in generator: # update the full list of events/cards cards.add(card) events.add(event) tf_sets.add(tf_set) # now compute the associate value, error[square] if (card, event, tf_set) in total: value, error = total[(card, event, tf_set)] else: value, error = 0, 0 obj.calculate_total() value, error = (value + obj.value, error + obj.error**2) total[(card, event, tf_set)] = (value, error) if tf_set == 1: if value: if card not in likelihood: likelihood[card], err_likelihood[card] = 0, 0 likelihood[card] -= math.log(value) err_likelihood[card] += error / value else: likelihood[card] = float('Inf') err_likelihood[card] = float('nan') # write the weights file: fsock = open(pjoin(self.me_dir, 'Events', name, 'weights.out'), 'w') logger.info('Write output file with weight information: %s' % fsock.name) fsock.write( '# Weight (un-normalize) for each card/event/set of transfer fct\n' ) fsock.write( '# format: LHCO_event_number card_id tf_set_id value integration_error\n' ) events = list(events) events.sort() cards = list(cards) cards.sort() tf_sets = list(tf_sets) tf_sets.sort() for event in events: for card in cards: for tf_set in tf_sets: try: value, error = total[(card, event, tf_set)] except KeyError: continue error = math.sqrt(error) fsock.write( '%s %s %s %s %s \n' % (event.replace('@', ' '), card, tf_set, value, error)) # write the likelihood file: fsock = open( pjoin(self.me_dir, 'Events', name, 'un-normalized_likelihood.out'), 'w') fsock.write( '# Warning: this Likelihood needs a bin by bin normalization !\n') fsock.write( '# IF more than one set of transfer function are define. ONLY the first one is ' ) fsock.write('# include in this file.') fsock.write('# format: card_id value integration_error\n') for card in cards: value, error = likelihood[card], err_likelihood[card] error = math.sqrt(error) fsock.write('%s %s %s \n' % (card, value, error))
def do_launch(self, line): """end of the configuration launched the code""" args = self.split_arg(line) self.check_launch(args) model_line = self.banner.get('proc_card', 'full_model_line') if not self.has_standalone_dir: self.create_standalone_directory() ff = open(pjoin(self.me_dir, 'rw_me','Cards', 'param_card.dat'), 'w') ff.write(self.banner['slha']) ff.close() ff = open(pjoin(self.me_dir, 'rw_me','Cards', 'param_card_orig.dat'), 'w') ff.write(self.banner['slha']) ff.close() cmd = common_run_interface.CommonRunCmd.ask_edit_card_static(cards=['param_card.dat'], ask=self.ask, pwd=pjoin(self.me_dir,'rw_me')) new_card = open(pjoin(self.me_dir, 'rw_me', 'Cards', 'param_card.dat')).read() # Find new tag in the banner and add information if needed if 'initrwgt' in self.banner: if 'type=\'mg_reweighting\'' in self.banner['initrwgt']: blockpat = re.compile(r'''<weightgroup type=\'mg_reweighting\'\s*>(?P<text>.*?)</weightgroup>''', re.I+re.M+re.S) before, content, after = blockpat.split(self.banner['initrwgt']) header_rwgt_other = before + after pattern = re.compile('<weight id=\'mg_reweight_(?P<id>\d+)\'>(?P<info>[^<>]*)</weight>', re.S+re.I+re.M) mg_rwgt_info = pattern.findall(content) maxid = 0 for i, diff in mg_rwgt_info: if int(i) > maxid: maxid = int(i) maxid += 1 rewgtid = maxid else: header_rwgt_other = self.banner['initrwgt'] mg_rwgt_info = [] rewgtid = 1 else: self.banner['initrwgt'] = '' header_rwgt_other = '' mg_rwgt_info = [] rewgtid = 1 # add the reweighting in the banner information: #starts by computing the difference in the cards. s_orig = self.banner['slha'] s_new = new_card old_param = check_param_card.ParamCard(s_orig.splitlines()) new_param = check_param_card.ParamCard(s_new.splitlines()) card_diff = old_param.create_diff(new_param) if card_diff == '': raise self.InvalidCmd, 'original card and new card are identical' mg_rwgt_info.append((str(rewgtid), card_diff)) # re-create the banner. self.banner['initrwgt'] = header_rwgt_other self.banner['initrwgt'] += '\n<weightgroup type=\'mg_reweighting\'>\n' for tag, diff in mg_rwgt_info: self.banner['initrwgt'] += '<weight id=\'mg_reweight_%s\'>%s</weight>\n' % \ (tag, diff) self.banner['initrwgt'] += '\n</weightgroup>\n' self.banner['initrwgt'] = self.banner['initrwgt'].replace('\n\n', '\n') output = open( self.lhe_input.name +'rw', 'w') logger.info('starts to compute weight for events with the following modification to the param_card:') logger.info(card_diff) #write the banner to the output file self.banner.write(output, close_tag=False) # prepare the output file for the weight plot if self.mother: out_path = pjoin(self.mother.me_dir, 'Events', 'reweight.lhe') output2 = open(out_path, 'w') self.banner.write(output2, close_tag=False) new_banner = banner.Banner(self.banner) if not hasattr(self, 'run_card'): self.run_card = new_banner.charge_card('run_card') self.run_card['run_tag'] = 'reweight_%s' % rewgtid new_banner['slha'] = s_new del new_banner['initrwgt'] #ensure that original banner is kept untouched assert new_banner['slha'] != self.banner['slha'] assert 'initrwgt' in self.banner ff = open(pjoin(self.mother.me_dir,'Events',self.mother.run_name, '%s_%s_banner.txt' % \ (self.mother.run_name, self.run_card['run_tag'])),'w') new_banner.write(ff) ff.close() # Loop over all events tag_name = 'mg_reweight_%s' % rewgtid start = time.time() cross = 0 os.environ['GFORTRAN_UNBUFFERED_ALL'] = 'y' if self.lhe_input.closed: self.lhe_input = lhe_parser.EventFile(self.lhe_input.name) for event_nb,event in enumerate(self.lhe_input): #control logger if (event_nb % max(int(10**int(math.log10(float(event_nb)+1))),1000)==0): running_time = misc.format_timer(time.time()-start) logger.info('Event nb %s %s' % (event_nb, running_time)) if (event_nb==10001): logger.info('reducing number of print status. Next status update in 10000 events') weight = self.calculate_weight(event) cross += weight event.reweight_data[tag_name] = weight #write this event with weight output.write(str(event)) if self.mother: event.wgt = weight event.reweight_data = {} output2.write(str(event)) running_time = misc.format_timer(time.time()-start) logger.info('All event done (nb_event: %s) %s' % (event_nb+1, running_time)) output.write('</LesHouchesEvents>\n') output.close() os.environ['GFORTRAN_UNBUFFERED_ALL'] = 'n' if self.mother: output2.write('</LesHouchesEvents>\n') output2.close() # add output information if hasattr(self.mother, 'results'): run_name = self.mother.run_name results = self.mother.results results.add_run(run_name, self.run_card, current=True) results.add_detail('nb_event', event_nb+1) results.add_detail('cross', cross) results.add_detail('error', 'nan') self.mother.create_plot(mode='reweight', event_path=output2.name, tag=self.run_card['run_tag']) #modify the html output to add the original run if 'plot' in results.current.reweight: html_dir = pjoin(self.mother.me_dir, 'HTML', run_name) td = pjoin(self.mother.options['td_path'], 'td') MA = pjoin(self.mother.options['madanalysis_path']) path1 = pjoin(html_dir, 'plots_parton') path2 = pjoin(html_dir, 'plots_%s' % self.run_card['run_tag']) outputplot = path2 combine_plots.merge_all_plots(path2, path1, outputplot, td, MA) #results.update_status(level='reweight') #results.update(status, level, makehtml=True, error=False) #old_name = self.mother.results.current['run_name'] #new_run = '%s_rw_%s' % (old_name, rewgtid) #self.mother.results.add_run( new_run, self.run_card) #self.mother.results.add_detail('nb_event', event_nb+1) #self.mother.results.add_detail('cross', cross) #self.mother.results.add_detail('error', 'nan') #self.mother.do_plot('%s -f' % new_run) #self.mother.update_status('Reweight %s done' % rewgtid, 'madspin') #self.mother.results.def_current(old_name) #self.run_card['run_tag'] = self.run_card['run_tag'][9:] #self.mother.run_name = old_name self.lhe_input.close() files.mv(output.name, self.lhe_input.name) logger.info('Event %s have now the additional weight' % self.lhe_input.name) logger.info('new cross-section is : %g pb' % cross) self.terminate_fortran_executables(new_card_only=True) #store result self.all_cross_section[rewgtid] = cross
def do_collect(self, line): """MadWeight Function: making the collect of the results""" self.configure() args = self.split_arg(line) self.check_collect(args) xml_reader = MWParserXML() name = self.MWparam.name # 1. Concatanate the file. ############################################# out_dir = pjoin(self.me_dir, 'Events', name) if '-refine' in args: out_path = pjoin(out_dir, 'refine.xml') else: out_path = pjoin(out_dir, 'output.xml') if os.path.exists(out_path): logger.warning('Output file already exists. Current one will be tagged with _old suffix') logger.warning('Run "collect -refine to instead update your current results."') files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) files.mv(pjoin(out_dir, 'weights.out'), pjoin(out_dir, 'weights.out')) for MWdir in self.MWparam.MW_listdir: out_dir = pjoin(self.me_dir, 'Events', name, MWdir) files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) out_dir = pjoin(self.me_dir, 'Events', name) fsock = open(out_path, 'w') fsock.write('<madweight>\n<banner>\n') # BANNER for card in ['proc_card_mg5.dat','MadWeight_card.dat','transfer_card.dat','param_card.dat','run_card.dat']: cname = card[:-4] fsock.write('<%s>\n' % cname) fsock.write(open(pjoin(self.me_dir,'Cards',card)).read().replace('<','!>')) fsock.write('</%s>\n' % cname) fsock.write('</banner>\n') at_least_one = False for MWdir in self.MWparam.MW_listdir: out_dir = pjoin(self.me_dir, 'Events', name, MWdir) input_dir = pjoin(self.me_dir, 'SubProcesses', MWdir, name) if not os.path.exists(out_dir): os.mkdir(out_dir) if '-refine' in args: out_path = pjoin(out_dir, 'refine.xml') else: out_path = pjoin(out_dir, 'output.xml') fsock2 = open(out_path,'w') fsock.write('<subprocess id=\'%s\'>\n' % MWdir) fsock2.write('<subprocess id=\'%s\'>\n' % MWdir) for output in glob.glob(pjoin(input_dir, 'output_*_*.xml')): at_least_one = True text = open(output).read() fsock2.write(text) fsock.write(text) os.remove(output) fsock.write('</subprocess>\n') fsock2.write('</subprocess>\n') fsock2.close() fsock.write('\n</madweight>\n') fsock.close() # 2. Special treatment for refine mode if '-refine' in args: xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level']) for MWdir in self.MWparam.MW_listdir: out_dir = pjoin(self.me_dir, 'Events',name, MWdir) ref_output = xml_reader2.read_file(pjoin(out_dir, 'refine.xml')) xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level']) base_output = xml_reader2.read_file(pjoin(out_dir, 'output.xml')) base_output.refine(ref_output) files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) base_output.write(pjoin(out_dir, 'output.xml'), MWdir) elif not at_least_one: logger.warning("Nothing to collect restore _old file as current.") out_dir = pjoin(self.me_dir, 'Events', name) files.mv(pjoin(out_dir, 'output_old.xml'), pjoin(out_dir, 'output.xml')) files.mv(pjoin(out_dir, 'weights_old.out'), pjoin(out_dir, 'weights.out')) for MWdir in self.MWparam.MW_listdir: out_dir = pjoin(self.me_dir, 'Events', name, MWdir) files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) # 3. Read the (final) log file for extracting data total = {} likelihood = {} err_likelihood = {} cards = set() events = set() tf_sets = set() for MW_dir in self.MWparam.MW_listdir: out_dir = pjoin(self.me_dir, 'Events', name, MW_dir) xml_reader = MWParserXML() data = xml_reader.read_file(pjoin(out_dir, 'output.xml')) # log_level = self.MWparam['mw_run']['log_level'] generator = ((int(i),j,int(k),data[i][j][k]) for i in data for j in data[i] for k in data[i][j]) for card, event, tf_set, obj in generator: # update the full list of events/cards cards.add(card) events.add(event) tf_sets.add(tf_set) # now compute the associate value, error[square] if (card,event, tf_set) in total: value, error = total[(card, event, tf_set)] else: value, error = 0, 0 obj.calculate_total() value, error = (value + obj.value, error + obj.error**2) total[(card, event, tf_set)] = (value, error) if tf_set == 1: if value: if card not in likelihood: likelihood[card], err_likelihood[card] = 0, 0 likelihood[card] -= math.log(value) err_likelihood[card] += error / value else: likelihood[card] = float('Inf') err_likelihood[card] = float('nan') # write the weights file: fsock = open(pjoin(self.me_dir, 'Events', name, 'weights.out'), 'w') logger.info('Write output file with weight information: %s' % fsock.name) fsock.write('# Weight (un-normalize) for each card/event/set of transfer fct\n') fsock.write('# format: LHCO_event_number card_id tf_set_id value integration_error\n') events = list(events) events.sort() cards = list(cards) cards.sort() tf_sets = list(tf_sets) tf_sets.sort() for event in events: for card in cards: for tf_set in tf_sets: try: value, error = total[(card, event,tf_set)] except KeyError: continue error = math.sqrt(error) fsock.write('%s %s %s %s %s \n' % (event.replace('@', ' '), card, tf_set, value, error)) # write the likelihood file: fsock = open(pjoin(self.me_dir, 'Events', name, 'un-normalized_likelihood.out'), 'w') fsock.write('# Warning: this Likelihood needs a bin by bin normalization !\n') fsock.write('# IF more than one set of transfer function are define. ONLY the first one is ') fsock.write('# include in this file.') fsock.write('# format: card_id value integration_error\n') for card in cards: value, error = likelihood[card], err_likelihood[card] error = math.sqrt(error) fsock.write('%s %s %s \n' % (card, value, error))
def runIOTests(arg=[''],update=True,force=0,synchronize=False): """ running the IOtests associated to expression. By default, this launch all the tests created in classes inheriting IOTests. """ # Update the tarball, while removing the .backups. def noBackUps(tarinfo): if tarinfo.name.endswith('.BackUp'): return None else: return tarinfo if synchronize: print "Please, prefer updating the reference file automatically "+\ "rather than by hand." tar = tarfile.open(_hc_comparison_tarball, "w:bz2") tar.add(_hc_comparison_files, \ arcname=path.basename(_hc_comparison_files), filter=noBackUps) tar.close() # I am too lazy to work out the difference with the existing tarball and # put it in the log. So this is why one should refrain from editing the # reference files by hand. text = " \nModifications performed by hand on %s at %s in"%(\ str(datetime.date.today()),misc.format_timer(0.0)[14:]) text += '\n MadGraph5_aMC@NLO v. %(version)s, %(date)s\n'%misc.get_pkg_info() log = open(_hc_comparison_modif_log,mode='a') log.write(text) log.close() print "INFO:: Ref. tarball %s updated"%str(_hc_comparison_tarball) return if len(arg)!=1 or not isinstance(arg[0],str): print "Exactly one argument, and in must be a string, not %s."%arg return arg=arg[0] # Extract the tarball for hardcoded comparison if necessary if not path.isdir(_hc_comparison_files): if path.isfile(_hc_comparison_tarball): tar = tarfile.open(_hc_comparison_tarball,mode='r:bz2') tar.extractall(path.dirname(_hc_comparison_files)) tar.close() else: os.makedirs(_hc_comparison_files) # Make a backup of the comparison file directory in order to revert it if # the user wants to ignore the changes detected (only when updating the refs) hc_comparison_files_BackUp = _hc_comparison_files+'_BackUp' if update and path.isdir(_hc_comparison_files): if path.isdir(hc_comparison_files_BackUp): shutil.rmtree(hc_comparison_files_BackUp) shutil.copytree(_hc_comparison_files,hc_comparison_files_BackUp) IOTestManager.testFolders_filter = arg.split('/')[0].split('&') IOTestManager.testNames_filter = arg.split('/')[1].split('&') IOTestManager.filesChecked_filter = '/'.join(arg.split('/')[2:]).split('&') #print "INFO:: Using folders %s"%str(IOTestManager.testFolders_filter) #print "INFO:: Using test names %s"%str(IOTestManager.testNames_filter) #print "INFO:: Using file paths %s"%str(IOTestManager.filesChecked_filter) # Initiate all the IOTests from all the setUp() IOTestsInstances = [] start = time.time() for IOTestsClass in IOTestFinder(): # Instantiate the class IOTestsInstances.append(IOTestsClass()) # Run the setUp IOTestsInstances[-1].setUp() # Find the testIO defined and use them in load mode only, we will run # them later here. IOTestsFunctions = IOTestFinder() IOTestsFunctions.collect_function(IOTestsClass,prefix='testIO') for IOTestFunction in IOTestsFunctions: start = time.time() eval('IOTestsInstances[-1].'+IOTestFunction.split('.')[-1]+\ '(load_only=True)') setUp_time = time.time() - start if setUp_time > 0.5: print colored%(34,"Loading IOtest %s is slow (%s)"% (colored%(32,'.'.join(IOTestFunction.split('.')[-3:])), colored%(34,'%.2fs'%setUp_time))) if len(IOTestsInstances)==0: print "No IOTest found." return # runIOTests cannot be made a classmethod, so I use an instance, but it does # not matter which one as no instance attribute will be used. try: modifications = IOTestsInstances[-1].runIOTests( update = update, force = force, verbose=True, testKeys=IOTestManager.all_tests.keys()) except KeyboardInterrupt: if update: # Remove the BackUp of the reference files. if not path.isdir(hc_comparison_files_BackUp): print "\nWARNING:: Update interrupted and modifications already "+\ "performed could not be reverted." else: shutil.rmtree(_hc_comparison_files) mv(hc_comparison_files_BackUp,_hc_comparison_files) print colored%(34, "\nINFO:: Update interrupted, existing modifications reverted.") sys.exit(0) else: print "\nINFO:: IOTest runs interrupted." sys.exit(0) tot_time = time.time() - start if modifications == 'test_over': print colored%(32,"\n%d IOTests "%len(IOTestManager.all_tests.keys()))+\ "successfully tested in %s."%(colored%(34,'%.2fs'%tot_time)) sys.exit(0) elif not isinstance(modifications,dict): print "Error during the files update." sys.exit(0) if sum(len(v) for v in modifications.values())>0: # Display the modifications text = colored%(34, " \nModifications performed on %s at %s in"%(\ str(datetime.date.today()),misc.format_timer(0.0)[14:])) text += colored%(34, '\n MadGraph5_aMC@NLO v. %(version)s, %(date)s\n'%misc.get_pkg_info()) for key in modifications.keys(): if len(modifications[key])==0: continue text += colored%(32,"The following reference files have been %s :"%key) text += '\n'+'\n'.join([" %s"%mod for mod in modifications[key]]) text += '\n' print text try: answer = Cmd.timed_input(question= "Do you want to apply the modifications listed above? [y/n] >",default="y") except KeyboardInterrupt: answer = 'n' if answer == 'y': log = open(_hc_comparison_modif_log,mode='a') log.write(text) log.close() if IOTestManager._compress_ref_fodler: tar = tarfile.open(_hc_comparison_tarball, "w:bz2") tar.add(_hc_comparison_files, \ arcname=path.basename(_hc_comparison_files), filter=noBackUps) tar.close() print colored%(32,"INFO:: tarball %s updated"%str(_hc_comparison_tarball)) else: print colored%(32,"INFO:: Reference %s updated"%\ str(os.path.basename(_hc_comparison_files))) if len(modifications['created'])>0: print colored%(31,"Some ref. files have been created; add "+\ "them to the revision with\n "+ "bzr add tests/input_files/IOTestsComparison") # Make sure to remove the BackUp files filelist = glob.glob(os.path.join(_hc_comparison_files, '*','*','*.BackUp')) for f in filelist: os.remove(f) else: if path.isdir(hc_comparison_files_BackUp): shutil.rmtree(_hc_comparison_files) shutil.copytree(hc_comparison_files_BackUp,_hc_comparison_files) print colored%(32,"INFO:: No modifications applied.") else: print colored%(31, "ERROR:: Could not revert the modifications. No backup found.") else: print colored%(32,"\nNo modifications performed. No update necessary.") # Remove the BackUp of the reference files. if path.isdir(hc_comparison_files_BackUp): shutil.rmtree(hc_comparison_files_BackUp)
def runIOTests(arg=[''], update=True, force=0, synchronize=False): """ running the IOtests associated to expression. By default, this launch all the tests created in classes inheriting IOTests. """ # Update the tarball, while removing the .backups. def noBackUps(tarinfo): if tarinfo.name.endswith('.BackUp'): return None else: return tarinfo if synchronize: print "Please, prefer updating the reference file automatically "+\ "rather than by hand." tar = tarfile.open(_hc_comparison_tarball, "w:bz2") tar.add(_hc_comparison_files, \ arcname=path.basename(_hc_comparison_files), filter=noBackUps) tar.close() # I am too lazy to work out the difference with the existing tarball and # put it in the log. So this is why one should refrain from editing the # reference files by hand. text = " \nModifications performed by hand on %s at %s in"%(\ str(datetime.date.today()),misc.format_timer(0.0)[14:]) text += '\n MadGraph5_aMC@NLO v. %(version)s, %(date)s\n' % misc.get_pkg_info( ) log = open(_hc_comparison_modif_log, mode='a') log.write(text) log.close() print "INFO:: Ref. tarball %s updated" % str(_hc_comparison_tarball) return if len(arg) != 1 or not isinstance(arg[0], str): print "Exactly one argument, and in must be a string, not %s." % arg return arg = arg[0] # Extract the tarball for hardcoded comparison if necessary if not path.isdir(_hc_comparison_files): if path.isfile(_hc_comparison_tarball): tar = tarfile.open(_hc_comparison_tarball, mode='r:bz2') tar.extractall(path.dirname(_hc_comparison_files)) tar.close() else: os.makedirs(_hc_comparison_files) # Make a backup of the comparison file directory in order to revert it if # the user wants to ignore the changes detected (only when updating the refs) hc_comparison_files_BackUp = _hc_comparison_files + '_BackUp' if update and path.isdir(_hc_comparison_files): if path.isdir(hc_comparison_files_BackUp): shutil.rmtree(hc_comparison_files_BackUp) shutil.copytree(_hc_comparison_files, hc_comparison_files_BackUp) IOTestManager.testFolders_filter = arg.split('/')[0].split('&') IOTestManager.testNames_filter = arg.split('/')[1].split('&') IOTestManager.filesChecked_filter = '/'.join(arg.split('/')[2:]).split('&') #print "INFO:: Using folders %s"%str(IOTestManager.testFolders_filter) #print "INFO:: Using test names %s"%str(IOTestManager.testNames_filter) #print "INFO:: Using file paths %s"%str(IOTestManager.filesChecked_filter) # Initiate all the IOTests from all the setUp() IOTestsInstances = [] start = time.time() for IOTestsClass in IOTestFinder(): # Instantiate the class IOTestsInstances.append(IOTestsClass()) # Run the setUp IOTestsInstances[-1].setUp() # Find the testIO defined and use them in load mode only, we will run # them later here. IOTestsFunctions = IOTestFinder() IOTestsFunctions.collect_function(IOTestsClass, prefix='testIO') for IOTestFunction in IOTestsFunctions: start = time.time() # Add all the tests automatically (i.e. bypass filters) if the # specified test is the name of the IOtest. the [7:] is to # skip the testIO prefix name_filer_bu = None if IOTestFunction.split('.')[-1][7:] in \ IOTestManager.testNames_filter: name_filer_bu = IOTestManager.testNames_filter IOTestManager.testNames_filter = ['ALL'] existing_tests = IOTestManager.all_tests.keys() eval('IOTestsInstances[-1].'+IOTestFunction.split('.')[-1]+\ '(load_only=True)') if name_filer_bu: new_tests = [test[0] for test in IOTestManager.all_tests.keys() \ if test not in existing_tests] IOTestManager.testNames_filter = name_filer_bu + new_tests name_filer_bu = None setUp_time = time.time() - start if setUp_time > 0.5: print colored % ( 34, "Loading IOtest %s is slow (%s)" % (colored % (32, '.'.join(IOTestFunction.split('.')[-3:])), colored % (34, '%.2fs' % setUp_time))) if len(IOTestsInstances) == 0: print "No IOTest found." return # runIOTests cannot be made a classmethod, so I use an instance, but it does # not matter which one as no instance attribute will be used. try: modifications = IOTestsInstances[-1].runIOTests( update=update, force=force, verbose=True, testKeys=IOTestManager.all_tests.keys()) except KeyboardInterrupt: if update: # Remove the BackUp of the reference files. if not path.isdir(hc_comparison_files_BackUp): print "\nWARNING:: Update interrupted and modifications already "+\ "performed could not be reverted." else: shutil.rmtree(_hc_comparison_files) mv(hc_comparison_files_BackUp, _hc_comparison_files) print colored % ( 34, "\nINFO:: Update interrupted, existing modifications reverted." ) sys.exit(0) else: print "\nINFO:: IOTest runs interrupted." sys.exit(0) tot_time = time.time() - start if modifications == 'test_over': print colored%(32,"\n%d IOTests "%len(IOTestManager.all_tests.keys()))+\ "successfully tested in %s."%(colored%(34,'%.2fs'%tot_time)) sys.exit(0) elif not isinstance(modifications, dict): print "Error during the files update." sys.exit(0) if len(modifications['missing']) > 0: text = '\n' text += colored % ( 31, "The following files were not generated by the tests, fix this!") text += '\n' + '\n'.join( [" %s" % mod for mod in modifications['missing']]) print text modifications['missing'] = [] if sum(len(v) for v in modifications.values()) > 0: # Display the modifications text = colored%(34, " \nModifications performed on %s at %s in"%(\ str(datetime.date.today()),misc.format_timer(0.0)[14:])) text += colored % ( 34, '\n MadGraph5_aMC@NLO v. %(version)s, %(date)s\n' % misc.get_pkg_info()) for key in modifications.keys(): if len(modifications[key]) == 0: continue text += colored % ( 32, "The following reference files have been %s :" % key) text += '\n' + '\n'.join( [" %s" % mod for mod in modifications[key]]) text += '\n' print text try: answer = Cmd.timed_input( question= "Do you want to apply the modifications listed above? [y/n] >", default="y") except KeyboardInterrupt: answer = 'n' if answer == 'y': log = open(_hc_comparison_modif_log, mode='a') log.write(text) log.close() if IOTestManager._compress_ref_fodler: tar = tarfile.open(_hc_comparison_tarball, "w:bz2") tar.add(_hc_comparison_files, \ arcname=path.basename(_hc_comparison_files), filter=noBackUps) tar.close() print colored % (32, "INFO:: tarball %s updated" % str(_hc_comparison_tarball)) else: print colored%(32,"INFO:: Reference %s updated"%\ str(os.path.basename(_hc_comparison_files))) if len(modifications['created']) > 0: print colored%(31,"Some ref. files have been created; add "+\ "them to the revision with\n "+ "bzr add tests/input_files/IOTestsComparison") # Make sure to remove the BackUp files filelist = glob.glob( os.path.join(_hc_comparison_files, '*', '*', '*.BackUp')) for f in filelist: os.remove(f) else: if path.isdir(hc_comparison_files_BackUp): shutil.rmtree(_hc_comparison_files) shutil.copytree(hc_comparison_files_BackUp, _hc_comparison_files) print colored % (32, "INFO:: No modifications applied.") else: print colored % ( 31, "ERROR:: Could not revert the modifications. No backup found." ) else: print colored % (32, "\nNo modifications performed. No update necessary.") # Remove the BackUp of the reference files. if path.isdir(hc_comparison_files_BackUp): shutil.rmtree(hc_comparison_files_BackUp)