def OnPrint( self, evt ): this_time = datetime.datetime.now().strftime("%d/%m/%y") nombre = "ventas"+this_time+".pdf " #import pdb;pdb.set_trace() if self.desde == None or self.hasta == None: wx.MessageBox('Primero debe seleccionar un rango para el reporte', 'Error al generar el reporte', wx.OK | wx.ICON_ERROR) else: dlg = wx.FileDialog( self, message="Choose a file", defaultFile=nombre, wildcard="*.pdf", style=wx.OPEN | wx.MULTIPLE | wx.CHANGE_DIR ) if dlg.ShowModal() == wx.ID_OK: paths = dlg.GetPaths() path = None for path in paths: path = path dlg.Destroy() RPTR = Reporter( path, (self.desde, self.hasta), "ganancia" ) RPTR.doReport()
def __init__(self, config, start, end, vo, verbose, hour_limit, eff_limit, is_test): Reporter.__init__(self, config, start, end, verbose = False) self.hour_limit = hour_limit self.vo = vo self.eff_limit = eff_limit self.is_test = is_test self.verbose = verbose
def __init__(self, ppi): process.__init__(self, ppi) print("cmBPIState CTOR") self.authvalue = "" self.BPIAuthRecRep = Reporter(ppi) #self.BPIAuthRecRep.set_testCaseTitle("CM BPI Authorization State Record") self.results = {} self.startReport()
def learn(self): """Run learning algorithm""" reporter = Reporter() config = self.config total_n_trajectories = np.zeros(len(self.envs)) for iteration in range(config["n_iter"]): self.session.run([self.reset_accum_grads]) for i, learner in enumerate(self.task_learners): # Collect trajectories until we get timesteps_per_batch total timesteps trajectories = learner.get_trajectories() total_n_trajectories[i] += len(trajectories) all_state = np.concatenate( [trajectory["state"] for trajectory in trajectories]) # Compute discounted sums of rewards rets = [ discount_rewards(trajectory["reward"], config["gamma"]) for trajectory in trajectories ] max_len = max(len(ret) for ret in rets) padded_rets = [ np.concatenate([ret, np.zeros(max_len - len(ret))]) for ret in rets ] # Compute time-dependent baseline baseline = np.mean(padded_rets, axis=0) # Compute advantage function advs = [ret - baseline[:len(ret)] for ret in rets] all_action = np.concatenate( [trajectory["action"] for trajectory in trajectories]) all_adv = np.concatenate(advs) # Do policy gradient update step episode_rewards = np.array([ trajectory["reward"].sum() for trajectory in trajectories ]) # episode total rewards episode_lengths = np.array([ len(trajectory["reward"]) for trajectory in trajectories ]) # episode lengths self.session.run( [self.add_accum_grads[i]], feed_dict={ self.state: all_state, self.action_taken: all_action, self.advantage: all_adv }) # summary = self.session.run([self.master.summary_op], feed_dict={ # self.reward: reward # # self.master.episode_length: trajectory["steps"] # }) # self.writer.add_summary(summary[0], iteration) # self.writer.flush() print("Task:", i) reporter.print_iteration_stats(iteration, episode_rewards, episode_lengths, total_n_trajectories[i]) # Apply accumulated gradient after all the gradients of each task are summed self.session.run([self.apply_gradients])
def __init__(self, configuration, start, end, vo, template, is_test, verbose): Reporter.__init__(self, configuration, start, end, verbose) self.is_test = is_test self.vo = vo self.template = template self.title = "Production Jobs Success Rate %s - %s" % (self.start_time, self.end_time) self.run = Jobs() self.clusters = {} self.connectStr = None
def test_reuters_with_graph_sim(): reporter = Reporter(config['windows'], config['methods']) #categories = ['crude', 'livestock', 'earn', 'jobs', 'ship', 'corn', 'trade', 'interest'] categories = ['livestock', 'jobs'] #categories = ['crude', 'livestock', 'jobs', 'ship', 'corn', 'trade', 'interest'] for cat in categories: print '----', cat, '----' reporter.get_category_graph(categories, cat, 'rw_oc', 2)
def __init__(self, configFileName): self.start = time.time() configLoader = ConfigsLoader(configFileName) self.configs = configLoader.get_configs() self.textProcessor = TextProcessor(self.configs) self.trainerWrapper = TrainerWrapper(self.configs) self.reporter = Reporter(self.configs) self.text_cleaner = TextCleaner(self.configs) self.read_config_variables()
def test_reuters_with_bayse(): reporter = Reporter(config['windows'], config['methods']) #categories = ['livestock', 'jobs'] #categories = ['crude', 'livestock', 'jobs', 'ship', 'corn', 'trade', 'interest'] #categories = ['crude', 'livestock', 'earn', 'acq', 'grain', 'wheat', 'money', 'jobs', 'ship', 'corn', 'trade', 'interest'] categories = ['crude', 'livestock', 'earn', 'jobs', 'ship', 'corn', 'trade', 'interest'] #categories = ['acq', 'alum', 'barley', 'bop', 'carcass', 'castor-oil', 'cocoa', 'coconut', 'coconut-oil', 'coffee', 'copper', 'copra-cake', 'corn', 'cotton', 'cotton-oil', 'cpi', 'cpu', 'crude', 'dfl', 'dlr', 'dmk', 'earn', 'fuel', 'gas', 'gnp', 'gold', 'grain', 'groundnut', 'groundnut-oil', 'heat', 'hog', 'housing', 'income', 'instal-debt', 'interest', 'ipi', 'iron-steel', 'jet', 'jobs', 'l-cattle', 'lead', 'lei', 'lin-oil', 'livestock', 'lumber', 'meal-feed', 'money-fx', 'money-supply', 'naphtha', 'nat-gas', 'nickel', 'nkr', 'nzdlr', 'oat', 'oilseed', 'orange', 'palladium', 'palm-oil', 'palmkernel', 'pet-chem', 'platinum', 'potato', 'propane', 'rand', 'rape-oil', 'rapeseed', 'reserves', 'retail', 'rice', 'rubber', 'rye', 'ship', 'silver', 'sorghum', 'soy-meal', 'soy-oil', 'soybean', 'strategic-metal', 'sugar', 'sun-meal', 'sun-oil', 'sunseed', 'tea', 'tin', 'trade', 'veg-oil', 'wheat', 'wpi', 'yen', 'zinc'] for cat in categories: print '----', cat, '----' reporter.get_category_top(categories, cat, 'rw_oc', 3) reporter.get_category_top_tf(categories, cat, 'rw_oc', 3)
def __init__(self, config_file, start, end, is_test=True, verbose=False): """ :param config_file: :param start: :param end: :param is_test: :param verbose: :return: """ Reporter.__init__(self, config_file, start, end, verbose) self.is_test = is_test self.experiments = {} self.connect_str = None
def __init__(self, ppi, iterations): print("cmResetTelnet CTOR") process.__init__(self, ppi) self.ResetCLIRep = Reporter(ppi) self.ResetCLIRep.set_testCaseTitle("CM Reset by CLI Record") self.individualResults = {} self.rawTimes= [] self.resultsList = [] self.overallResults = {} self.startReport() self.Pass_Ct = 0 self.Fail_Ct = 0 self.iterations = iterations
def learn_REINFORCE(self): """Learn using updates like in the REINFORCE algorithm.""" reporter = Reporter() config = self.master.config total_n_trajectories = 0 iteration = 0 while iteration < config["n_iter"] and not self.master.stop_requested: iteration += 1 self.master.session.run([self.master.reset_accum_grads]) # Collect trajectories until we get timesteps_per_batch total timesteps trajectories = self.task_learner.get_trajectories() total_n_trajectories += len(trajectories) all_state = np.concatenate( [trajectory["state"] for trajectory in trajectories]) # Compute discounted sums of rewards rets = [ discount_rewards(trajectory["reward"], config["gamma"]) for trajectory in trajectories ] max_len = max(len(ret) for ret in rets) padded_rets = [ np.concatenate([ret, np.zeros(max_len - len(ret))]) for ret in rets ] # Compute time-dependent baseline baseline = np.mean(padded_rets, axis=0) # Compute advantage function advs = [ret - baseline[:len(ret)] for ret in rets] all_action = np.concatenate( [trajectory["action"] for trajectory in trajectories]) all_adv = np.concatenate(advs) # Do policy gradient update step episode_rewards = np.array([ trajectory["reward"].sum() for trajectory in trajectories ]) # episode total rewards episode_lengths = np.array([ len(trajectory["reward"]) for trajectory in trajectories ]) # episode lengths self.master.session.run( [self.add_accum_grad], feed_dict={ self.master.state: all_state, self.master.action_taken: all_action, self.master.advantage: all_adv }) print("Task:", self.thread_id) reporter.print_iteration_stats(iteration, episode_rewards, episode_lengths, total_n_trajectories) self.master.session.run([self.master.apply_gradients])
def test_doc(): files = [f for f in listdir(config['src']) if isfile(join(config['src'], f))] #files = ['aita2011', 'Мур_алг_статико-дин_расп', 'Програм_Хопфилд'] for f in files: f_full_name = join(config['src'], f) reporter = Reporter(config['windows'], config['methods']) #reporter.report_for_file(f_full_name) #reporter.save_report_detailed(f_full_name, config['trg'] + f + 'DetailCsvRes' + '.csv') #reporter.save_report_top_words(f_full_name, config['trg'] + f + 'TopCsvRes' + '.csv') #reporter.save_report_excel_graph(f_full_name, config['trg'] + f + 'GraphCsvRes' + '.csv') v1, v2 = reporter.similiarity_of_texts('./data/psy/3', f_full_name) print f, ': ', v1, v2 print f, ': ', v1 * (1 + v2)
def learn(self): """Run learning algorithm""" reporter = Reporter() config = self.config total_n_trajectories = 0 for iteration in range(config["n_iter"]): # Collect trajectories until we get timesteps_per_batch total timesteps trajectories = self.get_trajectories() total_n_trajectories += len(trajectories) all_state = np.concatenate( [trajectory["state"] for trajectory in trajectories]) # Compute discounted sums of rewards rets = [ discount_rewards(trajectory["reward"], config["gamma"]) for trajectory in trajectories ] max_len = max(len(ret) for ret in rets) padded_rets = [ np.concatenate([ret, np.zeros(max_len - len(ret))]) for ret in rets ] # Compute time-dependent baseline baseline = np.mean(padded_rets, axis=0) # Compute advantage function advs = [ret - baseline[:len(ret)] for ret in rets] all_action = np.concatenate( [trajectory["action"] for trajectory in trajectories]) all_adv = np.concatenate(advs) # Do policy gradient update step episode_rewards = np.array([ trajectory["reward"].sum() for trajectory in trajectories ]) # episode total rewards episode_lengths = np.array([ len(trajectory["reward"]) for trajectory in trajectories ]) # episode lengths result = self.session.run( [self.summary_op, self.train], feed_dict={ self.state: all_state, self.a_n: all_action, self.adv_n: all_adv, self.episode_lengths: np.mean(episode_lengths), self.rewards: np.mean(episode_rewards) }) self.writer.add_summary(result[0], iteration) self.writer.flush() reporter.print_iteration_stats(iteration, episode_rewards, episode_lengths, total_n_trajectories)
def __init__(self, configuration, start, end, vo, template, is_test, verbose, no_email): Reporter.__init__(self, configuration, start, end, verbose) self.no_email = no_email self.is_test = is_test self.vo = vo self.template = template self.title = "Production Jobs Success Rate {0} - {1}".format(self.start_time, self.end_time) self.run = Jobs() self.clusters = {} self.connectStr = None self.datesplit_pattern = re.compile('[-/ :]') self.usermatch_CILogon = re.compile('.+CN=UID:(\w+)') self.usermatch_FNAL = re.compile('.+/(\w+\.fnal\.gov)') self.jobparts = re.compile('\w+\.(\w+\.\w+\.\w+)#(\w+\.\w+)#.+') self.realhost_pattern = re.compile('\s\(primary\)')
def __init__(self, patient_path): """ :param patient_path: name of the folder containing all the information about the data example: ./data/data-1 """ Thread.__init__(self) self.__patient_path = patient_path self.__low_pass_filter = LowPassFilter(filename=self.__patient_path) self.__band_pass_filter = BandPassFilter(filename=self.__patient_path) self.__hampel_filter = HampelFilter(filename=self.__patient_path) self.__gravity_filter = GravityFilter(filename=self.__patient_path) self.__score = Score(filename=self.__patient_path) self.__reporter = Reporter(filepath=self.__patient_path)
def learn(self): """Run learning algorithm""" reporter = Reporter() config = self.config possible_actions = np.arange(self.nA) total_n_trajectories = 0 for iteration in range(config["n_iter"]): # Collect trajectories until we get timesteps_per_batch total timesteps trajectories = self.get_trajectories() total_n_trajectories += len(trajectories) all_action = np.concatenate( [trajectory["action"] for trajectory in trajectories]) all_action = (possible_actions == all_action[:, None]).astype( np.float32) all_state = np.concatenate( [trajectory["state"] for trajectory in trajectories]) # Compute discounted sums of rewards returns = np.concatenate([ discount_rewards(trajectory["reward"], config["gamma"]) for trajectory in trajectories ]) qw_new = self.get_critic_value(all_state) episode_rewards = np.array([ trajectory["reward"].sum() for trajectory in trajectories ]) # episode total rewards episode_lengths = np.array([ len(trajectory["reward"]) for trajectory in trajectories ]) # episode lengths results = self.sess.run( [self.summary_op, self.critic_train, self.actor_train], feed_dict={ self.critic_state_in: all_state, self.critic_target: returns, self.actor_input: all_state, self.actions_taken: all_action, self.critic_feedback: qw_new, self.critic_rewards: returns, self.rewards: np.mean(episode_rewards), self.episode_lengths: np.mean(episode_lengths) }) self.writer.add_summary(results[0], iteration) self.writer.flush() reporter.print_iteration_stats(iteration, episode_rewards, episode_lengths, total_n_trajectories)
def onPrint( self, evt ): dlg = wx.FileDialog( self, message="Choose a file", defaultFile="reporte-tifosi.pdf", wildcard="*.pdf", style=wx.OPEN | wx.MULTIPLE | wx.CHANGE_DIR ) if dlg.ShowModal() == wx.ID_OK: paths = dlg.GetPaths() path = None for path in paths: path = path dlg.Destroy() RPTR = Reporter( path, self.DBM.getProductos(), "stock" ) RPTR.doReport()
def suiteRunner(patternString): bot = TelegramBot() patternTests = '' if patternString == 'all': patternTests = '*.py' else: patternTests = '*' + patternString + '*.py' tests = unittest.TestLoader().discover('.', pattern=patternTests) unittest.TextTestRunner.resultclass = Resulter reporter = Reporter.createInstance('Reports', patternString) resulter = unittest.TextTestRunner(verbosity=1).run(tests) # Make report and send notification if there're an errors or failures if len(resulter.errors) > 0 or len(resulter.failures) > 0: reporter.makeReport(resulter) bot.sendMessage('Errors:\n' + str(len(resulter.errors)) + '\nFailures:\n' + str(len(resulter.failures)) + '\nSkipped:\n' + str(len(resulter.skipped)) + '\nTestCount:\n' + str(resulter.testsRun)) bot.sendDocument(reporter.getReportFileName()) WebDriverWrapper.getInstance().quit()
def test_doc(): files = [ f for f in listdir(config['src']) if isfile(join(config['src'], f)) ] #files = ['aita2011', 'Мур_алг_статико-дин_расп', 'Програм_Хопфилд'] for f in files: f_full_name = join(config['src'], f) reporter = Reporter(config['windows'], config['methods']) #reporter.report_for_file(f_full_name) #reporter.save_report_detailed(f_full_name, config['trg'] + f + 'DetailCsvRes' + '.csv') #reporter.save_report_top_words(f_full_name, config['trg'] + f + 'TopCsvRes' + '.csv') #reporter.save_report_excel_graph(f_full_name, config['trg'] + f + 'GraphCsvRes' + '.csv') v1, v2 = reporter.similiarity_of_texts('./data/psy/3', f_full_name) print f, ': ', v1, v2 print f, ': ', v1 * (1 + v2)
def main(): start_time = time.time() # Read the issued commands scrape_data, student_data, student_faculty, execute_recommendation, logging_amount, logging_amount_to_file\ = CommandReader().get_user_input(sys.argv) # Initialize class instances status_reporter = Reporter(logging_amount, logging_amount_to_file) data_preserver = Preserver(status_reporter) status_reporter.preserver = data_preserver status_reporter.delete_old_logfile() web_scraper = Scraper(data_preserver, status_reporter) course_recommender = Recommender(data_preserver, status_reporter) if scrape_data: # Scrape UTA courses and study modules data web_scraper.scrape() real_student = True if student_data == 'y': # Scrape UTA student's personal study record to obtain completed courses web_scraper.scrape_student(student_faculty) else: # Make recommendations to an artificial student with predetermined completed courses real_student = False course_recommender.get_fake_student_completed_courses(student_data) if execute_recommendation: # Recommend courses to student course_recommender.recommend(real_student) status_reporter.program_execution_time(time.time() - start_time)
def main(): from optparse import OptionParser usage = "usage: python %prog [options] arg" parser = OptionParser(usage) parser.add_option('-f', '--file', dest='filename', default='%s/FM_LFOS/example1.cfr' % os.getenv('LFOS_DIR'), help='The Clafer file containing feature model and configuration.') parser.add_option('-t', '--tool', dest='IGtool', default='%s/tools/chocosolver.jar' % os.getenv('LFOS_DIR') , help='The Instance Generator tool.') parser.add_option('--fmodel', dest='fmodelfile', default='%s/FM_LFOS/feature_model.cfr' % os.getenv('LFOS_DIR'), help='The Clafer file containing only the feature model.') parser.add_option('-o', '--output', dest='output', default='FMTranslater.tex', help='Output file name to create a Latex file.') parser.add_option('-?', action='help', help='Shows this help message and exits') options, args = parser.parse_args() instance_data = exec_IG_tool(options.IGtool, options.filename) features = parse_feature_model(options.fmodelfile) # print '\n'.join(instance_data[0]) scheduler_instance = construct_structure(instance_data[0], features) assert isinstance(scheduler_instance, Feature) scheduler_instance.pretty_print() report = Reporter(options.output) report.open() for instance in scheduler_instance.traverse(): # print instance.name report.accept(instance) report.close()
def learn(self): """Run learning algorithm""" reporter = Reporter() config = self.config total_n_trajectories = 0 for iteration in range(config["n_iter"]): # Collect trajectories until we get timesteps_per_batch total timesteps trajectories = self.get_trajectories(self.env) total_n_trajectories += len(trajectories) all_action = np.concatenate( [trajectory["action"] for trajectory in trajectories]) all_ob = np.concatenate( [trajectory["ob"] for trajectory in trajectories]) # Compute discounted sums of rewards returns = np.concatenate([ discount_rewards(trajectory["reward"], config["gamma"]) for trajectory in trajectories ]) qw_new = self.get_critic_value(all_ob) print(qw_new) self.sess.run( [self.critic_train], feed_dict={ self.critic_state_in: all_ob, self.critic_target: returns.reshape(-1, 1) }) target = np.mean((returns - qw_new)**2) self.sess.run( [self.actor_train], feed_dict={ self.input_state: all_ob, self.actions_taken: all_action, self.target: target }) episode_rewards = np.array([ trajectory["reward"].sum() for trajectory in trajectories ]) # episode total rewards episode_lengths = np.array([ len(trajectory["reward"]) for trajectory in trajectories ]) # episode lengths reporter.print_iteration_stats(iteration, episode_rewards, episode_lengths, total_n_trajectories)
class TestResultDictProcess(TestCase): def setUp(self): self.r1 = Reporter() def tearDown(self): pass def test_single_result(self): actual = self.r1.single_function_recent_history([result_dict1]) actual_gcred = actual["submitGoogleCredentials"][0] actual_login = actual["login"][0] self.assertAlmostEqual(1.3807, actual_gcred, 4) self.assertAlmostEqual(11.4476, actual_login, 4) def test_several_results(self): actual = self.r1.single_function_recent_history([result_dict2, result_dict1]) actual_close = actual["closeOtherWindows"] self.assertAlmostEqual(0.0131, actual_close[0], 4)
def __init__(self): self.get_apk_from_manager() self.trigger = Trigger.get_trigger_for(self.get_filter_tag(), self.get_package_name(), self.get_description()) self.reporter = Reporter.get_reporter_for(self.get_filter_tag(), self.get_package_name(), self.get_description()) self.analyzer = Analyzer.get_analyzer_for(self.get_filter_tag(), self.get_package_name(), self.get_description()) self.emulator = Emulator.get_emulator_for(self.get_filter_tag(), self.get_package_name(), self.get_description()) self.error_queue = multiprocessing.Queue() self.setup_device()
def main(): logger.info('Start running loadtest') parser = OptionParser() parser.add_option("--config", dest="config", default="config.json") (options, args) = parser.parse_args() config = Config() config.loadFromFile(options.config) logger.info('Loadtest config: \n{}'.format(str(config))) logger.info('Start running workload') workload = WorkLoad(config) # Execute workload in a worker thread # since reporter must run in main thread workload.start() # Note: report must be running as the # last one since it blocks the main thread logger.info('Start running reporter') reporter = Reporter(config) reporter.run()
def test_getUserWithMoreTw( self ): #OK (falta el caso en que sean varios users en el 1er puesto) #Inicializamos los Tweets diccionarios: tweet1 = { "id_str": "123456", "user": { "name": "NASAOk", "id_str": "789456" }, "entities": { "hashtags": ["#mars", "#venus", "#earth"], "user_mentions": ["@NASA", "@planets"] }, "created_at": "Sun Mar 20 15:11:01 +0000 2018", } tweet2 = { "id_str": "112112", "user": { "name": "MauricioOK", "id_str": "451325" }, "entities": { "hashtags": ["#DonaldNoMeDejes"], "user_mentions": ["@donaldTrump", "@G20"] }, "created_at": "Sun Mar 20 21:08:01 +0000 2018", } tweet3 = { "id_str": "112545", "user": { "name": "MauricioOK", "id_str": "451325" }, "entities": { "hashtags": ["#DonaldNoMeDejes"], "user_mentions": ["@donaldTrump", "@G20"] }, "created_at": "Mon Mar 21 23:08:01 +0000 2018", } #Para pasarlos a un objeto Tweet: T1=Tweet(tweet1) #Para pasar estos objetos Tweet a Json usando el metodo to_json del objeto Tweet: t1=T1.to_json() #Lo que me llegaría de Connector().returnTweetsbyIDC(idC): tweets = [tweet1, tweet2, tweet3] mostTwUser = Reporter.Reporter().getUserWithMoreTw(tweets) return mostTwUser
def __init__(self, func, n_cpu = None, reporter=None, task='multiprocessing', entries='jobs', cb_func=None): if not reporter: self.reporter = Reporter(task, entries=entries) else: self.reporter = reporter self.func = func if n_cpu: self.pool = mp.Pool(n_cpu) else: self.pool = mp.Pool() self.cb_func = cb_func if self.cb_func: self.queue = mp.Queue() self.reader_p = mp.Process(target=self.reader_helper) self.reader_p.daemon = True self.reader_p.start() else: self.data = []
class MultiWorker: def __init__(self, func, n_cpu = None, reporter=None, task='multiprocessing', entries='jobs', cb_func=None): if not reporter: self.reporter = Reporter(task, entries=entries) else: self.reporter = reporter self.func = func if n_cpu: self.pool = mp.Pool(n_cpu) else: self.pool = mp.Pool() self.cb_func = cb_func if self.cb_func: self.queue = mp.Queue() self.reader_p = mp.Process(target=self.reader_helper) self.reader_p.daemon = True self.reader_p.start() else: self.data = [] def reader_helper(self): while True: print 'size', self.queue.qsize() while not self.queue.empty(): msg = self.queue.get() if (msg == '_QUEUEDONE'): break else: args = msg[0] kwargs = msg[1] self.cb_func(*args, **kwargs) time.sleep(2) def list_cb(self, results): self.reporter.increment_report() self.data.append(results) def queue_cb(self, results): print 'put', results self.queue.put(results) self.reporter.increment_report() def addJob(self, argsTuple): if self.cb_func: self.pool.apply_async(self.func, argsTuple, callback=self.queue_cb) else: self.pool.apply_async(self.func, argsTuple, callback=self.list_cb) def finishJobs(self): self.pool.close() self.pool.join() if self.cb_func: self.queue.put('_QUEUEDONE') print 'Pool finished, waiting to process results' self.reader_p.join() self.reporter.done()
def learn_Karpathy(self): """Learn using updates like in the Karpathy algorithm.""" reporter = Reporter() config = self.master.config self.master.session.run([self.master.reset_accum_grads]) iteration = 0 episode_nr = 0 mean_rewards = [] while not self.master.stop_requested: # Keep executing episodes until the master requests a stop (e.g. using SIGINT) iteration += 1 trajectory = self.task_learner.get_trajectory() reward = sum(trajectory['reward']) action_taken = trajectory['action'] discounted_episode_rewards = discount_rewards( trajectory['reward'], config['gamma']) # standardize discounted_episode_rewards -= np.mean(discounted_episode_rewards) std = np.std(discounted_episode_rewards) std = std if std > 0 else 1 discounted_episode_rewards /= std feedback = discounted_episode_rewards results = self.master.session.run( [self.loss, self.add_accum_grad], feed_dict={ self.master.state: trajectory["state"], self.master.action_taken: action_taken, self.master.advantage: feedback }) results = self.master.session.run( [self.master.summary_op], feed_dict={ self.master.loss: results[0], self.master.reward: reward, self.master.episode_length: trajectory["steps"] }) self.writer.add_summary(results[0], iteration) self.writer.flush() self.master.session.run([self.master.apply_gradients]) self.master.session.run([self.master.reset_accum_grads])
def learn(self, env): reporter = Reporter() self.session.run([self.reset_accumulative_grads]) iteration = 0 # amount of batches processed episode_nr = 0 episode_lengths = np.zeros(self.config['batch_size']) episode_rewards = np.zeros(self.config['batch_size']) mean_rewards = [] while True: # Keep executing episodes trajectory = self.get_trajectory(env, self.config["episode_max_length"]) episode_rewards[episode_nr % self.config['batch_size']] = sum( trajectory['reward']) episode_lengths[episode_nr % self.config['batch_size']] = len( trajectory['reward']) episode_nr += 1 action_taken = (np.arange( self.nA) == trajectory['action'][:, None]).astype( np.float32) # one-hot encoding discounted_episode_rewards = discount_rewards( trajectory['reward'], self.config['gamma']) # standardize discounted_episode_rewards -= np.mean(discounted_episode_rewards) std = np.std(discounted_episode_rewards) std = std if std > 0 else 1 discounted_episode_rewards /= std feedback = np.reshape( np.repeat(discounted_episode_rewards, self.nA), (len(discounted_episode_rewards), self.nA)) self.session.run( [self.accumulate_grads], feed_dict={ self.state: trajectory["state"], self.action_taken: action_taken, self.feedback: feedback }) if episode_nr % self.config['batch_size'] == 0: # batch is done iteration += 1 self.session.run([self.apply_gradients]) self.session.run([self.reset_accumulative_grads]) reporter.print_iteration_stats(iteration, episode_rewards, episode_lengths, episode_nr) mean_rewards.append(episode_rewards.mean()) if episode_nr % self.config['draw_frequency'] == 0: reporter.draw_rewards(mean_rewards)
def learn(self): reporter = Reporter() gradient1 = np.zeros_like(self.w1) gradient2 = np.zeros_like(self.w2) rmsprop1 = np.zeros_like(self.w1) rmsprop2 = np.zeros_like(self.w2) iteration = 0 # amount of batches processed episode_nr = 0 episode_lengths = np.zeros(self.config['batch_size']) episode_rewards = np.zeros(self.config['batch_size']) mean_rewards = [] while True: # Keep executing episodes trajectory = self.get_trajectory(self.config["episode_max_length"]) episode_rewards[episode_nr % self.config['batch_size']] = sum(trajectory['reward']) episode_lengths[episode_nr % self.config['batch_size']] = len(trajectory['reward']) episode_nr += 1 action_taken = (np.arange(self.nA) == trajectory['action'][:, None]).astype(np.float32) # one-hot encoding epdlogp = action_taken - trajectory['prob'] # episode_states = np.vstack(encountered_states) discounted_episode_rewards = discount_rewards(trajectory['reward'], self.config['gamma']) # print(discounted_episode_rewards) # standardize discounted_episode_rewards -= np.mean(discounted_episode_rewards) discounted_episode_rewards /= np.std(discounted_episode_rewards) epdlogp *= np.reshape(np.repeat(discounted_episode_rewards, self.nA), (len(discounted_episode_rewards), self.nA)) change_w1, change_w2 = self.backward_step(trajectory['state'], trajectory['x1'], epdlogp) gradient1 += change_w1 gradient2 += change_w2 if episode_nr % self.config['batch_size'] == 0: # batch is done iteration += 1 rmsprop1 = self.config['decay_rate'] * rmsprop1 + (1 - self.config['decay_rate']) * gradient1**2 rmsprop2 = self.config['decay_rate'] * rmsprop2 + (1 - self.config['decay_rate']) * gradient2**2 self.w1 += self.config['learning_rate'] * gradient1 / (np.sqrt(rmsprop1) + 1e-5) self.w2 += self.config['learning_rate'] * gradient2 / (np.sqrt(rmsprop2) + 1e-5) gradient1 = np.zeros_like(self.w1) gradient2 = np.zeros_like(self.w2) reporter.print_iteration_stats(iteration, episode_rewards, episode_lengths, episode_nr) mean_rewards.append(episode_rewards.mean()) if episode_nr % self.config['draw_frequency'] == 0: reporter.draw_rewards(mean_rewards)
def test_reportRawData(self): #Precondicion: necesitamos una campaña con tweets asociados: campaign = '{"email":"*****@*****.**","hashtags": ["#test", "#mock"], "mentions": ["@testCampaign", "@mockOK"], "startDate":"28 11 2018 18:02:00", "endDate":"25 12 2018 19:26:22"}' fields = json.loads(campaign) manager.Manager().insertCampaign(fields) manager.Manager().insertCampaign(fields) manager.Manager().insertCampaign(fields) #Inicializamos los Tweets diccionarios: tweet1 = { "id_str": "123456", "user": { "name": "NASAOk", "id_str": "789456" }, "entities": { "hashtags": ["#mars", "#venus", "#earth"], "user_mentions": ["@NASA", "@planets"] }, "created_at": "Sun Mar 20 15:11:01 +0000 2018", } tweet2 = { "id_str": "112112", "user": { "name": "MauricioOK", "id_str": "451325" }, "entities": { "hashtags": ["#DonaldNoMeDejes"], "user_mentions": ["@donaldTrump", "@G20"] }, "created_at": "Sun Mar 20 21:08:01 +0000 2018", } #Para pasarlos a un objeto Tweet: T1=Tweet(tweet1) #Para pasar estos objetos Tweet a Json usando el metodo to_json del objeto Tweet: t1=T1.to_json() tweetsJson = json.dumps([tweet1, tweet2]) #Insertamos los tweets en la Campaign 3 manager.Manager().insertTweets(tweetsJson, 3) raw_data = Reporter.Reporter().reportRawData(3) print(raw_data)
def test_reuters_with_bayse(): reporter = Reporter(config['windows'], config['methods']) #categories = ['livestock', 'jobs'] #categories = ['crude', 'livestock', 'jobs', 'ship', 'corn', 'trade', 'interest'] #categories = ['crude', 'livestock', 'earn', 'acq', 'grain', 'wheat', 'money', 'jobs', 'ship', 'corn', 'trade', 'interest'] categories = [ 'crude', 'livestock', 'earn', 'jobs', 'ship', 'corn', 'trade', 'interest' ] #categories = ['acq', 'alum', 'barley', 'bop', 'carcass', 'castor-oil', 'cocoa', 'coconut', 'coconut-oil', 'coffee', 'copper', 'copra-cake', 'corn', 'cotton', 'cotton-oil', 'cpi', 'cpu', 'crude', 'dfl', 'dlr', 'dmk', 'earn', 'fuel', 'gas', 'gnp', 'gold', 'grain', 'groundnut', 'groundnut-oil', 'heat', 'hog', 'housing', 'income', 'instal-debt', 'interest', 'ipi', 'iron-steel', 'jet', 'jobs', 'l-cattle', 'lead', 'lei', 'lin-oil', 'livestock', 'lumber', 'meal-feed', 'money-fx', 'money-supply', 'naphtha', 'nat-gas', 'nickel', 'nkr', 'nzdlr', 'oat', 'oilseed', 'orange', 'palladium', 'palm-oil', 'palmkernel', 'pet-chem', 'platinum', 'potato', 'propane', 'rand', 'rape-oil', 'rapeseed', 'reserves', 'retail', 'rice', 'rubber', 'rye', 'ship', 'silver', 'sorghum', 'soy-meal', 'soy-oil', 'soybean', 'strategic-metal', 'sugar', 'sun-meal', 'sun-oil', 'sunseed', 'tea', 'tin', 'trade', 'veg-oil', 'wheat', 'wpi', 'yen', 'zinc'] for cat in categories: print '----', cat, '----' reporter.get_category_top(categories, cat, 'rw_oc', 3) reporter.get_category_top_tf(categories, cat, 'rw_oc', 3)
def main(argv): abcConf = Config.Config(argv) abcList = list() expT = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S").replace(" ", "").replace(":", "") for run in range(abcConf.RUN_TIME): abc = ABC.ABC(abcConf) abc.setExperimentID(run, expT) start_time = time.time() * 1000 abc.initial() abc.memorize_best_source() while(not(abc.stopping_condition())): abc.send_employed_bees() abc.calculate_probabilities() abc.send_onlooker_bees() abc.memorize_best_source() abc.send_scout_bees() abc.increase_cycle() abc.globalTime = time.time() * 1000 - start_time abcList.append(abc) Reporter(abcList)
def __init__(self): logging.basicConfig(filename="/tmp/VMClient.log", level=logging.DEBUG) self.get_apk_from_manager() self.trigger = Trigger.get_trigger_for(self.get_filter_tag(), self.get_package_name(), self.get_description()) self.reporter = Reporter.get_reporter_for(self.get_filter_tag(), self.get_package_name(), self.get_description()) self.analyzer = Analyzer.get_analyzer_for(self.get_filter_tag(), self.get_package_name(), self.get_description()) self.emulator = Emulator.get_emulator_for(self.get_filter_tag(), self.get_package_name(), self.get_description()) self.error_queue = multiprocessing.Queue() self.setup_device() logging.debug("Init successful") logging.debug("Trigger: " + repr(self.trigger)) logging.debug("Reporter: " + repr(self.reporter)) logging.debug("Analyzer: " + repr(self.analyzer)) logging.debug("Emulator: " + repr(self.emulator))
def __init__(self, env, resultInfo={}): self.nesting = 0 self.failures = [] self.childResults = [] self.resultType = 'Step' self.env = env self.settings = env.settings self.loglevel = Result.LOG_WARNING self.devoutput = self.settings['dev-output'] self.filetrack = self.settings['file-tracking'] if self.settings['debug']: self.loglevel = Result.LOG_DEBUG elif self.settings['verbose']: self.loglevel = Result.LOG_INFO elif self.settings['quiet']: self.loglevel = Result.LOG_QUIET self.chatter = not self.settings['no-chatter'] self.params = resultInfo.copy() if 'status' not in self.params: self.params['status'] = "Unexecuted" if 'exception' not in self.params: self.params['exception'] = False if 'Out' not in self.params: self.params['Out'] = sys.stdout self.actualout = self.params['Out'] if 'Err' not in self.params: self.params['Err'] = self.params['Out'] if 'Type' not in self.params: self.params['Type'] = '<<Type Unset>>' if 'Id' not in self.params: self.params['Id'] = '<<Step Id Unset>>' if self.chatter: self.reporter = Reporter(self.params['Out']) else: self.reporter = NonChattyReporter(self.params['Out']) OutputTee.OutputTee.startResult(self)
def test_reuters_with_graph_sim(): reporter = Reporter(config['windows'], config['methods']) #categories = ['crude', 'livestock', 'earn', 'jobs', 'ship', 'corn', 'trade', 'interest'] categories = ['livestock', 'jobs'] #categories = ['crude', 'livestock', 'jobs', 'ship', 'corn', 'trade', 'interest'] for cat in categories: print '----', cat, '----' reporter.get_category_graph(categories, cat, 'rw_oc', 2) if __name__ == '__main__': #test_reuters_with_bayse() #test_reuters_with_graph_sim() reporter = Reporter(config['windows'], config['methods']) reporter.report_for_file('./data/rusTextChiliGuake') reporter.save_report_detailed('./data/rusTextChiliGuake', './tes/diplom.csv') # #---- crude ---- #{'earn': 24, 'jobs': 0, 'livestock': 0, 'corn': 0, 'trade': 3, 'interest': 0, 'crude': 155, 'ship': 7} #82.0105820106 #{'earn': 27, 'jobs': 0, 'livestock': 0, 'corn': 0, 'trade': 3, 'interest': 0, 'crude': 153, 'ship': 6} #80.9523809524 #---- livestock ---- #{'earn': 0, 'jobs': 0, 'livestock': 16, 'corn': 6, 'trade': 1, 'interest': 1, 'crude': 0, 'ship': 0} #66.6666666667 #{'earn': 0, 'jobs': 0, 'livestock': 18, 'corn': 5, 'trade': 1, 'interest': 0, 'crude': 0, 'ship': 0} #75.0
#first create server socket sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind(('192.168.1.222', 1060)) overseerlogger.info("(+) Listening socket created.") #create 5 Listeners and start them for i in range(5): t = Listener(sock, i+1) t.start() overseerlogger.info("(+) Listener "+str(i+1)+" started.") ########Initialize Alert Queue alertQueue = Queue() ########initialize Reporter reporter = Reporter(alertQueue, 0) reporter.start() overseerlogger.info("(+) Reporter started.") ########Initialize dict to hold phrases of active sessions and their states sessionsDict = dict() #Initialize list to hold SessionHanlder objects sessionHandlers = list() ########Manage session loop while True: #connect to the database db = DBManager() #get all sessions data from DB sessions = db.getSessions() db.close()
emails = re.split('[; ,]', self.config.get("email", "{}_email".format(self.vo.lower()))) + \ re.split('[: ,]', self.config.get("email", "test_to")) TextUtils.sendEmail(([], emails), "{} Production Jobs Success Rate on the OSG Sites ({} - {})".format(self.vo, self.start_time, self.end_time), {"html": text}, ("Gratia Operation", "*****@*****.**"), "smtp.fnal.gov") os.unlink(fn) # Delete HTML file if __name__ == "__main__": opts, args = Reporter.parse_opts() if opts.debug: logging.basicConfig(filename='jobsuccessreport.log', level=logging.DEBUG) else: logging.basicConfig(filename='jobsuccessreport.log', level=logging.ERROR) logging.getLogger('elasticsearch.trace').addHandler(logging.StreamHandler()) try: config = Configuration.Configuration() config.configure(opts.config) r = JobSuccessRateReporter(config, opts.start, opts.end, opts.vo, opts.template, opts.is_test, opts.verbose, opts.no_email) r.generate() r.send_report() except Exception as e:
def main(): reporter = Reporter(config()) reporter.create_index()
"{} Jobs with Low Efficiency ({}) on the OSG Sites ({} - {})".format( self.vo, self.eff_limit, self.start_time, self.end_time), {"html": text}, ("Gratia Operation", "*****@*****.**"), "smtp.fnal.gov") if self.verbose: os.remove(fn) print "Report sent" if __name__ == "__main__": opts, args = Reporter.parse_opts() try: # Set up the configuration config = Configuration.Configuration() config.configure(opts.config) # Grab VO vo = opts.vo # Grab the limits eff = config.config.get(opts.vo.lower(), "efficiency") min_hours = config.config.get(opts.vo.lower(), "min_hours") # Create an Efficiency object, create a report for the VO, and send it e = Efficiency(config, opts.start, opts.end, vo, opts.verbose, int(min_hours), float(eff), opts.is_test) # Run our elasticsearch query, get results as CSV resultfile = e.query_to_csv()
if not allGood: print("exiting now") exit() thisFilesPath = os.path.realpath(__file__) thisFilesDir = os.path.abspath(os.path.join(thisFilesPath, os.path.pardir)) Util.MAIN_FOLDER = thisFilesDir os.chdir(Util.MAIN_FOLDER) print("Starting...") successfully = False try: from Reporter import Reporter r = Reporter() if not r.load(): raise Exception("could not load document") if r.appendNewEntry(): r.checkAlarms() if r.save(): successfully = True if not Config.RSS_AND_HTML_BASE_URL: print("RSS_AND_HTML_BASE_URL not configured in Config.py\nWon't do conversion to rss or html") else: from Transformer import Transformer urlbase = Config.RSS_AND_HTML_BASE_URL if not urlbase.endswith("/"): urlbase = urlbase + "/" x = Transformer(copyToDir=Config.COPY_RSS_AND_HTML_HERE)
def setUp(self): self.r1 = Reporter()
def error(self, m, pos=NoPosition): Reporter.error(self, m, pos) print(pos.toDisplayString('Error: ' + m))
def warning(self, m, pos=NoPosition): Reporter.warning(self, m, pos) print(pos.toDisplayString('Warning: ' + m))
class MainProgram: def __init__(self, configFileName): self.start = time.time() configLoader = ConfigsLoader(configFileName) self.configs = configLoader.get_configs() self.textProcessor = TextProcessor(self.configs) self.trainerWrapper = TrainerWrapper(self.configs) self.reporter = Reporter(self.configs) self.text_cleaner = TextCleaner(self.configs) self.read_config_variables() def read_config_variables(self): self.col_names = self.configs.get("DATA", "col_names") self.limit_data = self.configs.getboolean("DEBUG", "limit_data") self.data_limit = self.configs.getint("DEBUG", "data_limit") self.current_dataset = self.configs.get("DATA", "current_dataset") self.target_column = self.configs.get("DATA", "target_column") self.do_preprocessing = self.configs.getboolean( "PREPROCESSING", "do_preprocessing") self.loss_function_name = self.configs.get("TRAINING", "loss_function") self.balanced_loss = self.configs.getboolean("TRAINING", "balanced_loss") self.chars_file_name = self.configs.get("DATA", "chars_csv_file_name") self.char_df_colNames = [self.configs.get("DATA", "char_df_column")] self.device_id = self.configs.get("TRAINING", "device_id") self.current_network = self.configs.get("ARCHITECTURE", "current_network") self.trainer_logs_out = self.configs.get("TRAINING", "trainer_logs") self.checkpoint_file = self.configs.get("DATA", "checkpoint_file") self.model_file = self.trainer_logs_out + self.checkpoint_file + 'ckpt.t7' def load_image_dataset(self): self.chars_df = load_csv_dataset(self.chars_file_name, ",", self.char_df_colNames) factory = ObjectFactory() factory.register_builder("Wikipedia_Title_Dataset", WikipediaTitleDatasetLoader) factory.register_builder("Poetry_Dataset", PoetryDatasetLoader) factory.register_builder("Dialect_Dataset", DialectDatasetLoader) data_loader = factory.create(self.current_dataset, **{'configParser': self.configs}) data_loader.read_dataset_from_file() data_loader.preprocess_dataset() self.training_data_raw, self.testing_data_raw = data_loader.get_data() if self.limit_data: self.training_data_raw = self.training_data_raw.groupby( self.target_column, group_keys=False).apply(lambda x: x.sample( int( np.rint(self.data_limit * len(x) / len( self.training_data_raw))))).sample( frac=1).reset_index(drop=True) if self.do_preprocessing: self.training_data_raw = self.text_cleaner.clean_text( self.training_data_raw) self.testing_data_raw = self.text_cleaner.clean_text( self.testing_data_raw) self.label_list = sorted( list(set(self.training_data_raw[self.target_column]))) self.counts = construct_count_array( self.training_data_raw[self.target_column]) self.labels = self.training_data_raw[self.target_column] self.number_of_classes = np.size(np.unique(self.labels)) self.X_train, self.y_train = self.textProcessor.split_data_x_y( self.training_data_raw) self.X_test, self.y_test = self.textProcessor.split_data_x_y( self.testing_data_raw) self.training_data = self.textProcessor.numpy_pair_to_pytorch_dataset( self.X_train, self.y_train, self.chars_df, self.X_train) self.testing_data = self.textProcessor.numpy_pair_to_pytorch_dataset( self.X_test, self.y_test, self.chars_df, self.X_train) print(self.label_list) return self.training_data, self.testing_data def build_model(self, out_dim): factory = ObjectFactory() factory.register_builder("DeepCLCNN", DeepCLCNN) factory.register_builder("DeepCLCNNwithWL", DeepCLCNNwithWL) factory.register_builder("DeepNoPoolingCLCNN", DeepNoPoolingCLCNN) factory.register_builder("DeepNoPoolingCLCNNwithWT", DeepNoPoolingCLCNNwithWT) factory.register_builder("DeepCLCNNNoEncoder", DeepCLCNNNoEncoder) factory.register_builder("DeepCLCNNwithWLNoEncoder", DeepCLCNNwithWLNoEncoder) factory.register_builder("DeepNoPoolingCLCNNNoEncoder", DeepNoPoolingCLCNNNoEncoder) factory.register_builder("DeepNoPoolingCLCNNwithWTNoEncoder", DeepNoPoolingCLCNNwithWTNoEncoder) factory.register_builder("RNNCharacter", RNNCharacter) model = factory.create( self.current_network, **{ 'configParser': self.configs, 'out_dim': out_dim }) return model def run(self): training_data, testing_data = self.load_image_dataset() self.model = self.build_model(len(self.label_list)) self.model.cuda() self.trainerWrapper.train_model(self.model, training_data, testing_data, self.number_of_classes, self.y_train, self.counts) self.end = time.time() print("Training took %s seconds!", str(self.end - self.start)) print("Now loading the model with the best performance") self.model = self.build_model(len(self.label_list)).cuda() #self.model = load_model(self.model_file, DistributedDataParallel(self.model)) self.model = load_model(self.model_file, self.model) print("Now testing no sliding window ....") self.start = time.time() self.reporter.test_model(self.model, testing_data, self.number_of_classes) self.end = time.time() print("Testing no sliding window took %s seconds!", str(self.end - self.start)) print("Now testing Sliding Window...") print("Printing the class distripution in the whole data") self.reporter.print_category_distribution( self.training_data_raw[self.target_column]) self.reporter.write_results()
def __init__(self): Reporter.__init__(self)
def info(self, m, pos=NoPosition): Reporter.info(self, m, pos) print(pos.toDisplayString('Info: ' + m))