def main(argv = sys.argv[1:]): args = parser.parse_args(argv) if args.statFiles == None: print 'Filenames missing.' sys.exit(-1) aucData = [] aucLabels = [] for statFile in args.statFiles: s = Statistics() s.load(statFile) avgRecall = [float(x)/s.problems for x in s.recallData] figure('Recall') plot(range(s.cutOff),avgRecall,label=statFile) legend(loc='lower right') ylabel('Average Recall') xlabel('Highest ranked premises') axis([0,s.cutOff,0.0,1.0]) figure('100%Recall') plot(range(s.cutOff),s.recall100Data,label=statFile) legend(loc='lower right') ylabel('100%Recall') xlabel('Highest ranked premises') axis([0,s.cutOff,0,s.problems]) aucData.append(s.aucData) aucLabels.append(statFile) figure('AUC Histogram') hist(aucData,bins=args.bins,label=aucLabels,histtype='bar') legend(loc='upper left') ylabel('Problems') xlabel('AUC') show()
def calc_statistics(args, app=None): if not app: app = create_app(config_override=CONFIG) with app.app_context(): from stats import Statistics statistics = Statistics() statistics.calc_all() db.session.commit()
def diff(db, trace, **kwargs): doopconn = doop.Connector(db) probe = gxl.Probe(trace) # Get statistics from a doop static analysis static = Statistics.from_doop(doopconn) # Get dynamic analysis statistics from a gxl trace dynamic = Statistics.from_trace(probe, doopconn) # Compute dynamic \ static diff = Statistics.difference(dynamic, static) # Construct a transformation chain diffchain = [(diff, 'Default')] def add_filter(refinement, msg): last = diffchain[-1][0] diffchain.append((refinement(last), msg)) refinement.report() # filter false positive calls to <clinit> add_filter(clinit.ClassInitRefinement(doopconn), '<clinit>') # filter native methods add_filter(native.Refinement(doopconn), 'No native') # filter calls to java.lang.ClassLoader methods add_filter(classloader.ClassLoaderRefinement(doopconn), 'No ClassLoader methods') # Compute (dynamic \ static) \ synthetic classpath = kwargs.get('cp', None) if classpath: add_filter(synthetic.Refinement(classpath), 'No synthetic') # Statically unknown classes / methods add_filter(reflect.NoFactsRefinement(doopconn), 'No facts') # Prune call-edges with unreachable origin add_filter(unreachable.Refinement(doopconn), 'Unreachable origin') nSteps = len(diffchain) steps = range(1, 1 + nSteps) for ((diff, msg), step) in zip(diffchain, steps): print "\n--- {0}. Step {1}/{2} ---\n".format(msg, step, nSteps) display(static, dynamic, diff) for tp in ('a2a', 'a2l', 'l2a', 'l2l'): if tp in kwargs.get('toprint', []): print '\nPrinting {0} edges...'.format(tp) for (s,t) in getattr(diffchain[-1][0], tp): print s, '===>', t return static, dynamic, diffchain
def __init__(self): self.__match_maker = MatchMaker() self.__stats = Statistics() self.__app = QtWidgets.QApplication(sys.argv) self.__MainWindow = QtWidgets.QMainWindow() super().__init__() super().setupUi(self.__MainWindow) self.__init_logic() self.__display_round() self.__display_stats()
def validate(model, val_iter): """Validate with mini-batches.""" model.eval() batch_stats = Statistics() with torch.no_grad(): for batch in val_iter: sents = batch.sent _, stats = model(sents) batch_stats.update(stats) torch.cuda.empty_cache() return batch_stats
def initPages(self): self.mode_stack = QtGui.QStackedWidget() main_menu = MainMenu(self) #0 self.stats = Statistics(self) #1 self.trainer = Trainer(self, self.stats) #2 self.generator = Generator(self) #3 self.mode_stack.addWidget(main_menu) self.mode_stack.addWidget(self.stats) self.mode_stack.addWidget(self.trainer) self.mode_stack.addWidget(self.generator) self.setCentralWidget(self.mode_stack)
def print_stat_comparison(self): real_orders, _, _ = DataLoader.load_split_data(self.config.real_root, self.sampling_window_start_time, self.sampling_window_end_time, self.config.product) sim_orders = self.sim_analysis.all_sims[0][0].compute() real_stats = Statistics.get_order_stats(real_orders) sim_stats = Statistics.get_order_stats(sim_orders) print_str = "" for k in real_stats.keys(): print_str += k + "\t\t\t\tReal: " + str(real_stats[k]) + "\t\t\tSim: " + str(sim_stats[k]) + "\n" print(print_str)
def test_from_many_sources(self): # create one mutation which is present in multiple sources m = models.Mutation() metadata_1 = models.InheritedMutation(mutation=m) metadata_2 = models.MC3Mutation(mutation=m) db.session.add_all([metadata_1, metadata_2]) from stats import Statistics statistics = Statistics() in_many_sources = statistics.from_more_than_one_source() assert in_many_sources == 1
def get_relative_prices(trades_df: dd, other_df: dd) -> dd: price_over_time: dd = Statistics().get_price_over_time( trades_df).groupby( ['time'])['most_recent_trade_price'].mean().to_frame() other_df = DataUtils().fuzzy_join(other_df, price_over_time, on='time') relative_prices = other_df['relative_price'] relative_prices = DataUtils().remove_tails(relative_prices, 3) return relative_prices
def test_hurst_windowed(self): day = 17 product = "LTC-USD" for i in range(0, 1): # day += 1 month = 5 st = datetime.datetime(2018, month, day, 0, 0, 0) et = datetime.datetime(2018, month, day, 23, 59, 59) _, trades, _ = DataLoader.load_split_data("/Users/jamesprince/project-data/data/consolidated-feed/" + product + "/", st, et, product) window_minutes = 10 step_minutes = 10 times, hurst_exps = Statistics.get_hurst_exponent_over_time(trades, st, et, step_minutes, window_minutes) Statistics.plot_metric_daily(times, hurst_exps, product, st, step_minutes, window_minutes, "Hurst Exponent")
def start_simulation_sim_time(no_of_jobs_server_1, no_of_jobs_server_2, arrival_rate, job_distribution, sim_time): list_of_servers = [] for _ in range(0,2): # Create 2 servers scheduler_i = FIFO() job_size_i = Expo(job_distribution) server_i = Server(job_size_i, scheduler_i) list_of_servers.append(server_i) policy_i = ShortestQueue() #Join shortest queue policy dispatcher = Dispatcher(policy_i, list_of_servers) #Create a dispatcher with JSQ policy statistics = Statistics() world = Global(statistics) p_arrivals = PoissonArrival(float(arrival_rate)) #For each job in server 1 we: #Create a job with arrival at time 0, then increase # of jobs in the server. #We get time the job enters service, either immediately or after the total processing done. #Set the enter service and service time needed. #Next calculate departure time and set it, then we can schedule the departure. server_1_processing_time = 0 #Forloops used to create a given # of jobs before starting sim for job in range(0, int(no_of_jobs_server_1)): job = Job(0) #All arrive at time 0 list_of_servers[0]._total_jobs +=1 enter_service = max(list_of_servers[0]._total_processing_time, 0) #Takes processing time (time when server is idle) and the time of arrival. Max of those is enter service for this job job.set_enter_service(enter_service) #Setting the enter service, first is 0, next is 0+service_1, next is 0+service_1 + service_2.. ... job.set_service(list_of_servers[0].get_service_time()) #Generates service time AND increases processing time for server departure_time = job._enter_service + job._service_time # Enter service will be correct because of processing time of server bookkeeping job.set_departure(departure_time) #Set the departure time as enter service+service time. (Its a timeperiod on the timeline) world.schedule_event(list_of_servers[0].departure, job._departure_time, [job, world]) #Schedule this departure to the world queue for job in range(0, int(no_of_jobs_server_2)): job = Job(0) list_of_servers[1]._total_jobs +=1 enter_service = max(list_of_servers[1]._total_processing_time, 0) job.set_enter_service(enter_service) job.set_service(list_of_servers[1].get_service_time()) departure_time = job._enter_service + job._service_time job.set_departure(departure_time) world.schedule_event(list_of_servers[1].departure, job._departure_time, [job, world]) initial_arrival = random.expovariate(p_arrivals._rate) params = [dispatcher, world] world.schedule_event(p_arrivals.generate_arrival, initial_arrival, params) #Schedule first arrival to start chain world.number_of_arr_dep = 0 #resetting the number of events before we start # Now we need to schedule the initial arrivals to start the chain of events. for x in range(1, 11): # Now that each dispatcher has an arrival, we can start looping through events while world.next_event() <= float(sim_time)*(x*0.1): # while the virtual time of next event is less than our simulation time.. world.process_event() # We take the event and process it (running the function(s)) print("{}%".format(x*10)) #for loop to step between while loops (every 10%)while world.next total_no_jobs = world._stats.number_of_jobs print(total_no_jobs) world._stats.print_stats()
def main(): """ The scanner's entry point. """ stats = Statistics() args = parse_cmd_args() # Create and set the given directories. if args.tor_dir and not os.path.exists(args.tor_dir): os.makedirs(args.tor_dir) logging.getLogger("stem").setLevel( logging.__dict__[args.verbosity.upper()]) log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s" logging.basicConfig(format=log_format, level=logging.__dict__[args.verbosity.upper()], filename=args.logfile) log.debug("Command line arguments: %s" % str(args)) socks_port, control_port = bootstrap_tor(args) controller = Controller.from_port(port=control_port) stem.connection.authenticate(controller) # Redirect Tor's logging to work around the following problem: # https://bugs.torproject.org/9862 log.debug("Redirecting Tor's logging to /dev/null.") controller.set_conf("Log", "err file /dev/null") # We already have the current consensus, so we don't need additional # descriptors or the streams fetching them. controller.set_conf("FetchServerDescriptors", "0") cached_consensus_path = os.path.join(args.tor_dir, "cached-consensus") if args.first_hop and (not util.relay_in_consensus(args.first_hop, cached_consensus_path)): log.critical("Given first hop \"%s\" not found in consensus. Is it" " offline?" % args.first_hop) return 1 for module_name in args.module: if args.analysis_dir is not None: datestr = time.strftime("%Y-%m-%d_%H:%M:%S%z") + "_" + module_name util.analysis_dir = os.path.join(args.analysis_dir, datestr) try: run_module(module_name, args, controller, socks_port, stats) except error.ExitSelectionError as err: log.error("Failed to run because : %s" % err) return 0
def main(): input_file = sys.argv[1] output_file = sys.argv[2] total_start = time.clock() #contours = np.load("contours.npy") contours = FloodEdgeDetector.find_contours(input_file) print("Contours Time", time.clock() - total_start) lines = Statistics.evaluate_contours(output_file, contours) print("Overall Time", time.clock() - total_start) ImageUtilities.show_contours_in_image(input_file, contours, lines)
def generateChart(self): self.graphView.setScrollView(self.scrollView) stat = Statistics(self.startDate, self.endDate) if self.reportType == "tasks": self.graphView.setData(stat.countTasks(), self.reportType) elif self.reportType == "projects": self.graphView.setData(stat.countProjects(), self.reportType) elif self.reportType == "slacking": self.graphView.setData(stat.countSlacking(), self.reportType) self.graphView.setScale(stat.maxValue) self.lblWorkTotal.setStringValue_(secToTimeStr(stat.totalWork)) self.lblAvgWork.setStringValue_(secToTimeStr(stat.avgWork)) self.lblSlackTotal.setStringValue_(secToTimeStr(stat.totalSlacking)) self.lblAvgSlack.setStringValue_(secToTimeStr(stat.avgSlacking)) self.graphView.setNeedsDisplay_(True)
def init_tlb(self, tlb_lst, tlb_len, tlb_stats, yconfig, data=True): for l in range(tlb_len): if yconfig[l]['private'] or data: if yconfig[l]['type'] == 'fully-associative': tlb_lst.append(Tlb(Structure.FULLY_ASSOCIATIVE, yconfig[l]['entry'])) elif yconfig[l]['type'] == '2-way': tlb_lst.append(Tlb(Structure.SET2_ASSOCIATIVE, yconfig[l]['entry'])) elif yconfig[l]['type'] == '4-way': tlb_lst.append(Tlb(Structure.SET4_ASSOCIATIVE, yconfig[l]['entry'])) else: raise NotImplementedError("unimplemented tlb type") tlb_stats.append(Statistics()) tlb_lst[-1].set_stats(tlb_stats[-1]) else: # shared tlb -> let the data and instruction tlbs have # the same object. tlb_lst.append(self.dtlb[l]) tlb_stats.append(self.dtlb_stats[l]) tlb_stats.append(Statistics())
def test_lypaunov_windowed(self): st = datetime.datetime(2018, 5, 17, 1, 0, 0) et = datetime.datetime(2018, 5, 17, 23, 0, 0) conf = configparser.ConfigParser() conf.read("../config/backtest.ini") config = BacktestConfig(conf) ob_seq, ob_state = reconstruct_orderbook(config, st, logging.getLogger("test")) orderbook_evo = OrderBookEvolutor(ob_state, st, ob_seq) feed_df = DataLoader.load_feed(self.root, st, et, "LTC-USD") evo = orderbook_evo.evolve_orderbook_discrete(feed_df, 1) window_minutes = 30 step_minutes = 5 num_samples = int((et - st).total_seconds() / (step_minutes * 60)) times = [] lyap_exps = [] for i in range(0, num_samples): window_st = st + datetime.timedelta(seconds=i * step_minutes * 60) window_et = window_st + datetime.timedelta(seconds=window_minutes * 60) evo_filt = evo[evo['time'] > window_st] evo_filt = evo_filt[evo_filt['time'] < window_et] midprices = evo_filt['midprice'].dropna() prices = np.asarray(midprices, dtype=np.float32) print(prices) res = nolds.lyap_e(prices) print(res) times.append(window_st) lyap_exps.append(res[0]) Statistics.plot_lyapunov_exponent(times, lyap_exps, "LTC-USD", st, step_minutes, window_minutes)
def initPages(self): self.mode_stack = QtGui.QStackedWidget() main_menu = MainMenu(self) #0 self.stats = Statistics(self) #1 self.trainer = Trainer(self,self.stats) #2 self.generator = Generator(self) #3 self.mode_stack.addWidget(main_menu) self.mode_stack.addWidget(self.stats) self.mode_stack.addWidget(self.trainer) self.mode_stack.addWidget(self.generator) self.setCentralWidget(self.mode_stack)
def reset_world(first_arrival): nonlocal list_of_servers, dispatcher, statistics, world # if we want to modify variable from closure, must place as nonlocal list_of_servers.clear() #clear the 2 servers for _ in range(0,2): # Create 2 servers scheduler_i = FIFO() job_size_i = Expo(job_distribution) # use job_distr from closure server_i = Server(job_size_i, scheduler_i) # create a new server to place in list list_of_servers.append(server_i) dispatcher = Dispatcher(policy_i, list_of_servers) # resetting the dispatcher with new servers statistics = Statistics() world = Global(statistics) set_up_servers() params = [dispatcher, world] world.schedule_event(p_arrivals.generate_arrival, first_arrival, params) #Schedule first arrival to start chain
def main(): """ The scanner's entry point. """ stats = Statistics() args = parse_cmd_args() # Create and set the given directories. if args.tor_dir and not os.path.exists(args.tor_dir): os.makedirs(args.tor_dir) if args.analysis_dir and not os.path.exists(args.analysis_dir): os.makedirs(args.analysis_dir) util.analysis_dir = args.analysis_dir logger.setLevel(logging.__dict__[args.verbosity.upper()]) logger.debug("Command line arguments: %s" % str(args)) socks_port, control_port = bootstrap_tor(args) controller = Controller.from_port(port=control_port) stem.connection.authenticate(controller) # Redirect Tor's logging to work around the following problem: # https://bugs.torproject.org/9862 logger.debug("Redirecting Tor's logging to /dev/null.") controller.set_conf("Log", "err file /dev/null") # We already have the current consensus, so we don't need additional # descriptors or the streams fetching them. controller.set_conf("FetchServerDescriptors", "0") cached_consensus_path = os.path.join(args.tor_dir, "cached-consensus") if args.first_hop and (not util.relay_in_consensus(args.first_hop, cached_consensus_path)): raise error.PathSelectionError("Given first hop \"%s\" not found in " "consensus. Is it offline?" % args.first_hop) for module_name in args.module: try: run_module(module_name, args, controller, socks_port, stats) except error.ExitSelectionError as err: logger.error("failed to run because : %s" % err) return 0
def _forward(self, sents): src = sents[:-1] tgt = sents[1:] tgt = tgt.view(-1) dec_out, _ = self.decoder(src) dec_out = self.dropout(dec_out) logit = self.generator(dec_out.view(-1, dec_out.size(2))) recon_loss = self.criterion(logit, tgt).sum() n_correct, n_words = self._correct(logit, tgt) stats = Statistics( loss=recon_loss.item(), n_correct=n_correct, n_words=n_words, n_sents=sents.size(1), ) return recon_loss, stats
def _decode(self, sents, mu, logvar, beta): src = sents[:-1] tgt = sents[1:] tgt = tgt.view(-1) recon_loss, n_correct, n_words = 0.0, 0, 0 dec_outs = 0.0 for _ in range(self.num_z_samples): z = self._reparameterize(mu, logvar) h = self.z2h(z) dec_state = self._build_dec_state(h) dec_out, _ = self.decoder(src, dec_state, z) dec_out = self.dropout(dec_out) if self.use_avg: dec_outs += dec_out else: logit = self.generator(dec_out.view(-1, dec_out.size(2))) recon_loss += self.criterion(logit, tgt).sum() n_correct_, n_words_ = self._correct(logit, tgt) n_correct += n_correct_ n_words += n_words_ if self.use_avg: dec_outs /= self.num_z_samples logit = self.generator(dec_outs.view(-1, dec_outs.size(2))) recon_loss = self.criterion(logit, tgt).sum() n_correct, n_words = self._correct(logit, tgt) else: recon_loss /= self.num_z_samples n_correct /= self.num_z_samples n_words /= self.num_z_samples kl_loss = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) loss = recon_loss + (beta * kl_loss) stats = Statistics( loss.item(), recon_loss.item(), kl_loss.item(), n_correct, n_words, sents.size(1), ) return loss, stats
def train(model, optimizer, train_iter, epoch, args): """Train with mini-batches.""" model.train() total_stats = Statistics() batch_stats = Statistics() num_batches = len(train_iter) for i, batch in enumerate(train_iter): if args.warmup > 0: args.beta = min(1, args.beta + 1.0 / (args.warmup * num_batches)) sents = batch.sent loss, stats = model(sents, args.beta) optimizer.zero_grad() loss.backward() utils.clip_grad_norm(optimizer, args) optimizer.step() total_stats.update(stats) batch_stats.update(stats) batch_stats = report_batch(batch_stats, epoch, i, num_batches, args) torch.cuda.empty_cache() return total_stats
def start_simulation(no_of_servers, server_scheduler, no_of_dispatchers, dispatcher_policy, arrival_rate, job_distribution, sim_time): list_of_servers = [] list_of_dispatchers = [] for _ in range(0, int(no_of_servers)): scheduler_i = FIFO() if server_scheduler == 1 else PS() job_size_i = Expo(job_distribution) server_i = Server(job_size_i, scheduler_i) list_of_servers.append(server_i) for _ in range(0, int(no_of_dispatchers)): policy_i = RND() if dispatcher_policy == 1 else ShortestQueue() dispatcher_i = Dispatcher(policy_i, list_of_servers) list_of_dispatchers.append(dispatcher_i) statistics = Statistics() world = Global(statistics) p_arrivals = PoissonArrival(float(arrival_rate)/len(list_of_dispatchers)) # Loop to generate first arrivals for each dispatcher for d in list_of_dispatchers: arrival = random.expovariate(p_arrivals._rate) params = [d, world] world.schedule_event(p_arrivals.generate_arrival, arrival, params) # Schedule the first arrivals for each dispatcher # Now that each dispatcher has an arrival, we can start looping through events for x in range(1, 101): # Now that each dispatcher has an arrival, we can start looping through events while world.next_event() <= float(sim_time)*(x*0.01): # while the virtual time of next event is less than our simulation time.. world.process_event() # We take the event and process it (running the function(s)) world._stats.write_to_file_jobs() print("{}%".format(x)) #for loop to step between while loops (every 10%)while world.next total_no_jobs = world._stats.number_of_jobs print(total_no_jobs) world._stats.print_stats() ## Should add an interactive stage here to see if user wants to save data save_stats = input("Do you want to save these stats to file Y/N?").upper() if save_stats == 'Y': file_name = input("What name for the file? > ") print("Thank you, your stats will be saved in the Simulation_results directory.") world._stats.write_to_file_stats(file_name)
def report_batch(batch_stats, epoch, step, num_batches, args): """Report batch statistics.""" if step % args.report_every == -1 % args.report_every: r = batch_stats t = r.elapsed_time() logger.info( "Epoch %3d | %4d/%4d batches | acc %5.2f | " "nll %8.2f | kl %6.2f | ppl %8.2f | " "beta %4.2f | %.0f tok/s", epoch, step + 1, num_batches, r.accuracy(), r.nll(), r.kl(), r.ppl(), args.beta, r.n_words / (t + 1e-5), ) sys.stdout.flush() batch_stats = Statistics() return batch_stats
def report_iw_nll(model, test_iter, n_iw_iter, n_iw_samples): """Calculate the importance-weighted estimate of NLL.""" model.eval() loss, n_sents, n_words = 0.0, 0, 0 with torch.no_grad(): for batch in tqdm.tqdm(test_iter, total=len(test_iter)): sents = batch.sent n_sents += sents.size(1) for i in range(sents.size(1)): sent = sents[:, i] # get one sentence sent = sent.masked_select(sent.ne(model.padding_idx)) # trim pad token n_words += sent.size(0) - 1 # skip start symbol sent = sent.unsqueeze(1) logw = [] for _ in range(n_iw_iter): logw.extend(model.estimate_log_prob(sent, n_iw_samples)) logw = torch.cat(logw) logp = torch.logsumexp(logw, dim=-1) - math.log(len(logw)) loss += logp.item() torch.cuda.empty_cache() return Statistics(loss=-loss, n_words=n_words, n_sents=n_sents)
def main(): """ The scanner's entry point. """ stats = Statistics() args = parse_cmd_args() logger.setLevel(logging.__dict__[args.verbosity.upper()]) logger.debug("Command line arguments: %s" % str(args)) bootstrap_tor(args) controller = Controller.from_port(port=45679) stem.connection.authenticate(controller) # Redirect Tor's logging to work around the following problem: # https://bugs.torproject.org/9862 logger.debug("Redirecting Tor's logging to /dev/null.") controller.set_conf("Log", "err file /dev/null") # We already have the current consensus, so we don't need additional # descriptors or the streams fetching them. controller.set_conf("FetchServerDescriptors", "0") if args.first_hop and \ (not util.relay_in_consensus(args.first_hop, util.get_consensus_path(args))): raise error.PathSelectionError("Given first hop \"%s\" not found in " "consensus. Is it offline?" % args.first_hop) for module_name in args.module: run_module(module_name, args, controller, stats) return 0
def esa_3_server(arr_rate, job_through, no_servers, file_name): list_of_servers = init_servers(no_servers, 1, 1) list_of_dispatchers = init_dispatchers(1, 2, list_of_servers) statistics = Statistics() world = Global(statistics) p_arrivals = PoissonArrival(float(arr_rate)) init_first_jobs(world, list_of_dispatchers, p_arrivals) #Create the first arrival to initiate loop #Main loop while world.next_event( ): # while the virtual time of next event is less than our simulation time.. if world._stats.number_of_jobs > job_through: # If we've reached the desired # of jobs data = world._stats.get_mean_sd_sojourn() #format of 'mean,sd' with open( './Simulation_results/esa_' + str(no_servers) + '_subfile_' + str(file_name) + '_server_test2.txt', 'a') as myfile: myfile.write(data + "\n") break world.process_event( ) # We take the event and process it (running the function(s))
def test_interactions(self): from models import Protein, Site, Kinase, KinaseGroup p1 = Protein(sites=[ Site(), Site(kinases=[Kinase()], kinase_groups=[KinaseGroup()]) ]) db.session.add(p1) p2 = Protein(sites=[Site(kinases=[Kinase()])]) db.session.add(p2) u_all_interactions = 0 u_kinases_covered = set() u_kinase_groups_covered = set() u_proteins_covered = set() for protein in models.Protein.query.all(): for site in protein.sites: kinases = site.kinases kinase_groups = site.kinase_groups u_all_interactions += len(kinases) + len(kinase_groups) u_kinases_covered.update(kinases) u_kinase_groups_covered.update(kinase_groups) if kinases or kinase_groups: u_proteins_covered.add(protein) from stats import Statistics statistics = Statistics() all_interactions = statistics.interactions() kinases_covered = statistics.kinases_covered() kinase_groups_covered = statistics.kinase_groups_covered() proteins_covered = statistics.proteins_covered() assert all_interactions == u_all_interactions assert kinases_covered == len(u_kinases_covered) assert kinase_groups_covered == len(u_kinase_groups_covered) assert proteins_covered == len(u_proteins_covered)
"KeyCode":("character(kind=c_char)", "c_char"), "KeySym":("integer(c_long)", "c_long"), # enum GWin32OSType "GWin32OSType":("integer(c_int)", "c_int") } # Two words types: TYPES2_DICT = { "long double": ("real(c_long_double)", "c_long_double"), "unsigned long":("integer(c_long)", "c_long"), "unsigned short":("integer(c_short)", "c_short"), "unsigned int":("integer(c_int)", "c_int") } # An instance of the Statistics class: my_stats = Statistics() # An instance of the Errors class: my_errors = Errors() #************************************************************************* # Pass 1: scan all header files to find all enum types, all pointers to # functions (funptr) and add derived GTK types #************************************************************************* print("\033[1m Pass 1: looking for enumerators, funptr and derived types...\033[0m") gtk_types = [] # These lists will be used by the iso_c_binding() function: gtk_enums = [] gtk_funptr = []
def mash(argv = sys.argv[1:]): # Initializing command-line arguments args = parser.parse_args(argv) # Set up logging logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', datefmt='%d-%m %H:%M:%S', filename=args.log, filemode='w') logger = logging.getLogger('main.py') #""" # remove old handler for tester #logger.root.handlers[0].stream.close() logger.root.removeHandler(logger.root.handlers[0]) file_handler = logging.FileHandler(args.log) file_handler.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') file_handler.setFormatter(formatter) logger.root.addHandler(file_handler) #""" if args.quiet: logger.setLevel(logging.WARNING) #console.setLevel(logging.WARNING) else: console = logging.StreamHandler(sys.stdout) console.setLevel(logging.INFO) formatter = logging.Formatter('# %(message)s') console.setFormatter(formatter) logging.getLogger('').addHandler(console) if not os.path.exists(args.outputDir): os.makedirs(args.outputDir) logger.info('Using the following settings: %s',args) # Pick algorithm if args.nb: logger.info('Using sparse Naive Bayes for learning.') model = sparseNBClassifier(args.NBDefaultPriorWeight,args.NBPosWeight,args.NBDefVal) elif args.snow: logger.info('Using naive bayes (SNoW) for learning.') model = SNoW() elif args.predef: logger.info('Using predefined predictions.') model = Predefined(args.predef) else: logger.info('No algorithm specified. Using sparse Naive Bayes.') model = sparseNBClassifier(args.NBDefaultPriorWeight,args.NBPosWeight,args.NBDefVal) # Initializing model if args.init: logger.info('Initializing Model.') startTime = time() # Load all data dicts = Dictionaries() dicts.init_all(args) # Create Model trainData = dicts.featureDict.keys() model.initializeModel(trainData,dicts) if args.learnTheories: depFile = os.path.join(args.inputDir,args.depFile) theoryModels = TheoryModels(args.theoryDefValPos,args.theoryDefValNeg,args.theoryPosWeight) theoryModels.init(depFile,dicts) theoryModels.save(args.theoryFile) model.save(args.modelFile) dicts.save(args.dictsFile) logger.info('All Done. %s seconds needed.',round(time()-startTime,2)) return 0 # Create predictions and/or update model else: lineCounter = 1 statementCounter = 1 computeStats = False dicts = Dictionaries() theoryModels = TheoryModels(args.theoryDefValPos,args.theoryDefValNeg,args.theoryPosWeight) # Load Files if os.path.isfile(args.dictsFile): #logger.info('Loading Dictionaries') #startTime = time() dicts.load(args.dictsFile) #logger.info('Done %s',time()-startTime) if os.path.isfile(args.modelFile): #logger.info('Loading Model') #startTime = time() model.load(args.modelFile) #logger.info('Done %s',time()-startTime) if os.path.isfile(args.theoryFile) and args.learnTheories: #logger.info('Loading Theory Models') #startTime = time() theoryModels.load(args.theoryFile) #logger.info('Done %s',time()-startTime) logger.info('All loading completed') # IO Streams OS = open(args.predictions,'w') IS = open(args.inputFile,'r') # Statistics if args.statistics: stats = Statistics(args.cutOff) if args.learnTheories: theoryStats = TheoryStatistics() predictions = None predictedTheories = None #Reading Input File for line in IS: # try: if True: if line.startswith('!'): problemId = dicts.parse_fact(line) # Statistics if args.statistics and computeStats: computeStats = False # Assume '!' comes after '?' if args.predef: predictions = model.predict(problemId) if args.learnTheories: tmp = [dicts.idNameDict[x] for x in dicts.dependenciesDict[problemId]] usedTheories = set([x.split('.')[0] for x in tmp]) theoryStats.update((dicts.idNameDict[problemId]).split('.')[0],predictedTheories,usedTheories,len(theoryModels.accessibleTheories)) stats.update(predictions,dicts.dependenciesDict[problemId],statementCounter) if not stats.badPreds == []: bp = string.join([str(dicts.idNameDict[x]) for x in stats.badPreds], ',') logger.debug('Bad predictions: %s',bp) statementCounter += 1 # Update Dependencies, p proves p dicts.dependenciesDict[problemId] = [problemId]+dicts.dependenciesDict[problemId] if args.learnTheories: theoryModels.update(problemId,dicts.featureDict[problemId],dicts.dependenciesDict[problemId],dicts) if args.snow: model.update(problemId,dicts.featureDict[problemId],dicts.dependenciesDict[problemId],dicts) else: model.update(problemId,dicts.featureDict[problemId],dicts.dependenciesDict[problemId]) elif line.startswith('p'): # Overwrite old proof. problemId,newDependencies = dicts.parse_overwrite(line) newDependencies = [problemId]+newDependencies model.overwrite(problemId,newDependencies,dicts) if args.learnTheories: theoryModels.overwrite(problemId,newDependencies,dicts) dicts.dependenciesDict[problemId] = newDependencies elif line.startswith('?'): startTime = time() computeStats = True if args.predef: continue name,features,accessibles,hints = dicts.parse_problem(line) # Create predictions logger.info('Starting computation for problem on line %s',lineCounter) # Update Models with hints if not hints == []: if args.learnTheories: accessibleTheories = set([(dicts.idNameDict[x]).split('.')[0] for x in accessibles]) theoryModels.update_with_acc('hints',features,hints,dicts,accessibleTheories) if args.snow: pass else: model.update('hints',features,hints) # Predict premises if args.learnTheories: predictedTheories,accessibles = theoryModels.predict(features,accessibles,dicts) # Add additional features on premise lvl if sine is enabled if args.sineFeatures: origFeatures = [f for f,_w in features] secondaryFeatures = [] for f in origFeatures: if dicts.featureCountDict[f] == 1: continue triggeredFormulas = dicts.featureTriggeredFormulasDict[f] for formula in triggeredFormulas: tFeatures = dicts.triggerFeaturesDict[formula] #tFeatures = [ff for ff,_fw in dicts.featureDict[formula]] newFeatures = set(tFeatures).difference(secondaryFeatures+origFeatures) for fNew in newFeatures: secondaryFeatures.append((fNew,args.sineWeight)) predictionsFeatures = features+secondaryFeatures else: predictionsFeatures = features predictions,predictionValues = model.predict(predictionsFeatures,accessibles,dicts) assert len(predictions) == len(predictionValues) # Delete hints if not hints == []: if args.learnTheories: theoryModels.delete('hints',features,hints,dicts) if args.snow: pass else: model.delete('hints',features,hints) logger.info('Done. %s seconds needed.',round(time()-startTime,2)) # Output predictionNames = [str(dicts.idNameDict[p]) for p in predictions[:args.numberOfPredictions]] predictionValues = [str(x) for x in predictionValues[:args.numberOfPredictions]] predictionsStringList = ['%s=%s' % (predictionNames[i],predictionValues[i]) for i in range(len(predictionNames))] predictionsString = string.join(predictionsStringList,' ') outString = '%s: %s' % (name,predictionsString) OS.write('%s\n' % outString) else: logger.warning('Unspecified input format: \n%s',line) sys.exit(-1) lineCounter += 1 """ except: logger.warning('An error occurred on line %s .',line) lineCounter += 1 continue """ OS.close() IS.close() # Statistics if args.statistics: if args.learnTheories: theoryStats.printAvg() stats.printAvg() # Save if args.saveModel: model.save(args.modelFile) if args.learnTheories: theoryModels.save(args.theoryFile) dicts.save(args.dictsFile) if not args.saveStats == None: if args.learnTheories: theoryStatsFile = os.path.join(args.outputDir,'theoryStats') theoryStats.save(theoryStatsFile) statsFile = os.path.join(args.outputDir,args.saveStats) stats.save(statsFile) return 0
def start_simulation_state(no_of_jobs_server_1, no_of_jobs_server_2, arrival_rate, job_distribution, given_x, given_y, sim_time): list_of_servers = [] for _ in range(0,2): # Create 2 servers scheduler_i = FIFO() job_size_i = Expo(job_distribution) server_i = Server(job_size_i, scheduler_i) list_of_servers.append(server_i) # Create dispatcher policy_i = ShortestQueue() #Join shortest queue policy dispatcher = Dispatcher(policy_i, list_of_servers) #Create a dispatcher with JSQ policy statistics = Statistics() world = Global(statistics) p_arrivals = PoissonArrival(float(arrival_rate)) arrival = 0 def set_up_servers(): nonlocal list_of_servers, world for job in range(0, int(no_of_jobs_server_1)): job = Job(0) #All arrive at time 0 list_of_servers[0]._total_jobs +=1 enter_service = max(list_of_servers[0]._total_processing_time, 0) #Takes processing time (time when server is idle) and the time of arrival. Max of those is enter service for this job job.set_enter_service(enter_service) #Setting the enter service, first is 0, next is 0+service_1, next is 0+service_1 + service_2.. ... job.set_service(list_of_servers[0].get_service_time()) #Generates service time AND increases processing time for server departure_time = job._enter_service + job._service_time # Enter service will be correct because of processing time of server bookkeeping job.set_departure(departure_time) #Set the departure time as enter service+service time. (Its a timeperiod on the timeline) world.schedule_event(list_of_servers[0].departure, job._departure_time, [job, world]) #Schedule this departure to the world queue # Now we have some jobs already in the server(s) before we start the main loop of adding arrivals e.t.c. for job in range(0, int(no_of_jobs_server_2)): job = Job(0) list_of_servers[1]._total_jobs +=1 enter_service = max(list_of_servers[1]._total_processing_time, 0) job.set_enter_service(enter_service) job.set_service(list_of_servers[1].get_service_time()) departure_time = job._enter_service + job._service_time job.set_departure(departure_time) world.schedule_event(list_of_servers[1].departure, job._departure_time, [job, world]) #cost function is basically sojourn time in these cases. #### IMPORTANT # Need here to make a check to see if we're out of bounds before each process_event # If we're out of bounds we 'stop' and save the stats. # Then reset the 'world' and run again, until the sim_time is over. # May be best to do a sub-routine to 'reset' the world (can be used for both simulation processes) def reset_world(first_arrival): nonlocal list_of_servers, dispatcher, statistics, world # if we want to modify variable from closure, must place as nonlocal list_of_servers.clear() #clear the 2 servers for _ in range(0,2): # Create 2 servers scheduler_i = FIFO() job_size_i = Expo(job_distribution) # use job_distr from closure server_i = Server(job_size_i, scheduler_i) # create a new server to place in list list_of_servers.append(server_i) dispatcher = Dispatcher(policy_i, list_of_servers) # resetting the dispatcher with new servers statistics = Statistics() world = Global(statistics) set_up_servers() params = [dispatcher, world] world.schedule_event(p_arrivals.generate_arrival, first_arrival, params) #Schedule first arrival to start chain #Now we have created two new servers, reset them and created a dispatcher with the new servers #Then we reset the world(nonlocal) and statistics to get a clean slate #Then we called function to set the initial jobs in the servers again (not same jobs!) reset_world(0) # Call function to setup our world # Now we need to schedule the initial arrivals to start the chain of events. for x in range(1, 11): # Now that each dispatcher has an arrival, we can start looping through events while world.next_event() <= float(sim_time)*(x*0.1): # while the virtual time of next event is less than our simulation time.. if list_of_servers[0]._total_jobs > int(given_x) or list_of_servers[1]._total_jobs > int(given_y): next_arrival_new_world = world.next_event() #Get the time for the first event for new world #reset the world here (e.g. clear all stats and world queue, then reset jobs in servers and 'restart') print('resetting world') world._stats.write_to_file_intermediate_stats('given_end_of_'+given_x+'_and_'+given_y) reset_world(next_arrival_new_world) # This function should reset statistics, world event queue, and server states #also remember to log the stats before reset. world.process_event() # We take the event and process it (running the function(s)) print("{}%".format(x*10)) #for loop to step between while loops (every 10%)while world.next total_no_jobs = world._stats.number_of_jobs print(total_no_jobs) world._stats.print_stats()
def start_simulation_less_than_n(no_of_jobs_server_1, no_of_jobs_server_2, arrival_rate, job_distribution, sim_time): list_of_servers = [] global jobs_ran global final_data stopping_n = no_of_jobs_server_2 for _ in range(0,2): # Create 2 servers scheduler_i = FIFO() job_size_i = Expo(job_distribution) server_i = Server(job_size_i, scheduler_i) list_of_servers.append(server_i) # Create dispatcher policy_i = ShortestQueue() #Join shortest queue policy dispatcher = Dispatcher(policy_i, list_of_servers) #Create a dispatcher with JSQ policy statistics = Statistics() world = Global(statistics) p_arrivals = PoissonArrival(float(arrival_rate)) arrival = 0 #For each job in server 1 we: #Create a job with arrival at time 0, then increase # of jobs in the server. #We get time the job enters service, either immediately or after the total processing done. #Set the enter service and service time needed. #Next calculate departure time and set it, then we can schedule the departure. server_1_processing_time = 0 for job in range(0, int(no_of_jobs_server_1)): job = Job(0) #All arrive at time 0 list_of_servers[0]._total_jobs +=1 enter_service = max(list_of_servers[0]._total_processing_time, 0) #Takes processing time (time when server is idle) and the time of arrival. Max of those is enter service for this job job.set_enter_service(enter_service) #Setting the enter service, first is 0, next is 0+service_1, next is 0+service_1 + service_2.. ... job.set_service(list_of_servers[0].get_service_time()) #Generates service time AND increases processing time for server departure_time = job._enter_service + job._service_time # Enter service will be correct because of processing time of server bookkeeping job.set_departure(departure_time) #Set the departure time as enter service+service time. (Its a timeperiod on the timeline) world.schedule_event(list_of_servers[0].departure, job._departure_time, [job, world]) #Schedule this departure to the world queue for job in range(0, int(no_of_jobs_server_2)): job = Job(0) list_of_servers[1]._total_jobs +=1 enter_service = max(list_of_servers[1]._total_processing_time, 0) job.set_enter_service(enter_service) job.set_service(list_of_servers[1].get_service_time()) departure_time = job._enter_service + job._service_time job.set_departure(departure_time) world.schedule_event(list_of_servers[1].departure, job._departure_time, [job, world]) initial_arrival = random.expovariate(p_arrivals._rate) params = [dispatcher, world] world.schedule_event(p_arrivals.generate_arrival, initial_arrival, params) #Schedule first arrival to start chain last_event = 0 world.number_of_arr_dep = 0 #resetting the number of events before we start # Now we need to schedule the initial arrivals to start the chain of events. # Now that each dispatcher has an arrival, we can start looping through events while world.next_event() <= float(sim_time): # while the virtual time of next event is less than our simulation time.. if(list_of_servers[0]._total_jobs<=stopping_n and list_of_servers[1]._total_jobs <= stopping_n): break last_event = world.next_event() world.process_event() # We take the event and process it (running the function(s)) #for loop to step between while loops (every 10%)while world.next #We've reached a stopping state. Record event parameters and print to file jobs_ran += world._stats.number_of_jobs # We stopped, we add the number of jobs ran this time to global variable recorded_x = list_of_servers[0]._total_jobs recorded_y = list_of_servers[1]._total_jobs recorded_T = last_event #Last event that happened (e.g. departure that caused the total jobs to be < 4) recorded_N = world.number_of_arr_dep #Get the number of events that happened' final_data.append((recorded_x, recorded_y, recorded_T, recorded_N))
class MainWindow(QtGui.QMainWindow): def __init__(self): super(MainWindow, self).__init__() self.setWindowTitle("Algae Count Estimator") self.setSizePolicy(QtGui.QSizePolicy.Fixed, QtGui.QSizePolicy.Fixed) self.setFixedSize(QtCore.QSize(800,620)) self.initPages() self.initMenuBar() self.show() """ initPages(): Creates and populates a QStackedWidget that contains the different modes (i.e. pages) of the application. """ def initPages(self): self.mode_stack = QtGui.QStackedWidget() main_menu = MainMenu(self) #0 self.stats = Statistics(self) #1 self.trainer = Trainer(self,self.stats) #2 self.generator = Generator(self) #3 self.mode_stack.addWidget(main_menu) self.mode_stack.addWidget(self.stats) self.mode_stack.addWidget(self.trainer) self.mode_stack.addWidget(self.generator) self.setCentralWidget(self.mode_stack) """ initMenuBar(): Creates a menu bar that is visible from any mode of the application. The options under the menus are called "actions". """ def initMenuBar(self): menu_bar = self.menuBar() #Create File menu and actions. file_menu = menu_bar.addMenu('&File') to_menu_action = QtGui.QAction('Main Menu', self) to_menu_action.triggered.connect(lambda: self.changeMode(ModeEnum.MENU)) exit_action = QtGui.QAction('Exit', self) exit_action.triggered.connect(self.exitProgram ) file_menu.addAction(to_menu_action) file_menu.addAction(exit_action) #Create Help menu and actions. help_menu = menu_bar.addMenu('&Help') about_action = QtGui.QAction('About', self) about_action.triggered.connect(self.aboutMenu) help_menu.addAction(about_action) """ changeMode(page_num): Changes the view of the user to the mode associated with page_num. page_num is an integer defined by ModeEnum. (see enum module). Special conditions that need to be met before switching to a mode may be addressed here. """ def changeMode(self, page_num): #Perform any work that needs to be done before switching modes. if page_num == ModeEnum.STATS: #Update the stats page before switching the view. self.stats.updateStatsUI() elif page_num == ModeEnum.TRAINER: #Attempt to start a new Trainer session if there is not #an active session. if (not self.trainer.has_active_session): if (not self.trainer.startNewSession()): return #Switch the mode. self.mode_stack.setCurrentIndex(page_num) """ exitProgram(): This exit routine is called when the user exits via File->Exit. Writes the stats to disk and closes the application. """ def exitProgram(self): self.mode_stack.widget(ModeEnum.STATS).writeStatsToFile() QtGui.qApp.quit() """ closeEvent(event): This exit routine is an overloaded version of Qt's closeEvent, and normally occurs when the user exits by clicking the 'X'. Before accepting the close event, stats are written to disk. """ def closeEvent(self, event): self.mode_stack.widget(ModeEnum.STATS).writeStatsToFile() event.accept() def aboutMenu(self): QtGui.QMessageBox.about(self, 'About ACE', '''<p><strong>ACE 1.0.0</strong><br/>The Algae Count Estimator</p><p>Copyright © 2014 Samuel Dunlap, Josh Minor, Matt Ralphs, Andrew Young<br/>Licensed under the terms of the MIT License</p> <p>Build for those who require assistance in the counting of blue-green algae, ACE is a tool that graphically generates algae colonies with known counts. Users may test their ability to count these colonies with this program and be provided statistics regarding how they've done.</p> <p>For more info, see the User's Manual located in the Docs folder.</p> <p>This project was originally commissioned for the Institute for Watershed Studies of Western Washington University.</p> ''')
def compare_returns(df, compare_hurst_exp: bool = False, compare_lyapunov_exponent=False, window: int = 10): """Compare returns of the simulated markets with the real market""" # TODO: change this hardcoded mess product = "LTC-USD" plt = Evaluation.get_plt(window) df = Evaluation.calculate_aux_rows(df, window) Evaluation.print_sim_summary(df) Evaluation.plot_linregress(df['rp_diff'], df['sp_diff'], window, xlabel="Real 5 minute returns", ylabel="Simulated 5 minute returns") if compare_lyapunov_exponent: step_minutes = 1 window_minutes = 240 st_str = df['start_time'].min() et_str = df['start_time'].max() st = datetime.datetime.strptime(st_str, "%Y-%m-%dT%H:%M:%S") + datetime.timedelta( minutes=window_minutes - 60) et = datetime.datetime.strptime(et_str, "%Y-%m-%dT%H:%M:%S") + datetime.timedelta( minutes=window_minutes - 60) _, trades, _ = DataLoader.load_split_data("/Users/jamesprince/project-data/data/consolidated-feed/LTC-USD/", st, et, product) times, lyap_exps = Statistics.get_lyapunov_exponent_over_time(trades, st, et, step_minutes, window_minutes) lyap_df = pd.DataFrame({'start_time': list(map(lambda t: t.isoformat(), times)), 'lyap_exp': lyap_exps}) joined_df = pd.merge(df, lyap_df, how='left', on='start_time') dropped_df = joined_df.dropna(subset=['lyap_exp']) x1 = dropped_df['lyap_exp'] x2 = dropped_df['rp_sp_diff'] plt.xlim(-0.2, 0.2) plt.ylim(-10, 10) Evaluation.plot_linregress(x1, x2, window, show_regression_line=True, xlabel="Lyapunov Exponent", ylabel="Diff between real and simulated returns") if compare_hurst_exp: st_str = df['start_time'].min() et_str = df['start_time'].max() step_minutes = 5 window_minutes = 5 st = datetime.datetime.strptime(st_str, "%Y-%m-%dT%H:%M:%S") - datetime.timedelta(minutes=window_minutes) et = datetime.datetime.strptime(et_str, "%Y-%m-%dT%H:%M:%S") - datetime.timedelta(minutes=window_minutes) _, trades, _ = DataLoader.load_split_data("/Users/jamesprince/project-data/data/consolidated-feed/LTC-USD/", st, et, product) times, lyap_exps = Statistics.get_hurst_exponent_over_time(trades, st, et, step_minutes, window_minutes) lyap_df = pd.DataFrame({'start_time': list(map(lambda t: t.isoformat(), times)), 'hurst_exp': lyap_exps}) print(lyap_df) joined_df = pd.merge(df, lyap_df, how='left', on='start_time') dropped_df = joined_df.dropna(subset=['hurst_exp']) x1 = dropped_df['hurst_exp'] x2 = dropped_df['rp_sp_diff'] Evaluation.plot_linregress(x1, x2, window, show_regression_line=True, xlabel="Hurst Exponent", ylabel="Diff between real and simulated returns")