def process_log(args): with open(args.e2e_file, 'r') as infile: lines = infile.readlines() batches = {} for line in lines: group_id = int(line.split("group_id=")[1].split(',')[0]) batch_id = int(line.split("batch_id=")[1].split(',')[0]) reconstruction = line.split("is_reconstruction=")[1].split( ',')[0] == "true" latency = int(line.split("=")[-1]) if batch_id not in batches: batches[batch_id] = Batch() batches[batch_id].add_query(latency, reconstruction) if args.batch_size > 1: for _, b in batches.items(): if len(b.latencies) > 1: b.latencies = [b.latencies[-1]] if len(b.decoded_latencies) > 1: b.decoded_latencies = [b.decoded_latencies[-1]] og_latencies = [] for _, b in batches.items(): og_latencies.extend(b.latencies) og_med = np.median(og_latencies) latencies = [] for _, b in batches.items(): if b.decoded: if args.mode == "coded": latencies.extend(b.decoded_latencies) elif args.mode == "cheap": if len(b.latencies) == 0: # If we don't have logs for "normal" latencies, then those # batches were likely removed from the queue before # processing due to a completed redundant task. In that # case, we just take the decoded latencies. latencies.extend(b.decoded_latencies) else: # For cheap and replication, the "decoded" latencies are the # minimum between the two latencies. We perform the max # operation between the approximate result and the median # latency so as not to log returning an approximate result # if that result was returned too soon. This simulates # having an aggressive timeout of median latency. mins = [ min(og, max(og_med, d)) for og, d in zip(b.latencies, b.decoded_latencies) ] latencies.extend(mins) else: latencies.extend(b.latencies) # Convert to ms latencies = [x / 1000 for x in latencies] print_stats(latencies)
def MonteCarlo(mesh,system,potential_field,electric_field, density_funcs, avg_dens, avg_electrons, avg_holes, current_values): reaper = [] current = 0 #next_step density function array nextDensity = density_funcs.combined_density.vector().array().astype('int') #holeDensity = density_funcs.holes.vector().array() #electronDensity = density_funcs.electrons.vector().array() #compute field, move_particles, calc current, recombinate, reap start = time.time() print "computing field" c_efield = pre_compute_field(mesh,electric_field) #Evaluates field at each mesh point print "nextDensity max:",max(nextDensity) start =time.time() print "About to move particles" c_efield = array(c_efield) #print max(map(lambda x:max(abs(x[0]),abs(x[1])),c_efield)),\ # min(map(lambda x:min(abs(x[0]),abs(x[1])),c_efield)) #raw_input() move_particles_c.move_particles(system, c_efield,nextDensity, mesh.dt,mesh.length_scale) print "About to update" current = move_particles_c.update_density(system, nextDensity, mesh.kdt) print "Recombination" move_particles_c.recombinate(system,nextDensity,mesh.kdt) print current print "time:",time.time()-start start = time.time() current += move_particles_c.replenish(system,nextDensity) print "replenish:",time.time()-start #reap_list(mesh.particles,reaper) #update current values current_values.append(current*constants.eC/ (mesh.dt*mesh.gen_num*mesh.num_vertices())) #update avg_dens scaled_density = calculate_scaled_density(mesh,nextDensity) avg_dens.inc(scaled_density) density_funcs.combined_density.vector()[:]= nextDensity.astype('double') density_funcs.scaled_density.vector()[:]=scaled_density print density_funcs.scaled_density.vector().array() #print "doom",density_funcs.combined_density.vector().array() #print stats stats.print_stats(current)
def shutdown(code=0): """ Shut down locust by firing quitting event, printing stats and exiting """ logger.info("Shutting down, bye..") events.quitting.fire() print_stats(runners.locust_runner.request_stats) print_percentile_stats(runners.locust_runner.request_stats) print_error_report() sys.exit(code)
def run_dataset(dataset): results = {} for benchmark in BENCHMARKS: times = [] for i in range(RUNS): elapsed_time = submit_spark_job(dataset, benchmark) times.append(elapsed_time) print_stats(times) results[benchmark] = times return results
def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False): if spawn_count is None: spawn_count = self.num_clients if self.num_requests is not None: RequestStats.global_max_requests = self.num_requests bucket = self.weight_locusts(spawn_count, stop_timeout) spawn_count = len(bucket) if self.state == STATE_INIT or self.state == STATE_STOPPED: self.state = STATE_HATCHING self.num_clients = spawn_count else: self.num_clients += spawn_count logger.info( "Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate)) occurence_count = dict([(l.__name__, 0) for l in self.locust_classes]) def hatch(): sleep_time = 1.0 / self.hatch_rate while True: if not bucket: logger.info("All locusts hatched: %s" % ", ".join([ "%s: %d" % (name, count) for name, count in occurence_count.iteritems() ])) events.hatch_complete.fire(self.num_clients) return locust = bucket.pop(random.randint(0, len(bucket) - 1)) occurence_count[locust.__name__] += 1 def start_locust(_): try: locust()() except RescheduleTaskImmediately: pass except GreenletExit: pass new_locust = self.locusts.spawn(start_locust, locust) if len(self.locusts) % 10 == 0: logger.debug("%i locusts hatched" % len(self.locusts)) gevent.sleep(sleep_time) hatch() if wait: self.locusts.join() logger.info("All locusts dead\n") print_stats(self.request_stats) print_percentile_stats( self.request_stats ) #TODO use an event listener, or such, for this?
def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False): if spawn_count is None: spawn_count = self.num_clients if self.num_requests is not None: RequestStats.global_max_requests = self.num_requests bucket = self.weight_locusts(spawn_count, stop_timeout) spawn_count = len(bucket) if self.state == STATE_INIT or self.state == STATE_STOPPED: self.state = STATE_HATCHING self.num_clients = spawn_count else: self.num_clients += spawn_count logger.info("Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate)) occurence_count = dict([(l.__name__, 0) for l in self.locust_classes]) def hatch(): sleep_time = 1.0 / self.hatch_rate while True: if not bucket: logger.info( "All locusts hatched: %s" % ", ".join(["%s: %d" % (name, count) for name, count in occurence_count.iteritems()]) ) events.hatch_complete.fire(self.num_clients) return locust = bucket.pop(random.randint(0, len(bucket) - 1)) occurence_count[locust.__name__] += 1 def start_locust(_): try: locust()() except RescheduleTaskImmediately: pass except GreenletExit: pass new_locust = self.locusts.spawn(start_locust, locust) if len(self.locusts) % 10 == 0: logger.debug("%i locusts hatched" % len(self.locusts)) gevent.sleep(sleep_time) hatch() if wait: self.locusts.join() logger.info("All locusts dead\n") print_stats(self.request_stats) print_percentile_stats(self.request_stats) # TODO use an event listener, or such, for this?
def do_threading(stats, ivy_filename, json_filename, logdir, nthreads, main_args, breadth_args, finisher_args, by_size): print(ivy_filename) print("json: ", json_filename) invfile = None success = False if not success and len(breadth_args) > 0: success, invfile = do_breadth("iteration", logdir, nthreads, json_filename, main_args, main_args + breadth_args, invfile, stats, by_size) if success: print("Invariant success!") if not success and len(finisher_args) > 0: success = do_finisher("finisher", logdir, nthreads, json_filename, main_args, main_args + finisher_args, invfile, stats) if success: print("Invariant success!") statfile = logdir + "/summary" stats.add_global_stat_files(all_stats_files) stats.print_stats(statfile) print("") print("statfile: " + statfile) print("") print("======== Summary ========") print("") with open(statfile, "r") as f: t = f.read() t = t.split("specifics:") print(t[0]) final_inv_file = logdir + "/invariants" with open(final_inv_file, "w") as f: if stats.was_success(): f.write("# Success: True\n") else: f.write("# Success: False\n") for inv in stats.get_invariants(): f.write("conjecture " + protocol_parsing.value_json_to_string(inv) + "\n")
def main(): args = parse_arguments() tga = TGAParser(args.inputfile) if args.encode: bitmap = tga.get_bitmap() low, low_encoded, high, high_quantified = encode(bitmap, args.k) tga.new_tga(low, "low.tga") tga.new_tga_from_bytes(low_encoded, "low_encoded") tga.new_tga(high, "high.tga") tga.new_tga(high_quantified, "high_quantified.tga") if not args.silent: print("\033[4m", "LOW PASS FILTER", "\033[0m") print_stats(bitmap, low) print("\n\033[4m" "HIGH PASS FILTER, QUANTIFIED", "\033[0m") print_stats(bitmap, high_quantified) else: tga.new_tga(decode(tga.bitmap, tga.width, tga.height), "low_decoded.tga")
daily_mod_electric_ref.plot(color='yellow', label='Synthetic time series with 2017 weather') #daily_electric_2009.plot(color='orange', label='Electricity demand time series 2009') plt.title('Daily electricty demand series from weather') plt.xlabel('day', fontsize=15) plt.ylabel('Energy (Mwh) per day', fontsize=15) plt.legend(loc='upper center') plt.show() # compare weekly series print('Weekly Comparison to 2017 electricity demand') stats.print_stats_header() stats.print_stats(electric_2018.resample('W').sum(), electric_2017.resample('W').sum(), '2018 Electric', nvars=1, plot=args.plot, xl='Electricity demand 2018', yl='Electricity demand 2017') stats.print_stats(mod_electric_ref.resample('W').sum(), electric_2017.resample('W').sum(), '2017 From 2018 series and weather', nvars=1, plot=args.plot, xl='Electricity demand synthetic', yl='Electricity demand 2017') # compare hourly series print('Hourly Comparison to 2017 electricity demand') stats.print_stats_header() stats.print_stats(electric_2018,
# python script to compare 2 csv time series files # usage: python comparecsv.py file1 file2 import sys from pandas import read_csv import statsmodels.api as sm import stats # get command line. file1 = sys.argv[1] file2 = sys.argv[2] field = sys.argv[3] print(file1, file2) data1 = read_csv(file1, header=0, sep=',', decimal='.', parse_dates=[0], index_col=0, squeeze=True, usecols=['time','space','heat',field] ) data2 = read_csv(file2, header=0, sep=',', decimal='.', parse_dates=[0], index_col=0, squeeze=True, usecols=['time','space','heat',field] ) print(data1.head(7)) print(data2.head(7)) stats.print_stats_header() stats.print_stats(data1[field], data2[field], 'oldVnew')
'EMBEDDED_SOLAR_GENERATION'] electric_total = electric_total * (gas_energy.sum() / electric_total.sum()) print(electric_total.head()) print(electric_total.tail()) # convert all to net demand gas_energy = gas_energy - electric_total space_and_water = space_and_water - electric_total watson_space_and_water = watson_space_and_water - electric_total hdd_space_and_water15 = hdd_space_and_water15 - electric_total hdd_space_and_water12 = hdd_space_and_water12 - electric_total # compute R2 and correlation stats.print_stats_header() stats.print_stats(space_and_water, gas_energy, 'Ruhnau ') stats.print_stats(watson_space_and_water, gas_energy, 'Watson ') stats.print_stats(hdd_space_and_water15, gas_energy, 'HDD 15.5') stats.print_stats(hdd_space_and_water12, gas_energy, 'HDD 12.8') # largest and smallest diffs stats.print_large_small(hdd_space_and_water15, gas_energy, 'HDD 15.5') # output plots gas_energy.plot(label='Gas Energy') space_and_water.plot(label='Heat Demand BDEW') watson_space_and_water.plot(label='Heat Demand Watson') hdd_space_and_water15.plot(label='Heat Demand HDD 15.5') hdd_space_and_water12.plot(label='Heat Demand HDD 12.8')
closed_set.add(current) for new_board in current.generate_moves(): if new_board in closed_set: continue tentative_g_score = g_score[current] + dist_between( current, new_board) if tentative_g_score >= g_score[new_board]: continue came_from[new_board] = current g_score[new_board] = tentative_g_score f_score = tentative_g_score + heuristic_cost_estimate( new_board, desired_board) open_items.put((f_score, new_board)) def run(_): board = random_start_board() return astar(board) if __name__ == '__main__': N = 20 with Pool(10) as p: results = p.map(run, range(N)) print_stats(results)
# start streaming queue = get_queue() http_session = get_http_session() session_id = session_data["id"] # first queue reader callback_storage = lambda data: store_tracking_session_location_element( get_http_session(), session_id, data) start_daemon_process(queue_reader, ( callback_storage, queue, )) # setup pozyx localizer = setup_poszyx(session_data) # write start_daemon_process(read_poszyx, ( queue, localizer, )) # cli cli(queue) session_data["stop_time"] = datetime.datetime.utcnow().isoformat() session_data["data"] = get_session_data(session_id) print(print_stats(session_data))
# ninja.plot(label='ninja') power_era_b.plot(label='era5') power_mid_b.plot(label='midas') power_nin_b.plot(label='ninja') plt.title('2018 Blackpool wind power from midas and ERA5 and ninja') plt.xlabel('Day of the year') plt.ylabel('Energy generated (MWh)') plt.legend(loc='upper right') plt.show() #ninja.index = ninja.index.tz_localize('Europe/London') ninja.index = ninja.index.tz_localize('UTC') stats.print_stats_header() stats.print_stats(wind_era, wind_midas, 'era and midas wind', 1, True, 'ERA5 wind speed (m/s)', 'MIDAS wind speed (m/s)' ) stats.print_stats(era['power_b'], midas['power_b'], 'era and midas power', 1, True, 'ERA Energy (MWh)', 'MIDAS Energy (MWh)' ) stats.print_stats(era['power_b'], ninja, 'era and ninja power', 1, True, 'ERA Energy (MWh)', 'Ninja Energy (MWh)' ) annual_era = {} annual_midas = {} locations = ['a', 'b', 'c', 'l', 's', 'w'] for location in locations: annual_era[location] = era['power_' + location].sum() print(location, annual_era[location]) annual_midas[location] = midas['power_' + location].sum() max_era = max(annual_era.values()) max_midas = max(annual_midas.values()) # print(max_era, max_midas)
total_synthetic = daily_synthetic.sum() percent = (total_real - total_synthetic) / total_real mean_factor = daily_real.mean() / daily_synthetic.mean() print("Total Real {} Synthetic {} Percentage {} Mean Factor {}".format(total_real, total_synthetic, percent, mean_factor) ) #fixed_synthetic = daily_synthetic * mean_factor #fixed_synthetic = (daily_synthetic * 0.4669) + 38.36 #fixed_synthetic = (daily_synthetic / 0.4669) - 38.36 #fixed_synthetic = (daily_synthetic - 38.36) / 0.4669 #fixed_synthetic = (daily_synthetic - 38.36) * 0.4669 #fixed_synthetic = (daily_synthetic + 38.36) * 0.4669 fixed_synthetic = (daily_synthetic * 1.4669) + 38.36 # compute R2 and correlation stats.print_stats_header() stats.print_stats(daily_synthetic, daily_real, 'Electric Daily', 1, True) stats.print_stats(fixed_synthetic * mean_factor, daily_real, 'Electric Fixed', 1, True) # output plots #print('COLUMNS') #print(dfd.columns) daily_synthetic.plot(label='Modelled Electricity Time Series', color='blue') daily_real.plot(label='Actual Measured Electricity Time Series', color='red') fixed_synthetic.plot(label='Fixed Electricity Time Series', color='green') plt.title('Comparison of predicted electricty demand and measured from heat pumps') plt.xlabel('Day of the year', fontsize=15) plt.ylabel('Electricity Demand (kWh)', fontsize=15) plt.legend(loc='upper right', fontsize=15) plt.show()
tower['wind70'].plot(label='Tower Wind Speed at 70m') tower['log70'].plot(label='Log law from 21m to 70m') tower['power70'].plot(label='Power law from 21m to 70m') era5['log70'].plot(label='ERA5 Log law from 10m to 70m') era5['power70'].plot(label='ERA5 Power law using 2 heights 70m') plt.title('Amsterdam Offshore Tower windspeed for December 2010') plt.xlabel('Day of the month') plt.ylabel('Wind Speed (M/S)') plt.legend(loc='upper right') plt.show() # output stats for 70m stats.print_stats_header() stats.print_stats(tower['log70'], tower['wind70'], 'Log law 70m', 1, True, 'Log law wind speed (m/s)', 'Actual wind speed (m/s)') stats.print_stats(tower['power70'], tower['wind70'], 'Power law 70m', 1, True, 'Power law wind speed (m/s)', 'Actual wind speed (m/s)') stats.print_stats(era5['log70'], tower['wind70'], 'ERA5 log law', 1, True, 'Power law wind speed (m/s)', 'Actual wind speed (m/s)') stats.print_stats(era5['power70'], tower['wind70'], 'ERA5 Power law', 1, True, 'Power law wind speed (m/s)', 'Actual wind speed (m/s)') # output wind plots for 116m tower['wind116'].plot(label='Tower Wind Speed at 116m') tower['log116'].plot(label='Log law from 21m to 116m') tower['power116'].plot(label='Power law from 21m to 116m') era5['log116'].plot(label='ERA5 Log law from 10m to 116m') era5['power116'].plot(label='ERA5 Power law using 2 heights 116m') plt.title('Amsterdam Offshore Tower windspeed for December 2010')
def print_results(results): for dataset in DATASETS: for benchmark in BENCHMARKS: times = results[dataset][benchmark] print(f"\n{benchmark} on {dataset} times:") print_stats(times)
plt.ylabel('Demand (MWh)') plt.legend(loc='upper right') plt.show() # plot the values so we see how similar they are plt.plot(demand19.values, label='2019') plt.plot(demand18.values, label='2018') plt.plot(demand17.values, label='2017') plt.plot(demand16.values, label='2016') plt.plot(demand15.values, label='2015') plt.title('England and Wales Electricity Demand') plt.xlabel('Hour of the year') plt.ylabel('Demand (MWh)') plt.legend(loc='upper right') plt.show() # drop the last 24 hours from 2016, (leap year) so they are the same length demand16.drop(demand16.tail(24).index, inplace=True) print('2019 {} 2018 {} 2017 {} 2016 {} 2015 {}'.format(len(demand19), len(demand18), len(demand17), len(demand16), len(demand15))) stats.print_stats_header() stats.print_stats(demand18, demand19, '2018') stats.print_stats(demand17, demand19, '2017') stats.print_stats(demand16, demand19, '2016') stats.print_stats(demand15, demand19, '2015')
destinee_filename = '/home/malcolm/uclan/data/destinee/heat.csv' destinee_heat = readers.read_destinee(destinee_filename) print(destinee_heat) # convert from TJ to Kwh then to GWh # destinee_electric_heat = destinee_heat['electric'] * 277777.77777778 * 1e-6 # convert from TWh to GWh destinee_electric_heat = destinee_heat['electric'] * 1e+3 print('Electric heat annual mine: {} destinee: {}'.format( heat.sum(), destinee_electric_heat.sum())) daily_heat = heat.resample('D').sum() daily_destinee_electric_heat = destinee_electric_heat.resample('D').sum() # daily plot of historic and electric daily_heat.plot(label='BDEW') daily_destinee_electric_heat.plot(label='DESSTINEE') plt.title('Daily UK Electric Heat 2010 BDEW and DESSTINEE') plt.xlabel('Day of the year') plt.ylabel('Demand (MWh)') plt.legend(loc='upper right') plt.show() #print(daily_heat.index) #print(daily_destinee_electric_heat.index) daily_destinee_electric_heat.index = daily_heat.index stats.print_stats_header() stats.print_stats(daily_heat, daily_destinee_electric_heat, 'DESSTINEE', 2, True)
def main(): all_instances = read_imp_arg_dataset(sys.argv[1]) # print_imp_arg_dataset(all_instances, 'test_annotations.txt') print '\nLoading predicate dict from {}'.format(predicate_dict_path) predicate_dict = pkl.load(open(predicate_dict_path, 'r')) # print_imp_arg_dataset_by_pred(all_instances, predicate_dict, 'ia_by_pred') treebank_reader = TreebankReader() propbank_reader = PropbankReader() propbank_reader.build_index() nombank_reader = NombankReader() nombank_reader.build_index() if not exists(corenlp_dict_path): print '\nNo existing CoreNLP dict found' corenlp_reader = CoreNLPReader.build(all_instances) corenlp_reader.save(corenlp_dict_path) else: corenlp_reader = CoreNLPReader.load(corenlp_dict_path) if not exists(all_predicates_path): all_predicates = [] for instance in all_instances: predicate = Predicate.build(instance) predicate.set_pred(predicate_dict[str(predicate.pred_pointer)]) all_predicates.append(predicate) print '\nChecking explicit arguments with Nombank instances' for predicate in all_predicates: nombank_instance = nombank_reader.search_by_pointer( predicate.pred_pointer) predicate.check_exp_args(nombank_instance, add_missing_args=False, remove_conflict_imp_args=False, verbose=False) print '\nParsing all implicit and explicit arguments' for predicate in tqdm(all_predicates, desc='Processed', ncols=100): predicate.parse_args(treebank_reader, corenlp_reader) print '\nSaving all parsed predicates to {}'.format( all_predicates_path) pkl.dump(all_predicates, open(all_predicates_path, 'w')) else: print '\nLoading all parsed predicates from {}'.format( all_predicates_path) start_time = timeit.default_timer() all_predicates = pkl.load(open(all_predicates_path, 'r')) elapsed = timeit.default_timer() - start_time print '\tDone in {:.3f} seconds'.format(elapsed) # check_multi_pobj(all_predicates) if not exists(candidate_dict_path): print '\nBuilding candidate dict from Propbank and Nombank' candidate_dict = CandidateDict(propbank_reader=propbank_reader, nombank_reader=nombank_reader, corenlp_reader=corenlp_reader, max_dist=max_candidate_dist) for predicate in tqdm(all_predicates, desc='Processed', ncols=100): candidate_dict.add_candidates(predicate.pred_pointer) candidate_dict.save(candidate_dict_path) else: candidate_dict = CandidateDict.load(candidate_dict_path, propbank_reader=propbank_reader, nombank_reader=nombank_reader, corenlp_reader=corenlp_reader, max_dist=max_candidate_dist) # candidate_dict.print_all_candidates('all_candidates.txt') print '\nAdding candidates to predicates' for predicate in all_predicates: for candidate in candidate_dict.get_candidates(predicate.pred_pointer): predicate.candidates.append(candidate) # print_all_predicate(all_predicates, 'all_predicates.txt', verbose=True, # include_candidates=True, include_dice_scores=True, # corenlp_reader=corenlp_reader) if not exists(all_rich_predicates_path): all_rich_predicates = [] for predicate in all_predicates: rich_predicate = RichPredicate.build(predicate, corenlp_reader, use_lemma=True, use_entity=True, use_corenlp_tokens=True) all_rich_predicates.append(rich_predicate) print '\nSaving all rich predicates to {}'.format( all_rich_predicates_path) pkl.dump(all_rich_predicates, open(all_rich_predicates_path, 'w')) print '\nPrinting statistics' print_stats(all_predicates)
def main(): parser, options, arguments = parse_options() #print "Options:", options, dir(options) #print "Arguments:", arguments #print "largs:", parser.largs #print "rargs:", parser.rargs if options.show_version: print("Locust %s" % (version)) sys.exit(0) locustfile = find_locustfile(options.locustfile) if not locustfile: print "Could not find any locustfile!" sys.exit(1) docstring, locusts = load_locustfile(locustfile) if options.list_commands: print "Available Locusts:" for name in locusts: print " " + name sys.exit(0) if not locusts: sys.stderr.write("No Locust class found!\n") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: sys.stderr.write("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.web and not options.slave: # spawn web greenlet print "Starting web monitor on port 8089" gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests) if not options.master and not options.slave: core.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host) # spawn client spawning/hatching greenlet if not options.web: core.locust_runner.start_hatching() elif options.master: core.locust_runner = MasterLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, redis_host=options.redis_host, redis_port=options.redis_port) elif options.slave: core.locust_runner = SlaveLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, redis_host=options.redis_host, redis_port=options.redis_port) if not options.web or options.print_stats: # spawn stats printing greenlet gevent.spawn(stats_printer) try: print "" print "Starting Locust %s" % version print "" core.locust_runner.greenlet.join() except KeyboardInterrupt, e: print_stats(core.locust_runner.request_stats) print_percentile_stats(core.locust_runner.request_stats) print "" print "Exiting, bye.." print ""
def print_stats(self, verbose=0): print_stats(self.all_predicates, verbose=verbose)
def main(): parser, options, arguments = parse_options() # setup logging setup_logging(options.loglevel, options.logfile) logger = logging.getLogger(__name__) if options.show_version: print "Locust %s" % (version) sys.exit(0) locustfile = find_locustfile(options.locustfile) if not locustfile: logger.error("Could not find any locustfile! See --help for available options.") sys.exit(1) docstring, locusts = load_locustfile(locustfile) if options.list_commands: print "Available Locusts:" for name in locusts: print " " + name sys.exit(0) if not locusts: logger.error("No Locust class found!") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: logger.error("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.show_task_ratio: console_logger.info("\n Task ratio per locust class") console_logger.info( "-" * 80) print_task_ratio(locust_classes) console_logger.info("\n Total task ratio") console_logger.info("-" * 80) print_task_ratio(locust_classes, total=True) sys.exit(0) if options.show_task_ratio_json: from json import dumps task_data = { "per_class": get_task_ratio_dict(locust_classes), "total": get_task_ratio_dict(locust_classes, total=True) } console_logger.info(dumps(task_data)) sys.exit(0) # if --master is set, make sure --no-web isn't set if options.master and options.no_web: logger.error("Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)") sys.exit(0) if not options.no_web and not options.slave: # spawn web greenlet logger.info("Starting web monitor on port 8089") main_greenlet = gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.ramp) # enable/disable gzip in WebLocust's HTTP client WebLocust.gzip = options.gzip if not options.master and not options.slave: runners.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host) # spawn client spawning/hatching greenlet if options.no_web: runners.locust_runner.start_hatching(wait=True) main_greenlet = runners.locust_runner.greenlet elif options.master: runners.locust_runner = MasterLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host) elif options.slave: runners.locust_runner = SlaveLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host) main_greenlet = runners.locust_runner.greenlet if options.print_stats or (options.no_web and not options.slave): # spawn stats printing greenlet gevent.spawn(stats_printer) try: logger.info("Starting Locust %s" % version) main_greenlet.join() except KeyboardInterrupt, e: events.quitting.fire() time.sleep(0.2) print_stats(runners.locust_runner.request_stats) print_percentile_stats(runners.locust_runner.request_stats) print_error_report() logger.info("Got KeyboardInterrupt. Exiting, bye..")
# subtracting the non heat gas and divide evenly amongst the days. # assuming that the non-gas energy is constant. # (if its not, this could go negative) gas_energy = gas_energy - (non_heat_gas / 365.0) # print(gas_energy.head(7)) print('GAS ENERGY SMALLEST') print(gas_energy.nsmallest()) # account for boiler efficiency of 90% # (this is inconsistent as 80% is used for the EU buildings figures) gas_energy = gas_energy * 0.8 # compute R2 and correlation stats.print_stats_header() stats.print_stats(space_and_water, gas_energy, 'HDD 15.5 ') stats.print_stats(space_and_waterS1, gas_energy, 'HDD 15.5 S1 ') stats.print_stats(space_and_waterS2, gas_energy, 'HDD 15.5 S2 ') # output plots gas_energy.plot(label='Gas Energy') space_and_water.plot(label='Heat Demand HDD 15.5') space_and_waterS1.plot(label='Heat Demand HDD 15.5 S1') space_and_waterS2.plot(label='Heat Demand HDD 15.5 S2') plt.title('Comparison of Heat Demand Methods') plt.xlabel('Day of the year') plt.ylabel('Heat Demand (MWh)') plt.legend(loc='upper right') plt.show()
], ) df = df.append(csv) total = total - 1 df.dtypes[0] = "UInt64" return df def run_benchmark(df): start = time.time() df.describe(include="all") elapsed = time.time() - start print(f"Elapsed time: {elapsed}") return elapsed RUNS = 5 results = {} for dataset in DATASETS: times = [] df = read_data(dataset) for i in range(RUNS): times.append(run_benchmark(df)) results[dataset] = times for dataset in DATASETS: print(f"Results for dataset {dataset}") print_stats(results[dataset])
queries = queries_file.read().replace("\n", "").split(";") queries.pop(-1) # remove trailing empty element # If kerberos is enabled, make sure we have the sasl library if args.kerberos: try: import sasl except ImportError: print("You need the sasl library in order to use kerberos") sys.exit(1) scheduler = ImpalaQueryScheduler( queries, args.threads, args.impala_hosts.split(","), args.stats_port, args.kerberos, args.kerberos_service ) # Keep an empty place holder for the final stats stats = None def signal_handler(signal, frame): global stats stats = get_stats_tables(url="http://localhost:%d" % args.stats_port) scheduler.shutdown() while not scheduler.finished: time.sleep(0.10) scheduler.start() signal.signal(signal.SIGINT, signal_handler) signal.pause() print("\n\n") print_stats(stats)
# main program # read in the data output_dir = "/home/malcolm/uclan/data/advanced_metering/testing/" filename = output_dir + 'methods.csv' df = pd.read_csv(filename, header=0, sep=',', parse_dates=[0], index_col=0, squeeze=True) # convert from Wh to kWh df = df * 0.001 # compute R2 and correlation stats.print_stats_header() stats.print_stats(df['B'], df['gas'], 'BDEW', 2, True) stats.print_stats(df['W'], df['gas'], 'Watson', 1, True) stats.print_stats(df['H'], df['gas'], 'HDD 12.8', 1, True) stats.print_stats(df['S'], df['gas'], 'HDD 15.5 ', 1, True) stats.monthly_stats_header() m_b = stats.monthly_stats(df['B'], df['gas'], 'BDEW') m_w = stats.monthly_stats(df['W'], df['gas'], 'Watson') m_h = stats.monthly_stats(df['H'], df['gas'], 'HDD 12.8') m_s = stats.monthly_stats(df['S'], df['gas'], 'HDD 15.5') plt.plot(range(12), m_b, label='BDEW') plt.plot(range(12), m_w, label='Watson') plt.plot(range(12), m_h, label='HDD 12.8') plt.plot(range(12), m_s, label='HDD 15.5') plt.xlabel('Month of the year', fontsize=15)
power_nin_c = ninja #pv_generation_compare.py power_era_c.plot(label='era5') power_mid_c.plot(label='midas') power_nin_c.plot(label='ninja') plt.title('2018 Cambourne PV power from midas and ERA5 and ninja') plt.xlabel('Day of the year') plt.ylabel('Energy generated (MWh)') plt.legend(loc='upper right') plt.show() ninja.index = ninja.index.tz_localize('Europe/London') stats.print_stats_header() stats.print_stats(era['ghi_c'], midas['ghi_c'], 'era and midas ghi') stats.print_stats(era['power_c'], midas['power_c'], 'era and midas power') #stats.print_stats(era['power_c'], ninja, 'era and ninja power') annual_era = {} annual_midas = {} locations = ['a', 'c', 'e', 'u'] for location in locations: annual_era[location] = era['power_' + location].sum() print(location, annual_era[location]) annual_midas[location] = midas['power_' + location].sum() max_era = max(annual_era.values()) max_midas = max(annual_midas.values()) print(max_era, max_midas) # PV rated power in kW ????
era5_temp.index = pd.DatetimeIndex(pd.to_datetime(era5_temp.index).date) print('ERA5') # print(era5_temp.index) print(era5_temp) gas_temp = readers.read_gas(gas_temp_filename) print('GAS') print(gas_temp.index) print(gas_temp) met_temp = readers.read_hadcet(met_temp_filename) print('MET') print(met_temp.index) print(met_temp) # output plots era5_temp.plot(label='ERA5') gas_temp.plot(label='Gas') met_temp.plot(label='Met') plt.title('2018 UK mean daily temperature from different sources') plt.xlabel('Day of the year') plt.ylabel('Temperature (Degrees C)') plt.legend(loc='upper right') plt.show() stats.print_stats_header() stats.print_stats(era5_temp, met_temp, 'ERA5 Met') stats.print_stats(era5_temp, gas_temp, 'Gas Met')
offshore.plot(label='offshore') blackpool.plot(label='blackpool') plt.title('2018 Daily mean Wind speed Blackpool Squires Gate and Offshore') plt.xlabel('Day of the year') plt.ylabel('Wind Speed (M/S)') plt.legend(loc='upper right') plt.show() # convert to daily by summing the hours, so this is energy/day in kWh midas = midas.resample('D').sum() marine = marine.resample('D').sum() # output power plots power_m = marine['power_m'] * 0.001 power_b = midas['power_b'] * 0.001 power_m.plot(label='Offshore') power_b.plot(label='Blackpool') plt.title('2018 wind power from Blackpool Squires gate and Offshore') plt.xlabel('Day of the year') plt.ylabel('Energy generated (MWh)') plt.legend(loc='upper right') plt.show() stats.print_stats_header() stats.print_stats(marine['wind_m'], midas['wind_b'], 'blackpool and offshore wind') stats.print_stats(power_m, power_b, 'offshore and blackpool power') print('Annual generation blackpool : {} offshore {}'.format(power_b.sum(), power_m.sum() ) )
gas_energy = gas_energy - (non_heat_gas / 365.0) # print(gas_energy.head(7)) # print('GAS ENERGY SMALLEST') # print(gas_energy.nsmallest()) # account for boiler efficiency of 90% # (this is inconsistent as 80% is used for the EU buildings figures) gas_energy = gas_energy * 0.8 total_gas_energy = gas_energy.sum() print('Total Thermal {} Eq {} Gas {}'.format(total_thermal, total_eq, total_gas_energy)) # compute R2 and correlation stats.print_stats_header() stats.print_stats(eq_space, gas_energy, 'EQuest') stats.print_stats(thermal, gas_energy, 'Thermal XL') # output plots gas_energy.plot(label='Gas Energy') eq_space.plot(label='Equest') thermal.plot(label='Thermal XL') plt.title('Comparison of Heat Demand Methods') plt.xlabel('Day of the year') plt.ylabel('Heat Demand (MWh)') plt.legend(loc='upper right') plt.show() # load duration curves
era5_temp = readers.read_loctemp(era_filename) # resample from hourly to daily print('ERA5') # print(era5_temp.index) print(era5_temp) # output plots era5_temp['t_interp'].plot(label='t_interp') era5_temp['t_weighted'].plot(label='t_weighted') era5_temp['t_min_min'].plot(label='t_(x1,y1)') era5_temp['t_min_max'].plot(label='t_(x2,y1)') era5_temp['t_max_min'].plot(label='t_(x1,y2)') era5_temp['t_max_max'].plot(label='t_(x2,y2)') midas_temp.plot(label='MIDAS') midas_hourly['air_temperature'].plot(label='MIDAS Hourly') plt.title('2018 Blackpool mean daily temperature from different sources') plt.xlabel('Day of the year') plt.ylabel('Temperature (Degrees C)') plt.legend(loc='upper right') plt.show() stats.print_stats_header() stats.print_stats(era5_temp['t_weighted'], midas_temp, 't_weighted Midas') stats.print_stats(era5_temp['t_interp'], midas_temp, 't_interp Midas') stats.print_stats(era5_temp['t_min_min'], midas_temp, 't_min_min Midas') stats.print_stats(era5_temp['t_min_max'], midas_temp, 't_min_max Midas') stats.print_stats(era5_temp['t_max_min'], midas_temp, 't_max_min Midas') stats.print_stats(era5_temp['t_max_max'], midas_temp, 't_max_max Midas')
def main(): parser, options, arguments = parse_options() #print "Options:", options, dir(options) #print "Arguments:", arguments #print "largs:", parser.largs #print "rargs:", parser.rargs if options.show_version: print("Locust %s" % (version)) sys.exit(0) locustfile = find_locustfile(options.locustfile) if not locustfile: print "Could not find any locustfile!" sys.exit(1) docstring, locusts = load_locustfile(locustfile) if options.list_commands: print "Available Locusts:" for name in locusts: print " " + name sys.exit(0) if not locusts: sys.stderr.write("No Locust class found!\n") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: sys.stderr.write("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.show_task_ratio: print "\n Task ratio per locust class" print "-" * 80 inspectlocust.print_task_ratio(locust_classes) print "\n Total task ratio" print "-" * 80 inspectlocust.print_task_ratio(locust_classes, total=True) sys.exit(0) if options.show_task_ratio_confluence: print "\nh1. Task ratio per locust class" print inspectlocust.print_task_ratio_confluence(locust_classes) print "\nh1. Total task ratio" print inspectlocust.print_task_ratio_confluence(locust_classes, total=True) sys.exit(0) # if --master is set, implicitly set --web if options.master: options.web = True if options.web and not options.slave: # spawn web greenlet print "Starting web monitor on port 8089" main_greenlet = gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.ramp) # enable/disable gzip in WebLocust's HTTP client WebLocust.gzip = options.gzip if not options.master and not options.slave: core.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host) # spawn client spawning/hatching greenlet if not options.web: core.locust_runner.start_hatching(wait=True) main_greenlet = core.locust_runner.greenlet elif options.master: core.locust_runner = MasterLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host) elif options.slave: core.locust_runner = SlaveLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host) main_greenlet = core.locust_runner.greenlet if options.ramp: import rampstats from rampstats import on_request_success, on_report_to_master, on_slave_report import events if options.slave: events.report_to_master += on_report_to_master if options.master: events.slave_report += on_slave_report else: events.request_success += on_request_success if options.print_stats or (not options.web and not options.slave): # spawn stats printing greenlet gevent.spawn(stats_printer) try: print print "Starting Locust %s" % version print main_greenlet.join() except KeyboardInterrupt, e: time.sleep(0.2) print "\n" print_stats(core.locust_runner.request_stats) print_percentile_stats(core.locust_runner.request_stats) print_error_report() print print "Exiting, bye.." print
def stats_printer(): while True: print_stats(statistics) gevent.sleep(3)
if gas_method == 'c': gas_energy = gas_energy_comp else: if gas_method == 'a': gas_energy = gas_energy_temp else: # (n) method in the IECSF20 paper gas_energy = gas_energy_const print('Gas method {} heat energy total {}'.format(gas_method, gas_energy.sum())) # compute R2 and correlation stats.print_stats_header() stats.print_stats(space_and_waterR, gas_energy, 'BDEW', 2, True) stats.print_stats(space_and_waterW, gas_energy, 'Watson', 1, True) stats.print_stats(space_and_waterH, gas_energy, 'HDD 12.8', 1, True) stats.print_stats(space_and_waterS, gas_energy, 'HDD 15.5 ', 1, True) # output plots gas_energy_comp.plot(label='Heat Demand Gas (compromise)', color='blue') gas_energy_const.plot(label='Heat Demand Gas (not temp)', color='blue', style='--') gas_energy_temp.plot(label='Heat Demand Gas (all temp)', color='blue', style='.') #gas_energy_temp.plot(label='Heat Demand Gas', color='blue') space_and_waterR.plot(label='Heat Demand BDEW', color='red')
#!/usr/bin/python import scatter import stats import sys (name1, bench1) = stats.load(sys.argv[1]) (name2, bench2) = stats.load(sys.argv[2]) aggregate = stats.compare(bench1, bench2) stats.print_stats(name1, name2, aggregate) times = scatter.load(sys.argv[1], sys.argv[2]) scatter.plot(name1, name2, times).show()