def process_log(args):
with open(args.e2e_file, 'r') as infile:
lines = infile.readlines()
batches = {}
for line in lines:
group_id = int(line.split("group_id=")[1].split(',')[0])
batch_id = int(line.split("batch_id=")[1].split(',')[0])
reconstruction = line.split("is_reconstruction=")[1].split(
',')[0] == "true"
latency = int(line.split("=")[-1])
if batch_id not in batches:
batches[batch_id] = Batch()
batches[batch_id].add_query(latency, reconstruction)
if args.batch_size > 1:
for _, b in batches.items():
if len(b.latencies) > 1:
b.latencies = [b.latencies[-1]]
if len(b.decoded_latencies) > 1:
b.decoded_latencies = [b.decoded_latencies[-1]]
og_latencies = []
for _, b in batches.items():
og_latencies.extend(b.latencies)
og_med = np.median(og_latencies)
latencies = []
for _, b in batches.items():
if b.decoded:
if args.mode == "coded":
latencies.extend(b.decoded_latencies)
elif args.mode == "cheap":
if len(b.latencies) == 0:
# If we don't have logs for "normal" latencies, then those
# batches were likely removed from the queue before
# processing due to a completed redundant task. In that
# case, we just take the decoded latencies.
latencies.extend(b.decoded_latencies)
else:
# For cheap and replication, the "decoded" latencies are the
# minimum between the two latencies. We perform the max
# operation between the approximate result and the median
# latency so as not to log returning an approximate result
# if that result was returned too soon. This simulates
# having an aggressive timeout of median latency.
mins = [
min(og, max(og_med, d))
for og, d in zip(b.latencies, b.decoded_latencies)
]
latencies.extend(mins)
else:
latencies.extend(b.latencies)
# Convert to ms
latencies = [x / 1000 for x in latencies]
print_stats(latencies)
def MonteCarlo(mesh,system,potential_field,electric_field,
density_funcs,
avg_dens,
avg_electrons,
avg_holes,
current_values):
reaper = []
current = 0
#next_step density function array
nextDensity = density_funcs.combined_density.vector().array().astype('int')
#holeDensity = density_funcs.holes.vector().array()
#electronDensity = density_funcs.electrons.vector().array()
#compute field, move_particles, calc current, recombinate, reap
start = time.time()
print "computing field"
c_efield = pre_compute_field(mesh,electric_field) #Evaluates field at each mesh point
print "nextDensity max:",max(nextDensity)
start =time.time()
print "About to move particles"
c_efield = array(c_efield)
#print max(map(lambda x:max(abs(x[0]),abs(x[1])),c_efield)),\
# min(map(lambda x:min(abs(x[0]),abs(x[1])),c_efield))
#raw_input()
move_particles_c.move_particles(system,
c_efield,nextDensity,
mesh.dt,mesh.length_scale)
print "About to update"
current = move_particles_c.update_density(system,
nextDensity,
mesh.kdt)
print "Recombination"
move_particles_c.recombinate(system,nextDensity,mesh.kdt)
print current
print "time:",time.time()-start
start = time.time()
current += move_particles_c.replenish(system,nextDensity)
print "replenish:",time.time()-start
#reap_list(mesh.particles,reaper)
#update current values
current_values.append(current*constants.eC/
(mesh.dt*mesh.gen_num*mesh.num_vertices()))
#update avg_dens
scaled_density = calculate_scaled_density(mesh,nextDensity)
avg_dens.inc(scaled_density)
density_funcs.combined_density.vector()[:]= nextDensity.astype('double')
density_funcs.scaled_density.vector()[:]=scaled_density
print density_funcs.scaled_density.vector().array()
#print "doom",density_funcs.combined_density.vector().array()
#print stats
stats.print_stats(current)
def shutdown(code=0):
"""
Shut down locust by firing quitting event, printing stats and exiting
"""
logger.info("Shutting down, bye..")
events.quitting.fire()
print_stats(runners.locust_runner.request_stats)
print_percentile_stats(runners.locust_runner.request_stats)
print_error_report()
sys.exit(code)
def run_dataset(dataset):
results = {}
for benchmark in BENCHMARKS:
times = []
for i in range(RUNS):
elapsed_time = submit_spark_job(dataset, benchmark)
times.append(elapsed_time)
print_stats(times)
results[benchmark] = times
return results
def shutdown(code=0):
"""
Shut down locust by firing quitting event, printing stats and exiting
"""
logger.info("Shutting down, bye..")
events.quitting.fire()
print_stats(runners.locust_runner.request_stats)
print_percentile_stats(runners.locust_runner.request_stats)
print_error_report()
sys.exit(code)
def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False):
if spawn_count is None:
spawn_count = self.num_clients
if self.num_requests is not None:
RequestStats.global_max_requests = self.num_requests
bucket = self.weight_locusts(spawn_count, stop_timeout)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
self.num_clients = spawn_count
else:
self.num_clients += spawn_count
logger.info(
"Hatching and swarming %i clients at the rate %g clients/s..." %
(spawn_count, self.hatch_rate))
occurence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info("All locusts hatched: %s" % ", ".join([
"%s: %d" % (name, count)
for name, count in occurence_count.iteritems()
]))
events.hatch_complete.fire(self.num_clients)
return
locust = bucket.pop(random.randint(0, len(bucket) - 1))
occurence_count[locust.__name__] += 1
def start_locust(_):
try:
locust()()
except RescheduleTaskImmediately:
pass
except GreenletExit:
pass
new_locust = self.locusts.spawn(start_locust, locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
print_stats(self.request_stats)
print_percentile_stats(
self.request_stats
) #TODO use an event listener, or such, for this?
def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False):
if spawn_count is None:
spawn_count = self.num_clients
if self.num_requests is not None:
RequestStats.global_max_requests = self.num_requests
bucket = self.weight_locusts(spawn_count, stop_timeout)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
self.num_clients = spawn_count
else:
self.num_clients += spawn_count
logger.info("Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate))
occurence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info(
"All locusts hatched: %s"
% ", ".join(["%s: %d" % (name, count) for name, count in occurence_count.iteritems()])
)
events.hatch_complete.fire(self.num_clients)
return
locust = bucket.pop(random.randint(0, len(bucket) - 1))
occurence_count[locust.__name__] += 1
def start_locust(_):
try:
locust()()
except RescheduleTaskImmediately:
pass
except GreenletExit:
pass
new_locust = self.locusts.spawn(start_locust, locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
print_stats(self.request_stats)
print_percentile_stats(self.request_stats) # TODO use an event listener, or such, for this?
def do_threading(stats, ivy_filename, json_filename, logdir, nthreads,
main_args, breadth_args, finisher_args, by_size):
print(ivy_filename)
print("json: ", json_filename)
invfile = None
success = False
if not success and len(breadth_args) > 0:
success, invfile = do_breadth("iteration", logdir, nthreads,
json_filename, main_args,
main_args + breadth_args, invfile, stats,
by_size)
if success:
print("Invariant success!")
if not success and len(finisher_args) > 0:
success = do_finisher("finisher", logdir, nthreads, json_filename,
main_args, main_args + finisher_args, invfile,
stats)
if success:
print("Invariant success!")
statfile = logdir + "/summary"
stats.add_global_stat_files(all_stats_files)
stats.print_stats(statfile)
print("")
print("statfile: " + statfile)
print("")
print("======== Summary ========")
print("")
with open(statfile, "r") as f:
t = f.read()
t = t.split("specifics:")
print(t[0])
final_inv_file = logdir + "/invariants"
with open(final_inv_file, "w") as f:
if stats.was_success():
f.write("# Success: True\n")
else:
f.write("# Success: False\n")
for inv in stats.get_invariants():
f.write("conjecture " +
protocol_parsing.value_json_to_string(inv) + "\n")
def main():
args = parse_arguments()
tga = TGAParser(args.inputfile)
if args.encode:
bitmap = tga.get_bitmap()
low, low_encoded, high, high_quantified = encode(bitmap, args.k)
tga.new_tga(low, "low.tga")
tga.new_tga_from_bytes(low_encoded, "low_encoded")
tga.new_tga(high, "high.tga")
tga.new_tga(high_quantified, "high_quantified.tga")
if not args.silent:
print("\033[4m", "LOW PASS FILTER", "\033[0m")
print_stats(bitmap, low)
print("\n\033[4m" "HIGH PASS FILTER, QUANTIFIED", "\033[0m")
print_stats(bitmap, high_quantified)
else:
tga.new_tga(decode(tga.bitmap, tga.width, tga.height),
"low_decoded.tga")
daily_mod_electric_ref.plot(color='yellow',
label='Synthetic time series with 2017 weather')
#daily_electric_2009.plot(color='orange', label='Electricity demand time series 2009')
plt.title('Daily electricty demand series from weather')
plt.xlabel('day', fontsize=15)
plt.ylabel('Energy (Mwh) per day', fontsize=15)
plt.legend(loc='upper center')
plt.show()
# compare weekly series
print('Weekly Comparison to 2017 electricity demand')
stats.print_stats_header()
stats.print_stats(electric_2018.resample('W').sum(),
electric_2017.resample('W').sum(),
'2018 Electric',
nvars=1,
plot=args.plot,
xl='Electricity demand 2018',
yl='Electricity demand 2017')
stats.print_stats(mod_electric_ref.resample('W').sum(),
electric_2017.resample('W').sum(),
'2017 From 2018 series and weather',
nvars=1,
plot=args.plot,
xl='Electricity demand synthetic',
yl='Electricity demand 2017')
# compare hourly series
print('Hourly Comparison to 2017 electricity demand')
stats.print_stats_header()
stats.print_stats(electric_2018,
# python script to compare 2 csv time series files # usage: python comparecsv.py file1 file2 import sys from pandas import read_csv import statsmodels.api as sm import stats # get command line. file1 = sys.argv[1] file2 = sys.argv[2] field = sys.argv[3] print(file1, file2) data1 = read_csv(file1, header=0, sep=',', decimal='.', parse_dates=[0], index_col=0, squeeze=True, usecols=['time','space','heat',field] ) data2 = read_csv(file2, header=0, sep=',', decimal='.', parse_dates=[0], index_col=0, squeeze=True, usecols=['time','space','heat',field] ) print(data1.head(7)) print(data2.head(7)) stats.print_stats_header() stats.print_stats(data1[field], data2[field], 'oldVnew')
'EMBEDDED_SOLAR_GENERATION']
electric_total = electric_total * (gas_energy.sum() / electric_total.sum())
print(electric_total.head())
print(electric_total.tail())
# convert all to net demand
gas_energy = gas_energy - electric_total
space_and_water = space_and_water - electric_total
watson_space_and_water = watson_space_and_water - electric_total
hdd_space_and_water15 = hdd_space_and_water15 - electric_total
hdd_space_and_water12 = hdd_space_and_water12 - electric_total
# compute R2 and correlation
stats.print_stats_header()
stats.print_stats(space_and_water, gas_energy, 'Ruhnau ')
stats.print_stats(watson_space_and_water, gas_energy, 'Watson ')
stats.print_stats(hdd_space_and_water15, gas_energy, 'HDD 15.5')
stats.print_stats(hdd_space_and_water12, gas_energy, 'HDD 12.8')
# largest and smallest diffs
stats.print_large_small(hdd_space_and_water15, gas_energy, 'HDD 15.5')
# output plots
gas_energy.plot(label='Gas Energy')
space_and_water.plot(label='Heat Demand BDEW')
watson_space_and_water.plot(label='Heat Demand Watson')
hdd_space_and_water15.plot(label='Heat Demand HDD 15.5')
hdd_space_and_water12.plot(label='Heat Demand HDD 12.8')
closed_set.add(current)
for new_board in current.generate_moves():
if new_board in closed_set:
continue
tentative_g_score = g_score[current] + dist_between(
current, new_board)
if tentative_g_score >= g_score[new_board]:
continue
came_from[new_board] = current
g_score[new_board] = tentative_g_score
f_score = tentative_g_score + heuristic_cost_estimate(
new_board, desired_board)
open_items.put((f_score, new_board))
def run(_):
board = random_start_board()
return astar(board)
if __name__ == '__main__':
N = 20
with Pool(10) as p:
results = p.map(run, range(N))
print_stats(results)
# start streaming
queue = get_queue()
http_session = get_http_session()
session_id = session_data["id"]
# first queue reader
callback_storage = lambda data: store_tracking_session_location_element(
get_http_session(), session_id, data)
start_daemon_process(queue_reader, (
callback_storage,
queue,
))
# setup pozyx
localizer = setup_poszyx(session_data)
# write
start_daemon_process(read_poszyx, (
queue,
localizer,
))
# cli
cli(queue)
session_data["stop_time"] = datetime.datetime.utcnow().isoformat()
session_data["data"] = get_session_data(session_id)
print(print_stats(session_data))
# ninja.plot(label='ninja')
power_era_b.plot(label='era5')
power_mid_b.plot(label='midas')
power_nin_b.plot(label='ninja')
plt.title('2018 Blackpool wind power from midas and ERA5 and ninja')
plt.xlabel('Day of the year')
plt.ylabel('Energy generated (MWh)')
plt.legend(loc='upper right')
plt.show()
#ninja.index = ninja.index.tz_localize('Europe/London')
ninja.index = ninja.index.tz_localize('UTC')
stats.print_stats_header()
stats.print_stats(wind_era, wind_midas, 'era and midas wind', 1, True, 'ERA5 wind speed (m/s)', 'MIDAS wind speed (m/s)' )
stats.print_stats(era['power_b'], midas['power_b'], 'era and midas power', 1, True, 'ERA Energy (MWh)', 'MIDAS Energy (MWh)' )
stats.print_stats(era['power_b'], ninja, 'era and ninja power', 1, True, 'ERA Energy (MWh)', 'Ninja Energy (MWh)' )
annual_era = {}
annual_midas = {}
locations = ['a', 'b', 'c', 'l', 's', 'w']
for location in locations:
annual_era[location] = era['power_' + location].sum()
print(location, annual_era[location])
annual_midas[location] = midas['power_' + location].sum()
max_era = max(annual_era.values())
max_midas = max(annual_midas.values())
# print(max_era, max_midas)
total_synthetic = daily_synthetic.sum()
percent = (total_real - total_synthetic) / total_real
mean_factor = daily_real.mean() / daily_synthetic.mean()
print("Total Real {} Synthetic {} Percentage {} Mean Factor {}".format(total_real, total_synthetic, percent, mean_factor) )
#fixed_synthetic = daily_synthetic * mean_factor
#fixed_synthetic = (daily_synthetic * 0.4669) + 38.36
#fixed_synthetic = (daily_synthetic / 0.4669) - 38.36
#fixed_synthetic = (daily_synthetic - 38.36) / 0.4669
#fixed_synthetic = (daily_synthetic - 38.36) * 0.4669
#fixed_synthetic = (daily_synthetic + 38.36) * 0.4669
fixed_synthetic = (daily_synthetic * 1.4669) + 38.36
# compute R2 and correlation
stats.print_stats_header()
stats.print_stats(daily_synthetic, daily_real, 'Electric Daily', 1, True)
stats.print_stats(fixed_synthetic * mean_factor, daily_real, 'Electric Fixed', 1, True)
# output plots
#print('COLUMNS')
#print(dfd.columns)
daily_synthetic.plot(label='Modelled Electricity Time Series', color='blue')
daily_real.plot(label='Actual Measured Electricity Time Series', color='red')
fixed_synthetic.plot(label='Fixed Electricity Time Series', color='green')
plt.title('Comparison of predicted electricty demand and measured from heat pumps')
plt.xlabel('Day of the year', fontsize=15)
plt.ylabel('Electricity Demand (kWh)', fontsize=15)
plt.legend(loc='upper right', fontsize=15)
plt.show()
tower['wind70'].plot(label='Tower Wind Speed at 70m')
tower['log70'].plot(label='Log law from 21m to 70m')
tower['power70'].plot(label='Power law from 21m to 70m')
era5['log70'].plot(label='ERA5 Log law from 10m to 70m')
era5['power70'].plot(label='ERA5 Power law using 2 heights 70m')
plt.title('Amsterdam Offshore Tower windspeed for December 2010')
plt.xlabel('Day of the month')
plt.ylabel('Wind Speed (M/S)')
plt.legend(loc='upper right')
plt.show()
# output stats for 70m
stats.print_stats_header()
stats.print_stats(tower['log70'], tower['wind70'], 'Log law 70m', 1, True,
'Log law wind speed (m/s)', 'Actual wind speed (m/s)')
stats.print_stats(tower['power70'], tower['wind70'], 'Power law 70m', 1, True,
'Power law wind speed (m/s)', 'Actual wind speed (m/s)')
stats.print_stats(era5['log70'], tower['wind70'], 'ERA5 log law', 1, True,
'Power law wind speed (m/s)', 'Actual wind speed (m/s)')
stats.print_stats(era5['power70'], tower['wind70'], 'ERA5 Power law', 1, True,
'Power law wind speed (m/s)', 'Actual wind speed (m/s)')
# output wind plots for 116m
tower['wind116'].plot(label='Tower Wind Speed at 116m')
tower['log116'].plot(label='Log law from 21m to 116m')
tower['power116'].plot(label='Power law from 21m to 116m')
era5['log116'].plot(label='ERA5 Log law from 10m to 116m')
era5['power116'].plot(label='ERA5 Power law using 2 heights 116m')
plt.title('Amsterdam Offshore Tower windspeed for December 2010')
def print_results(results):
for dataset in DATASETS:
for benchmark in BENCHMARKS:
times = results[dataset][benchmark]
print(f"\n{benchmark} on {dataset} times:")
print_stats(times)
plt.ylabel('Demand (MWh)')
plt.legend(loc='upper right')
plt.show()
# plot the values so we see how similar they are
plt.plot(demand19.values, label='2019')
plt.plot(demand18.values, label='2018')
plt.plot(demand17.values, label='2017')
plt.plot(demand16.values, label='2016')
plt.plot(demand15.values, label='2015')
plt.title('England and Wales Electricity Demand')
plt.xlabel('Hour of the year')
plt.ylabel('Demand (MWh)')
plt.legend(loc='upper right')
plt.show()
# drop the last 24 hours from 2016, (leap year) so they are the same length
demand16.drop(demand16.tail(24).index, inplace=True)
print('2019 {} 2018 {} 2017 {} 2016 {} 2015 {}'.format(len(demand19),
len(demand18),
len(demand17),
len(demand16),
len(demand15)))
stats.print_stats_header()
stats.print_stats(demand18, demand19, '2018')
stats.print_stats(demand17, demand19, '2017')
stats.print_stats(demand16, demand19, '2016')
stats.print_stats(demand15, demand19, '2015')
destinee_filename = '/home/malcolm/uclan/data/destinee/heat.csv'
destinee_heat = readers.read_destinee(destinee_filename)
print(destinee_heat)
# convert from TJ to Kwh then to GWh
# destinee_electric_heat = destinee_heat['electric'] * 277777.77777778 * 1e-6
# convert from TWh to GWh
destinee_electric_heat = destinee_heat['electric'] * 1e+3
print('Electric heat annual mine: {} destinee: {}'.format(
heat.sum(), destinee_electric_heat.sum()))
daily_heat = heat.resample('D').sum()
daily_destinee_electric_heat = destinee_electric_heat.resample('D').sum()
# daily plot of historic and electric
daily_heat.plot(label='BDEW')
daily_destinee_electric_heat.plot(label='DESSTINEE')
plt.title('Daily UK Electric Heat 2010 BDEW and DESSTINEE')
plt.xlabel('Day of the year')
plt.ylabel('Demand (MWh)')
plt.legend(loc='upper right')
plt.show()
#print(daily_heat.index)
#print(daily_destinee_electric_heat.index)
daily_destinee_electric_heat.index = daily_heat.index
stats.print_stats_header()
stats.print_stats(daily_heat, daily_destinee_electric_heat, 'DESSTINEE', 2,
True)
def main():
all_instances = read_imp_arg_dataset(sys.argv[1])
# print_imp_arg_dataset(all_instances, 'test_annotations.txt')
print '\nLoading predicate dict from {}'.format(predicate_dict_path)
predicate_dict = pkl.load(open(predicate_dict_path, 'r'))
# print_imp_arg_dataset_by_pred(all_instances, predicate_dict, 'ia_by_pred')
treebank_reader = TreebankReader()
propbank_reader = PropbankReader()
propbank_reader.build_index()
nombank_reader = NombankReader()
nombank_reader.build_index()
if not exists(corenlp_dict_path):
print '\nNo existing CoreNLP dict found'
corenlp_reader = CoreNLPReader.build(all_instances)
corenlp_reader.save(corenlp_dict_path)
else:
corenlp_reader = CoreNLPReader.load(corenlp_dict_path)
if not exists(all_predicates_path):
all_predicates = []
for instance in all_instances:
predicate = Predicate.build(instance)
predicate.set_pred(predicate_dict[str(predicate.pred_pointer)])
all_predicates.append(predicate)
print '\nChecking explicit arguments with Nombank instances'
for predicate in all_predicates:
nombank_instance = nombank_reader.search_by_pointer(
predicate.pred_pointer)
predicate.check_exp_args(nombank_instance,
add_missing_args=False,
remove_conflict_imp_args=False,
verbose=False)
print '\nParsing all implicit and explicit arguments'
for predicate in tqdm(all_predicates, desc='Processed', ncols=100):
predicate.parse_args(treebank_reader, corenlp_reader)
print '\nSaving all parsed predicates to {}'.format(
all_predicates_path)
pkl.dump(all_predicates, open(all_predicates_path, 'w'))
else:
print '\nLoading all parsed predicates from {}'.format(
all_predicates_path)
start_time = timeit.default_timer()
all_predicates = pkl.load(open(all_predicates_path, 'r'))
elapsed = timeit.default_timer() - start_time
print '\tDone in {:.3f} seconds'.format(elapsed)
# check_multi_pobj(all_predicates)
if not exists(candidate_dict_path):
print '\nBuilding candidate dict from Propbank and Nombank'
candidate_dict = CandidateDict(propbank_reader=propbank_reader,
nombank_reader=nombank_reader,
corenlp_reader=corenlp_reader,
max_dist=max_candidate_dist)
for predicate in tqdm(all_predicates, desc='Processed', ncols=100):
candidate_dict.add_candidates(predicate.pred_pointer)
candidate_dict.save(candidate_dict_path)
else:
candidate_dict = CandidateDict.load(candidate_dict_path,
propbank_reader=propbank_reader,
nombank_reader=nombank_reader,
corenlp_reader=corenlp_reader,
max_dist=max_candidate_dist)
# candidate_dict.print_all_candidates('all_candidates.txt')
print '\nAdding candidates to predicates'
for predicate in all_predicates:
for candidate in candidate_dict.get_candidates(predicate.pred_pointer):
predicate.candidates.append(candidate)
# print_all_predicate(all_predicates, 'all_predicates.txt', verbose=True,
# include_candidates=True, include_dice_scores=True,
# corenlp_reader=corenlp_reader)
if not exists(all_rich_predicates_path):
all_rich_predicates = []
for predicate in all_predicates:
rich_predicate = RichPredicate.build(predicate,
corenlp_reader,
use_lemma=True,
use_entity=True,
use_corenlp_tokens=True)
all_rich_predicates.append(rich_predicate)
print '\nSaving all rich predicates to {}'.format(
all_rich_predicates_path)
pkl.dump(all_rich_predicates, open(all_rich_predicates_path, 'w'))
print '\nPrinting statistics'
print_stats(all_predicates)
def main():
parser, options, arguments = parse_options()
#print "Options:", options, dir(options)
#print "Arguments:", arguments
#print "largs:", parser.largs
#print "rargs:", parser.rargs
if options.show_version:
print("Locust %s" % (version))
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
print "Could not find any locustfile!"
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
print "Available Locusts:"
for name in locusts:
print " " + name
sys.exit(0)
if not locusts:
sys.stderr.write("No Locust class found!\n")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
sys.stderr.write("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.web and not options.slave:
# spawn web greenlet
print "Starting web monitor on port 8089"
gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests)
if not options.master and not options.slave:
core.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host)
# spawn client spawning/hatching greenlet
if not options.web:
core.locust_runner.start_hatching()
elif options.master:
core.locust_runner = MasterLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, redis_host=options.redis_host, redis_port=options.redis_port)
elif options.slave:
core.locust_runner = SlaveLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, redis_host=options.redis_host, redis_port=options.redis_port)
if not options.web or options.print_stats:
# spawn stats printing greenlet
gevent.spawn(stats_printer)
try:
print ""
print "Starting Locust %s" % version
print ""
core.locust_runner.greenlet.join()
except KeyboardInterrupt, e:
print_stats(core.locust_runner.request_stats)
print_percentile_stats(core.locust_runner.request_stats)
print ""
print "Exiting, bye.."
print ""
def print_stats(self, verbose=0):
print_stats(self.all_predicates, verbose=verbose)
def main():
parser, options, arguments = parse_options()
# setup logging
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
if options.show_version:
print "Locust %s" % (version)
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error("Could not find any locustfile! See --help for available options.")
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
print "Available Locusts:"
for name in locusts:
print " " + name
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info( "-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
# if --master is set, make sure --no-web isn't set
if options.master and options.no_web:
logger.error("Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)")
sys.exit(0)
if not options.no_web and not options.slave:
# spawn web greenlet
logger.info("Starting web monitor on port 8089")
main_greenlet = gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.ramp)
# enable/disable gzip in WebLocust's HTTP client
WebLocust.gzip = options.gzip
if not options.master and not options.slave:
runners.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host)
# spawn client spawning/hatching greenlet
if options.no_web:
runners.locust_runner.start_hatching(wait=True)
main_greenlet = runners.locust_runner.greenlet
elif options.master:
runners.locust_runner = MasterLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host)
elif options.slave:
runners.locust_runner = SlaveLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host)
main_greenlet = runners.locust_runner.greenlet
if options.print_stats or (options.no_web and not options.slave):
# spawn stats printing greenlet
gevent.spawn(stats_printer)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
except KeyboardInterrupt, e:
events.quitting.fire()
time.sleep(0.2)
print_stats(runners.locust_runner.request_stats)
print_percentile_stats(runners.locust_runner.request_stats)
print_error_report()
logger.info("Got KeyboardInterrupt. Exiting, bye..")
# subtracting the non heat gas and divide evenly amongst the days.
# assuming that the non-gas energy is constant.
# (if its not, this could go negative)
gas_energy = gas_energy - (non_heat_gas / 365.0)
# print(gas_energy.head(7))
print('GAS ENERGY SMALLEST')
print(gas_energy.nsmallest())
# account for boiler efficiency of 90%
# (this is inconsistent as 80% is used for the EU buildings figures)
gas_energy = gas_energy * 0.8
# compute R2 and correlation
stats.print_stats_header()
stats.print_stats(space_and_water, gas_energy, 'HDD 15.5 ')
stats.print_stats(space_and_waterS1, gas_energy, 'HDD 15.5 S1 ')
stats.print_stats(space_and_waterS2, gas_energy, 'HDD 15.5 S2 ')
# output plots
gas_energy.plot(label='Gas Energy')
space_and_water.plot(label='Heat Demand HDD 15.5')
space_and_waterS1.plot(label='Heat Demand HDD 15.5 S1')
space_and_waterS2.plot(label='Heat Demand HDD 15.5 S2')
plt.title('Comparison of Heat Demand Methods')
plt.xlabel('Day of the year')
plt.ylabel('Heat Demand (MWh)')
plt.legend(loc='upper right')
plt.show()
],
)
df = df.append(csv)
total = total - 1
df.dtypes[0] = "UInt64"
return df
def run_benchmark(df):
start = time.time()
df.describe(include="all")
elapsed = time.time() - start
print(f"Elapsed time: {elapsed}")
return elapsed
RUNS = 5
results = {}
for dataset in DATASETS:
times = []
df = read_data(dataset)
for i in range(RUNS):
times.append(run_benchmark(df))
results[dataset] = times
for dataset in DATASETS:
print(f"Results for dataset {dataset}")
print_stats(results[dataset])
queries = queries_file.read().replace("\n", "").split(";")
queries.pop(-1) # remove trailing empty element
# If kerberos is enabled, make sure we have the sasl library
if args.kerberos:
try:
import sasl
except ImportError:
print("You need the sasl library in order to use kerberos")
sys.exit(1)
scheduler = ImpalaQueryScheduler(
queries, args.threads, args.impala_hosts.split(","), args.stats_port, args.kerberos, args.kerberos_service
)
# Keep an empty place holder for the final stats
stats = None
def signal_handler(signal, frame):
global stats
stats = get_stats_tables(url="http://localhost:%d" % args.stats_port)
scheduler.shutdown()
while not scheduler.finished:
time.sleep(0.10)
scheduler.start()
signal.signal(signal.SIGINT, signal_handler)
signal.pause()
print("\n\n")
print_stats(stats)
# main program
# read in the data
output_dir = "/home/malcolm/uclan/data/advanced_metering/testing/"
filename = output_dir + 'methods.csv'
df = pd.read_csv(filename, header=0, sep=',', parse_dates=[0], index_col=0, squeeze=True)
# convert from Wh to kWh
df = df * 0.001
# compute R2 and correlation
stats.print_stats_header()
stats.print_stats(df['B'], df['gas'], 'BDEW', 2, True)
stats.print_stats(df['W'], df['gas'], 'Watson', 1, True)
stats.print_stats(df['H'], df['gas'], 'HDD 12.8', 1, True)
stats.print_stats(df['S'], df['gas'], 'HDD 15.5 ', 1, True)
stats.monthly_stats_header()
m_b = stats.monthly_stats(df['B'], df['gas'], 'BDEW')
m_w = stats.monthly_stats(df['W'], df['gas'], 'Watson')
m_h = stats.monthly_stats(df['H'], df['gas'], 'HDD 12.8')
m_s = stats.monthly_stats(df['S'], df['gas'], 'HDD 15.5')
plt.plot(range(12), m_b, label='BDEW')
plt.plot(range(12), m_w, label='Watson')
plt.plot(range(12), m_h, label='HDD 12.8')
plt.plot(range(12), m_s, label='HDD 15.5')
plt.xlabel('Month of the year', fontsize=15)
power_nin_c = ninja
#pv_generation_compare.py
power_era_c.plot(label='era5')
power_mid_c.plot(label='midas')
power_nin_c.plot(label='ninja')
plt.title('2018 Cambourne PV power from midas and ERA5 and ninja')
plt.xlabel('Day of the year')
plt.ylabel('Energy generated (MWh)')
plt.legend(loc='upper right')
plt.show()
ninja.index = ninja.index.tz_localize('Europe/London')
stats.print_stats_header()
stats.print_stats(era['ghi_c'], midas['ghi_c'], 'era and midas ghi')
stats.print_stats(era['power_c'], midas['power_c'], 'era and midas power')
#stats.print_stats(era['power_c'], ninja, 'era and ninja power')
annual_era = {}
annual_midas = {}
locations = ['a', 'c', 'e', 'u']
for location in locations:
annual_era[location] = era['power_' + location].sum()
print(location, annual_era[location])
annual_midas[location] = midas['power_' + location].sum()
max_era = max(annual_era.values())
max_midas = max(annual_midas.values())
print(max_era, max_midas)
# PV rated power in kW ????
era5_temp.index = pd.DatetimeIndex(pd.to_datetime(era5_temp.index).date)
print('ERA5')
# print(era5_temp.index)
print(era5_temp)
gas_temp = readers.read_gas(gas_temp_filename)
print('GAS')
print(gas_temp.index)
print(gas_temp)
met_temp = readers.read_hadcet(met_temp_filename)
print('MET')
print(met_temp.index)
print(met_temp)
# output plots
era5_temp.plot(label='ERA5')
gas_temp.plot(label='Gas')
met_temp.plot(label='Met')
plt.title('2018 UK mean daily temperature from different sources')
plt.xlabel('Day of the year')
plt.ylabel('Temperature (Degrees C)')
plt.legend(loc='upper right')
plt.show()
stats.print_stats_header()
stats.print_stats(era5_temp, met_temp, 'ERA5 Met')
stats.print_stats(era5_temp, gas_temp, 'Gas Met')
offshore.plot(label='offshore')
blackpool.plot(label='blackpool')
plt.title('2018 Daily mean Wind speed Blackpool Squires Gate and Offshore')
plt.xlabel('Day of the year')
plt.ylabel('Wind Speed (M/S)')
plt.legend(loc='upper right')
plt.show()
# convert to daily by summing the hours, so this is energy/day in kWh
midas = midas.resample('D').sum()
marine = marine.resample('D').sum()
# output power plots
power_m = marine['power_m'] * 0.001
power_b = midas['power_b'] * 0.001
power_m.plot(label='Offshore')
power_b.plot(label='Blackpool')
plt.title('2018 wind power from Blackpool Squires gate and Offshore')
plt.xlabel('Day of the year')
plt.ylabel('Energy generated (MWh)')
plt.legend(loc='upper right')
plt.show()
stats.print_stats_header()
stats.print_stats(marine['wind_m'], midas['wind_b'], 'blackpool and offshore wind')
stats.print_stats(power_m, power_b, 'offshore and blackpool power')
print('Annual generation blackpool : {} offshore {}'.format(power_b.sum(), power_m.sum() ) )
gas_energy = gas_energy - (non_heat_gas / 365.0)
# print(gas_energy.head(7))
# print('GAS ENERGY SMALLEST')
# print(gas_energy.nsmallest())
# account for boiler efficiency of 90%
# (this is inconsistent as 80% is used for the EU buildings figures)
gas_energy = gas_energy * 0.8
total_gas_energy = gas_energy.sum()
print('Total Thermal {} Eq {} Gas {}'.format(total_thermal, total_eq,
total_gas_energy))
# compute R2 and correlation
stats.print_stats_header()
stats.print_stats(eq_space, gas_energy, 'EQuest')
stats.print_stats(thermal, gas_energy, 'Thermal XL')
# output plots
gas_energy.plot(label='Gas Energy')
eq_space.plot(label='Equest')
thermal.plot(label='Thermal XL')
plt.title('Comparison of Heat Demand Methods')
plt.xlabel('Day of the year')
plt.ylabel('Heat Demand (MWh)')
plt.legend(loc='upper right')
plt.show()
# load duration curves
era5_temp = readers.read_loctemp(era_filename)
# resample from hourly to daily
print('ERA5')
# print(era5_temp.index)
print(era5_temp)
# output plots
era5_temp['t_interp'].plot(label='t_interp')
era5_temp['t_weighted'].plot(label='t_weighted')
era5_temp['t_min_min'].plot(label='t_(x1,y1)')
era5_temp['t_min_max'].plot(label='t_(x2,y1)')
era5_temp['t_max_min'].plot(label='t_(x1,y2)')
era5_temp['t_max_max'].plot(label='t_(x2,y2)')
midas_temp.plot(label='MIDAS')
midas_hourly['air_temperature'].plot(label='MIDAS Hourly')
plt.title('2018 Blackpool mean daily temperature from different sources')
plt.xlabel('Day of the year')
plt.ylabel('Temperature (Degrees C)')
plt.legend(loc='upper right')
plt.show()
stats.print_stats_header()
stats.print_stats(era5_temp['t_weighted'], midas_temp, 't_weighted Midas')
stats.print_stats(era5_temp['t_interp'], midas_temp, 't_interp Midas')
stats.print_stats(era5_temp['t_min_min'], midas_temp, 't_min_min Midas')
stats.print_stats(era5_temp['t_min_max'], midas_temp, 't_min_max Midas')
stats.print_stats(era5_temp['t_max_min'], midas_temp, 't_max_min Midas')
stats.print_stats(era5_temp['t_max_max'], midas_temp, 't_max_max Midas')
def main():
parser, options, arguments = parse_options()
#print "Options:", options, dir(options)
#print "Arguments:", arguments
#print "largs:", parser.largs
#print "rargs:", parser.rargs
if options.show_version:
print("Locust %s" % (version))
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
print "Could not find any locustfile!"
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
print "Available Locusts:"
for name in locusts:
print " " + name
sys.exit(0)
if not locusts:
sys.stderr.write("No Locust class found!\n")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
sys.stderr.write("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.show_task_ratio:
print "\n Task ratio per locust class"
print "-" * 80
inspectlocust.print_task_ratio(locust_classes)
print "\n Total task ratio"
print "-" * 80
inspectlocust.print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_confluence:
print "\nh1. Task ratio per locust class"
print
inspectlocust.print_task_ratio_confluence(locust_classes)
print "\nh1. Total task ratio"
print
inspectlocust.print_task_ratio_confluence(locust_classes, total=True)
sys.exit(0)
# if --master is set, implicitly set --web
if options.master:
options.web = True
if options.web and not options.slave:
# spawn web greenlet
print "Starting web monitor on port 8089"
main_greenlet = gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.ramp)
# enable/disable gzip in WebLocust's HTTP client
WebLocust.gzip = options.gzip
if not options.master and not options.slave:
core.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host)
# spawn client spawning/hatching greenlet
if not options.web:
core.locust_runner.start_hatching(wait=True)
main_greenlet = core.locust_runner.greenlet
elif options.master:
core.locust_runner = MasterLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host)
elif options.slave:
core.locust_runner = SlaveLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host)
main_greenlet = core.locust_runner.greenlet
if options.ramp:
import rampstats
from rampstats import on_request_success, on_report_to_master, on_slave_report
import events
if options.slave:
events.report_to_master += on_report_to_master
if options.master:
events.slave_report += on_slave_report
else:
events.request_success += on_request_success
if options.print_stats or (not options.web and not options.slave):
# spawn stats printing greenlet
gevent.spawn(stats_printer)
try:
print
print "Starting Locust %s" % version
print
main_greenlet.join()
except KeyboardInterrupt, e:
time.sleep(0.2)
print "\n"
print_stats(core.locust_runner.request_stats)
print_percentile_stats(core.locust_runner.request_stats)
print_error_report()
print
print "Exiting, bye.."
print
def stats_printer():
while True:
print_stats(statistics)
gevent.sleep(3)
if gas_method == 'c':
gas_energy = gas_energy_comp
else:
if gas_method == 'a':
gas_energy = gas_energy_temp
else:
# (n) method in the IECSF20 paper
gas_energy = gas_energy_const
print('Gas method {} heat energy total {}'.format(gas_method,
gas_energy.sum()))
# compute R2 and correlation
stats.print_stats_header()
stats.print_stats(space_and_waterR, gas_energy, 'BDEW', 2, True)
stats.print_stats(space_and_waterW, gas_energy, 'Watson', 1, True)
stats.print_stats(space_and_waterH, gas_energy, 'HDD 12.8', 1, True)
stats.print_stats(space_and_waterS, gas_energy, 'HDD 15.5 ', 1, True)
# output plots
gas_energy_comp.plot(label='Heat Demand Gas (compromise)', color='blue')
gas_energy_const.plot(label='Heat Demand Gas (not temp)',
color='blue',
style='--')
gas_energy_temp.plot(label='Heat Demand Gas (all temp)',
color='blue',
style='.')
#gas_energy_temp.plot(label='Heat Demand Gas', color='blue')
space_and_waterR.plot(label='Heat Demand BDEW', color='red')
#!/usr/bin/python import scatter import stats import sys (name1, bench1) = stats.load(sys.argv[1]) (name2, bench2) = stats.load(sys.argv[2]) aggregate = stats.compare(bench1, bench2) stats.print_stats(name1, name2, aggregate) times = scatter.load(sys.argv[1], sys.argv[2]) scatter.plot(name1, name2, times).show()
