def write_results_to_file(self, results):
""" Write results to CSV file"""
rows = []
try:
for item in results:
mode, gamma, c, ngram, bow, tuples = item
f1_avg, f1_array = tuples
if isinstance(gamma, float):
gamma = "%.4f" % gamma
if isinstance(c, float):
c = "%.0f" % c
f1_avg_4f = "%.4f" % f1_avg
metriclist = self.string_metrics(f1_array)
row = [mode, gamma, c, ngram, bow, f1_avg_4f]
row += metriclist
rows.append(row)
except TypeError:
print "Error: Type of parameter result"
print results
helpers.write_to_csv(self.RESULTFILE, "a", rows)
def write_begin(self):
""" Write header for results to CSV file """
# Create headers for rounds
list_roundnr = []
for i in range(1, self.CROSS_VALIDATION + 1):
roundnr_string = "Round %i" % i
list_roundnr.append(roundnr_string)
headers = [["MODE", "GAMMA", "C", "NGRAM", "BOW", "F1 AVG"]]
rows = [headers[0] + list_roundnr]
helpers.write_to_csv(self.RESULTFILE, "wb", rows)
def main():
performances_df = get_performances_df()
write_to_csv(performances_df, 'data', 'its', 'performances.csv')
title_index_df = get_title_index_df()
write_to_csv(title_index_df, 'data', 'its', 'title-index.csv')
tweets_df = get_tweets_df()
write_to_csv(tweets_df, 'data', 'its', 'tweets.csv')
sheets_df = get_sheets_df()
write_to_csv(sheets_df, 'data', 'its', 'sheets.csv')
def main():
logger.info(
"Starting data extraction for self-stabilization overhead experiment")
exec_time_series = api.get_time_series_for_q(Q_EXEC_TIME)
msgs_sent_time_series = api.get_time_series_for_q(Q_MSGS_SENT)
bytes_sent_time_series = api.get_time_series_for_q(Q_BYTES_SENT)
if len(exec_time_series) == 0:
logger.warning("No results found, quitting")
return
data_points = transform(exec_time_series, msgs_sent_time_series,
bytes_sent_time_series)
csv_path = helpers.write_to_csv(EXPERIMENT, data_points)
snapshot_path = helpers.get_snapshot()
helpers.collect_to_res_folder(EXPERIMENT, [csv_path, snapshot_path])
helpers.cleanup()
def main(url):
df = get_annotations_df(url)
write_to_csv(df, 'data', 'annotations.csv')
def main():
df = get_sheets_df()
write_to_csv(df, 'data', 'its', 'sheets.csv')
weekend_signup_ratio)
signup_events = gse.generate_signup_events(user_table, signup_event_list,
start, end, starting_flow,
ending_flow, step_time,
weekend_signup_flow_ratio)
login_events = gle.generate_login_events(
signup_events[0], signup_event_list, start_user_dist, end_user_dist,
starts_odds_of_returning, end_odds_of_returning, start, end,
start_days_to_return, end_days_to_return, weekend_login_ratio)
events = login_events
events = gae.generate_arbitrary_events(events, event_params_from_login)
events = gae.generate_arbitrary_events(events, event_params_from_view_company)
events = gtt.generate_test_treaments(events, test_treatment_params)
user_table = signup_events[1]
# billing_table = gb.generate_billing_table(user_table,company_dist,start_monetize_rate,end_monetize_rate,start_churn_rate,end_churn_rate,
# contract_length,contract_range,start,end)
event_table = events
## Clean and write to CSV
clean_user_table = h.prepare_table(user_table, user_headers, user_include)
clean_event_table = h.prepare_table(event_table, event_headers, event_include)
# clean_billing_table = h.prepare_table(billing_table,billing_headers,billing_include)
h.write_to_csv("fake_dimension_users.csv", clean_user_table)
h.write_to_csv("fake_fact_events.csv", clean_event_table)
# h.write_to_csv("fake_dimension_subscriptions.csv",clean_billing_table)
def save_to_csv(self, proxies):
write_to_csv(self.csv_path, proxies)
key=lambda m: float(m["metric"]["exp_param"]))
for i in range(len(conv_lat_asc)):
# if int(conv_lat_asc[i]["metric"]["view"]) != int(msgs_sent_asc[i]["metric"]["exp_param"]):
# raise ValueError("Results not matching")
conv_lat = str(float(conv_lat_asc[i]["value"][1])).replace(".", ",")
view = int(conv_lat_asc[i]["metric"]["view"])
msgs_sent = int(msgs_sent_asc[i]["value"][1])
bts_sent = int(bts_sent_asc[i]["value"][1])
data_points.append({
"old_view": view,
"conv_lat": conv_lat,
"msgs_sent": msgs_sent,
"bytes_sent": bts_sent
})
# build key:val pairs for data points and return
return data_points
if __name__ == "__main__":
logger.info("Starting data extraction for convergence latency experiment")
conv_lat_time_series = api.get_time_series_for_q(Q_CONV_LAT)
msgs_sent_time_series = api.get_time_series_for_q(Q_MSGS_SENT)
bytes_sent_time_series = api.get_time_series_for_q(Q_BYTES_SENT)
data_points = transform(conv_lat_time_series, msgs_sent_time_series,
bytes_sent_time_series)
csv_path = helpers.write_to_csv(EXPERIMENT, data_points)
snapshot_path = helpers.get_snapshot()
helpers.collect_to_res_folder(EXPERIMENT, [csv_path, snapshot_path])
helpers.cleanup()
def main():
new_df = get_new_df()
write_to_csv(new_df, 'data', 'cac', 'new.csv')
ingested_df = get_ingested_df()
write_to_csv(ingested_df, 'data', 'cac', 'ingested.csv')
event_params_from_view_company = [start,end,from_event,sequence_type,event_list,start_event_distribution,end_event_distribution,start_odds_to_continue,
end_odds_to_continue,start_number_of_events,end_number_of_events]
## MAKE DATA
user_table = gu.generate_user_table(users,start,end,monthly_growth,weekend_signup_ratio)
signup_events = gse.generate_signup_events(user_table,signup_event_list,start,end,starting_flow,ending_flow,step_time,weekend_signup_flow_ratio)
login_events = gle.generate_login_events(signup_events[0],signup_event_list,start_user_dist,end_user_dist,starts_odds_of_returning,
end_odds_of_returning,start,end,start_days_to_return,end_days_to_return,weekend_login_ratio)
events = login_events
events = gae.generate_arbitrary_events(events,event_params_from_login)
events = gae.generate_arbitrary_events(events,event_params_from_view_company)
events = gtt.generate_test_treaments(events,test_treatment_params)
user_table = signup_events[1]
# billing_table = gb.generate_billing_table(user_table,company_dist,start_monetize_rate,end_monetize_rate,start_churn_rate,end_churn_rate,
# contract_length,contract_range,start,end)
event_table = events
## Clean and write to CSV
clean_user_table = h.prepare_table(user_table,user_headers,user_include)
clean_event_table = h.prepare_table(event_table,event_headers,event_include)
# clean_billing_table = h.prepare_table(billing_table,billing_headers,billing_include)
h.write_to_csv("fake_dimension_users.csv",clean_user_table)
h.write_to_csv("fake_fact_events.csv",clean_event_table)
# h.write_to_csv("fake_dimension_subscriptions.csv",clean_billing_table)
def main():
df = get_title_index_df()
write_to_csv(df, 'data', 'its', 'title-index.csv')
def main(obj):
df = get_pybossa_df(obj)
write_to_csv(df, 'data', '{}.csv'.format(obj))
def main():
df = get_performances_df()
write_to_csv(df, 'data', 'its', 'performances.csv')
if len(tuple(dict[date][i]['checkin'])) != len(
tuple(dict[date][i]['checkout'])):
continue
else:
in_time = dict[date][i]['checkin']
out_time = dict[date][i]['checkout']
length = len(dict[date][i]['checkin'])
if length > 0 and len(in_time) > 0:
runningSum = 0
for l, i, k in zip(range(length), range(0, length, 2),
range(1, length, 2)):
to_a['in' + str((l + 1))] = in_time[i]
to_a['out' + str(l + 1)] = out_time[k]
runningSum += helpers.to_decimal(
out_time[k]) - helpers.to_decimal(in_time[i])
to_a['Raw_hours'] = runningSum
to_a['hoursWorked_str'] = helpers.to_ftime(runningSum)
# Writing to array
for_csv.append(to_a)
print(len(for_csv))
print(for_csv)
row_names = [
"employeeID", "date", "in1", "out1", "in2", "out2", "in3", "out3", "in4",
"out4", "in5", "out5", 'hoursWorked_str', 'Raw_hours'
]
helpers.write_to_csv(for_csv, 'may', row_names)
from definitions import PATH
files = sorted(glob.glob(PATH["IMAGE_IN"] + "*.png"))
processed = glob.glob(PATH["IMAGE_OUT"] + "*.png")
print('Current progress is {}%'.format((len(processed) / len(files)) * 100))
for file in files:
if helpers.is_processed(file, processed):
continue
file_properties = helpers.get_properties_from_filename(file)
img = Image.open(file)
file_properties["original_size"] = img.size
# crop the image in one smaller image of 256x256
crop_dimensions = file_properties["crop"]
img_crop = img.crop(crop_dimensions)
file_properties["name"] = '{}-{}-[{}].png'.format(
file_properties["id"], file_properties["crop"],
str(file_properties["slice"]))
cw = CountingWindow(img_crop)
cw.open_window()
file_properties["count"], file_properties["locations"] = cw.get_results()
if file_properties["count"] is not None and file_properties[
"locations"] is not None:
helpers.write_to_csv(file_properties, PATH["CSV"])
img_crop.save(PATH["IMAGE_OUT"] + file_properties["name"])
