def wait_file(file=FILE_LAST_EXPERIMENT, interval=0.1):
"""
Waits for `FILE_LAST_EXPERIMENT` to be initiated.
It signals that accounts were already funded.
"""
when = file_date(file)
print("Waiting %s to be initiated" % file)
while True:
time.sleep(interval)
new_when = file_date(file)
if new_when != when:
break
return
def measure(block_num, pause_between_queries=0.3, relaxation_rounds=1):
"""
when a (or more) new block appeared, add them to the total, and print a line.
"""
when_before = file_date(file=FILE_LAST_EXPERIMENT)
tx_count = w3.eth.getBlockTransactionCount(block_num)
start_time = timeit.default_timer()
start_epochtime = time.time()
# TODO: perhaps additional to elapsed system time, show blocktime?
print('starting timer, at block', block_num, 'which has ', tx_count,
' transactions; at epochtime', start_epochtime)
peak_tps_avg, count = 0, 0
tps_avg = {
} # memorize all of them, so we can return value at 'block_last'
while True:
new_block_num = w3.eth.blockNumber
if block_num != new_block_num: # when a new block appears:
tx_count, peak_tps_avg, tps_avg[
new_block_num] = analyze_new_blocks(block_num, new_block_num,
tx_count, start_time,
peak_tps_avg)
block_num = new_block_num
# for the first 3 rounds, always reset the peak_tps_avg again!
if count < relaxation_rounds:
peak_tps_avg = 0
count += 1
# send.py --> store_experiment_data() is called AFTER last tx was mined.
# THEN do another 10 empty blocks...
# only THEN end this loop:
if file_date(file=FILE_LAST_EXPERIMENT) != when_before:
print(
"Received signal from send.py when updating last-experiment.json"
)
block_last = json.load(open(FILE_LAST_EXPERIMENT,
'r'))['send']['block_last']
final_tps_avg = get_nearest_entry(tps_avg, block_last)
break
# do not query too often; as little side effect on node as possible
time.sleep(pause_between_queries)
print("Experiment ended! Current blocknumber = %d" % (w3.eth.blockNumber))
write_measures(peak_tps_avg, final_tps_avg, start_epochtime)
def control(label_to_titles, s):
utils.log(str(len(label_to_titles)) + " titles loaded")
label_to_titles = add_image_thumbs(label_to_titles, s)
utils.write_to_s3(label_to_titles,
'batch_titles_fd_' + utils.file_date() + '.json',
directory=s["s3dir"])
return label_to_titles
def filter_on_day(statuses):
statuses_filtered = []
current_day_key = utils.file_date()
for stat in statuses:
dd = utils.make_local(stat["created_at"])
key = str(dd.day) + "-" + str(dd.month) + "-" + str(dd.year)
if key == current_day_key:
stat["created_at_tz"] = str(dd)
statuses_filtered.append(stat)
filtered_cnt = len(statuses) - len(statuses_filtered)
if filtered_cnt > 0:
print(
str(filtered_cnt) + " statuses filtered for not being in day " +
current_day_key)
return statuses_filtered
def writeImageToS3(im_thumb, format, source_url):
try:
writable_image = modify_image(im_thumb, format)
write_dir = "data-aws/shared_data/images" + utils.file_date() + "/"
# write_dir = s["s3dir"] + 'production/images/'+utils.file_date()+'/'
write_name = make_image_key(source_url) + "." + format
print(write_name)
utils.write_binary_to_s3(writable_image,
write_name,
public=True,
directory=write_dir)
display_url = 'https://s3.amazonaws.com/twitter-satellite/' + write_dir + write_name
except Exception as e:
print(e)
print("error making image for " + source_url)
display_url = None
return display_url
def control(filt_label_dict, label_to_titles, s):
utils.log(str(len(filt_label_dict)) + " loaded, " + str(len(label_to_titles)) + " titles loaded")
label_data = process_label_dict(filt_label_dict, label_to_titles)
sorted_label_data_list = label_data_to_list(label_data)
meta = make_meta(label_data)
final_data = {"label_data":sorted_label_data_list, "meta_data": meta}
# name = "d3-" + utils.file_date() + "-label_format" + ".json"
# utils.write_to_s3(
# json.dumps(final_data),
# name,
# directory=s["s3dir"] + 'production/',
# public=True)
normalized_name = s["name"] + ".json"
utils.write_to_s3(
json.dumps(final_data),
normalized_name,
directory='data-aws/shared_data/production/'+utils.file_date() + "/",
public=True)
return None
# if "thumb_fetch_successful" not in titles[url]["news_image"] and 'media_url_news' in titles[url]["news_image"]:
if 'media_url_news' in titles[url]["news_image"]:
top_image = titles[url]["news_image"]['media_url_news']
print("Need image for " + top_image + ", " + url)
## CACHE CHECKING
f_base = utils.get_from_cache(top_image, "thumbs", print_hits=True)
if f_base == None:
f_base = make_thumb(top_image)
titles[url]["news_image"] = f_base
else:
print("already tried fetching thumb for " + url)
return titles
def control(label_to_titles, s):
utils.log(str(len(label_to_titles)) + " titles loaded")
label_to_titles = add_image_thumbs(label_to_titles, s)
utils.write_to_s3(label_to_titles,
'batch_titles_fd_' + utils.file_date() + '.json',
directory=s["s3dir"])
return label_to_titles
if __name__ == "__main__":
sd = utils.getDefaultSettings()
label_to_titles = utils.read_from_s3('batch_titles_fd_' +
utils.file_date() + '.json',
directory=sd["s3dir"])
control(label_to_titles, sd)
label_data = process_label_dict(filt_label_dict, label_to_titles)
sorted_label_data_list = label_data_to_list(label_data)
meta = make_meta(label_data)
final_data = {"label_data":sorted_label_data_list, "meta_data": meta}
# name = "d3-" + utils.file_date() + "-label_format" + ".json"
# utils.write_to_s3(
# json.dumps(final_data),
# name,
# directory=s["s3dir"] + 'production/',
# public=True)
normalized_name = s["name"] + ".json"
utils.write_to_s3(
json.dumps(final_data),
normalized_name,
directory='data-aws/shared_data/production/'+utils.file_date() + "/",
public=True)
return None
if __name__ == "__main__":
debug = True
sd = utils.getDefaultSettings()
filt_label_dict = utils.read_from_s3(
utils.file_name(prefix='_batch_filt_label_dict_enhanced_'),
directory=sd["s3dir"])
label_to_titles = utils.read_from_s3(
'batch_titles_fd_' + utils.file_date() + '.json',
directory=sd["s3dir"])
control(filt_label_dict, label_to_titles, sd)
def filter_on_day(stat):
current_day_key = utils.file_date()
dd = utils.make_local(stat["created_at"])
key = str(dd.day) + "-" + str(dd.month) + "-" + str(dd.year)
return_bool = key == current_day_key
return return_bool