def store(self):
with simpleflock.SimpleFlock("/tmp/foolock1"):
with open(self.outfile, 'wb') as f:
f.write("ip\test_rtt\tTTFB\tTotal\tproxy\tmyProxy\n")
for r in sorted(self.routes.values(),
key=operator.attrgetter('total')):
f.write(r.__str__())
def getVivaldiProxy():
with simpleflock.SimpleFlock('/tmp/foolock1'):
with open('proxy_route_table', 'r') as f:
global PROXY, ema025, temp_ema025, ema005, temp_ema005, ema075, temp_ema075
r = csv.reader(f, delimiter='\t')
new_proxy = None
for row in r:
if row[5] == 'True':
new_proxy = row[0]
new_ttfb = row[2]
break
if new_proxy != PROXY:
PROXY = new_proxy
if PROXY not in ema025:
ema025[PROXY] = EMA(0.25)
temp_ema025[PROXY] = EMA(0.25)
ema005[PROXY] = EMA(0.05)
temp_ema005[PROXY] = EMA(0.05)
ema075[PROXY] = EMA(0.75)
temp_ema075[PROXY] = EMA(0.75)
else:
ema025[PROXY].last = float(new_ttfb)
temp_ema025[PROXY].last = float(new_ttfb)
ema005[PROXY].last = float(new_ttfb)
temp_ema005[PROXY].last = float(new_ttfb)
ema075[PROXY].last = float(new_ttfb)
temp_ema075[PROXY].last = float(new_ttfb)
def getVivaldiDistance(proxy):
result = 0.0
with simpleflock.SimpleFlock('/tmp/foolock1'):
with open('proxy_route_table','r') as f:
r = csv.reader(f,delimiter='\t')
for row in r:
if row[0]==proxy:
result = float(row[1])/1000
return result
def add(maps): try: if len(sys.argv) <= 1: with simpleflock.SimpleFlock(MAPCACHE_CONFIG, timeout=15): _add(maps) else: _add(maps) except: print "Mapcache filelock!"
def chooseRepos(filename, count):
choices = []
with sf.SimpleFlock("/tmp/" + filename + "_lock"):
with open(filename) as rf:
choices.extend([line.strip() for line in rf.readlines()[:count]])
call("sed -i '1," + str(count) + "d' " + filename, shell=True)
if not choices:
raise EOFError("File Empty")
return choices
def save_shared_values(self):
"Write shared variables out to the cache file"
for key in self.shared:
self.shared[key] = _env.get(key, None)
with simpleflock.SimpleFlock(self.lock_file):
with open(self.cache_file, "wb") as filehandle:
pickle.dump(self.shared, filehandle)
self.timestamp = time.time()
def getVivaldiProxy():
with simpleflock.SimpleFlock('/tmp/foolock1'):
with open('proxy_route_table','r') as f:
global PROXY
r = csv.reader(f,delimiter='\t')
new_proxy = None
for row in r:
if row[5] == 'True':
new_proxy = row[0]
break
PROXY = new_proxy
def readTTFB(self):
if self.proxy:
if os.path.isfile(FILE):
values = []
with simpleflock.SimpleFlock("/tmp/foolock"):
with open(FILE, 'rb') as infile:
values = infile.readline().strip().split(',')
ip = values[0]
ttfb = values[1]
if self.proxy == ip:
self.updateTTFB(self.proxy, ttfb, 0)
def __enter__(self):
c = Cache(self.url, verbose=self.verbose)
with simpleflock.SimpleFlock(c.fileLock, timeout=10):
if c.is_valid():
self.payload = c.read()
else:
self.payload['overview'] = self.getOverview()
self.payload['queues'] = self.getQueueStats()
self.payload['healthcheck'] = self.getHealthcheck()
c.write(self.payload)
if self.verbose > 1:
print(json.dumps(self.payload))
return self
def _load_shared_values(self):
"Load shared variables from the cache file"
with simpleflock.SimpleFlock(self.lock_file):
try:
self.shared = pickle.load(open(self.cache_file, "rb"))
except IOError:
return
except Exception as the_exception:
print("Failed to reload shared values.", the_exception)
return
for key, value in self.shared.items():
if value is None and key in _env:
del _env[key]
else:
_env[key] = value
self.timestamp = time.time()
def add_record(file_name):
if len(file_name) == 0:
abort(400)
if not request.json:
return jsonify({'status': 'error: not a valid json'}), 201
out = request.json
file_path = "/dev/shm/logging/"
if not os.path.exists(file_path):
os.makedirs(file_path)
with open(file_path + file_name + '.json', "a") as myfile:
out['server_time'] = str(datetime.datetime.utcnow())
with simpleflock.SimpleFlock(file_path + file_name + '.lock',
timeout=5):
with timeout():
myfile.write(json.dumps(out) + '\n')
return jsonify({'status': (out if debug else 'log added')}), 201
def set_cache(obj):
"""Update the cache with an object (dict, e.g.)
The object is cached on disk. A lock file is used to coordinate
updates to this cache among threads and processes.
If another process has the cache locked (only used for writing),
then this function does nothing; i.e. it assumes that somebody
else is taking care of the update ....
Arguments:
obj - object to write to the cache
Return:
none
"""
lock_path = _pathname(__LOCK_FILE)
try:
with simpleflock.SimpleFlock(lock_path, timeout=0):
_pickle_and_cache_models(obj)
except IOError as e:
if e.errno != errno.EWOULDBLOCK:
logging.error('creating lock file {}: {}'.format(lock_path, e))
raise
ema025[PROXY].compute(final_out)
temp_ema025[PROXY].last = ema025[PROXY].last
ema005[PROXY].compute(final_out)
temp_ema005[PROXY].last = ema005[PROXY].last
ema075[PROXY].compute(final_out)
temp_ema075[PROXY].last = ema075[PROXY].last
else:
temp_time = time()
if (temp_time-last_time)>last_value:
last_value = temp_time-last_time
#out = last_value
temp_ema025[PROXY].compute(float(last_value))
temp_ema005[PROXY].compute(float(last_value))
temp_ema075[PROXY].compute(float(last_value))
type = 'est'
t = time()-START
if (t-last_vivaldi) > VIVALDI_PERIOD:
vivaldi_time = getVivaldiDistance(PROXY)
last_vivaldi = t
with open(FILE,'a') as f:
f.write("{0:.1f},{1},{2},{3},{4},{5},{6},{7},{8},{9}\n".format(t,out,ema025[PROXY].last,temp_ema025[PROXY].last,ema005[PROXY].last,temp_ema005[PROXY].last,ema075[PROXY].last,temp_ema075[PROXY].last,type,vivaldi_time))
with open(EXP_FILE,'a') as f:
f.write("{0:.1f},{1}\n".format(t,PROXY))
with simpleflock.SimpleFlock('/tmp/foolock'):
with open(VIV_FILE,'wb') as viv:
viv.write("{},{}\n".format(PROXY,temp_ema005[PROXY].last))
#getVivaldiProxy()
#print("{0:.1f},{1},{2},{3},{4},{5},{6},{7},{8},{9}\n".format(t,out,ema025.last,temp_ema025.last,ema005.last,temp_ema005.last,ema075.last,temp_ema075.last,type,vivaldi_time))
def check_art():
try:
with simpleflock.SimpleFlock("check-art.lock", timeout=1):
run_command()
except BlockingIOError:
numu_app.logger.error("Unable to achieve lock.")
def writeEmails(user_emails_list):
with sf.SimpleFlock("/tmp/scrape_write_email"):
with open("username_email_github.csv", "a") as f:
for username,email in user_emails_list:
f.write(username+","+email+"\n")
def patchUnfinished(username_list_filename, choices):
with sf.SimpleFlock("/tmp/" + username_list_filename + "_lock"):
with open(username_list_filename, "a") as wf:
wf.write("\n".join(choices) + "\n")
import csv
csvfile = open("UMLFiles_List_V2.0.csv")
data = csv.reader(csvfile)
all_xmis = []
for (name, link) in data:
if link[-3:] == "xmi":
all_xmis.append((name, link.replace("tree/master", "raw/master")))
import requests
import glob
import simpleflock
from os.path import exists
for project_name, xmi_link in all_xmis:
filename = "scraped_xmis/" + project_name.replace(
"/", "_") + xmi_link.split("/")[-1]
with simpleflock.SimpleFlock("/tmp/scrape_select_file"):
if exists(filename):
continue
else:
open(filename, "w").close()
print(xmi_link)
page = requests.get(xmi_link)
with simpleflock.SimpleFlock("/tmp/scrape_file_write"):
with open(filename, "wb") as f:
f.write(page.content)
def user_processing():
try:
with simpleflock.SimpleFlock("user-processing.lock", timeout=1):
run_command()
except BlockingIOError:
numu_app.logger.error("Unable to achieve lock.")
def save(batches):
deadline_batches_index = numpy.where(
numpy.isfinite([x.batch.deadline for x in batches]))
best_effort_batches_index = numpy.where(
numpy.isinf([x.batch.deadline for x in batches]))
best_effort_completion = [
(batches[i].batch.size - batches[i].batch.level) /
batches[i].batch.size for i in best_effort_batches_index[0]
]
# print(best_effort_batches_index)
# print(best_effort_completion)
# print(numpy.mean(best_effort_completion), numpy.max(best_effort_completion))
deadline_completion = [(batches[i].batch.size - batches[i].batch.level) /
batches[i].batch.size
for i in deadline_batches_index[0]]
ratio_success_deadline = []
dead_function = lambda x, s: s * (x * (options[
'average_service_time'] - options['average_service_time'] / numpy.min(
[options['no_workers'], s])) + options['average_service_time'] /
numpy.min([options['no_workers'], s]))
strict_array = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]
fn = options['filename'] + '.pkl'
with simpleflock.SimpleFlock(options['filename'] + '.lock', timeout=60):
try:
results = pd.read_pickle(fn)
print('Results already exists in this directory.. appending...')
except:
print('file', fn, "doesn't exists")
results = pd.DataFrame(columns=[
'total_time', 'no_batches', 'no_workers', 'seed', 'scheduler',
'deadline_success_ratio', 'mean_deadline', 'median_deadline',
'std_deadline', 'mean_best_effort', 'median_best_effort',
'std_best_effort', 'execution_time', 'no_best_effort',
'no_deadline', 'strictness', 'log'
])
for s in range(0, len(strict_array)):
for i in deadline_batches_index[0]:
batches[i].batch.deadline = dead_function(
strict_array[s],
batches[i].batch.size) + batches[i].batch.arrive_time
batches[i].batch.success = batches[
i].batch.completion_time <= batches[i].batch.deadline
ratio_success_deadline.append(
numpy.sum([
batches[i].batch.success for i in deadline_batches_index[0]
]) / numpy.size(deadline_batches_index))
results = results.append(pd.DataFrame(
{
'total_time':
options['total_time'],
'no_batches':
options['no_best_effort_batches'] +
options['no_deadline_batches'],
'no_workers':
options['no_workers'],
'seed':
options['seed'],
'scheduler':
options['scheduler'],
'execution_time':
res['execution_time'],
'deadline_success_ratio':
ratio_success_deadline[s],
'mean_deadline':
numpy.mean(deadline_completion),
'median_deadline':
numpy.median(deadline_completion),
'std_deadline':
numpy.std(deadline_completion),
'mean_best_effort':
numpy.mean(best_effort_completion),
'median_best_effort':
numpy.median(best_effort_completion),
'std_best_effort':
numpy.std(best_effort_completion),
'no_best_effort':
numpy.size(best_effort_batches_index),
'no_deadline':
numpy.size(deadline_batches_index),
'strictness':
strict_array[s],
'log':
options['log']
},
index=[0]),
ignore_index=True)
results.to_pickle(fn)
results.to_csv(options['filename'] + '.csv', index=False)
if not numpy.mod(options['seed'], 25):
plt.plot(*zip(*res['fairness_index']))
plt.xlabel('Time [s]')
plt.ylabel('Fairness index')
axes = plt.gca()
axes.set_ylim([-3010, 10])
plt.savefig(options['filename'] + '_' + options['scheduler'] + '_' +
str(options['seed']) + '.eps')
with open(
options['filename'] + str(options['seed']) +
options['scheduler'] + '_plot.pkl', 'w') as f:
pickle.dump(res['fairness_index'], f)
#TODO: scan for plates at startup so we can handle wrong-address
# or plate_type mismatch exceptions
while True:
try:
line = sys.stdin.readline()
# TODO: add error handling for invalid JSON
msg = json.loads(line)
addr = msg['addr']
plate_type = msg['plate_type']
cmd = msg['cmd']
args = msg['args']
resp = {}
if (plate_type == "RELAY"):
with simpleflock.SimpleFlock("/tmp/relay.lock", timeout=3):
if (cmd == "setLED"):
RP.setLED(addr)
resp['LED'] = 1
elif (cmd == "clrLED"):
RP.clrLED(addr)
resp['LED'] = 0
elif (cmd == "toggleLED"):
RP.toggleLED(addr)
resp['LED'] = "UNKNOWN"
elif (cmd == "getID"):
resp['ID'] = RP.getID(addr)
elif (cmd == "getHWrev"):
resp['HWrev'] = RP.getHWrev(addr)
elif (cmd == "getFWrev"):
resp['FWrev'] = RP.getFWrev(addr)
