def __init__(self):
try:
pynvml.nvmlInit()
self.gpu_count = pynvml.nvmlDeviceGetCount()
except pynvml.NVMLError:
self.gpu_count = 0
if psutil:
net = psutil.net_io_counters()
self.network_init = {
"sent": net.bytes_sent,
"recv": net.bytes_recv
}
else:
print(
"psutil not installed, only GPU stats will be reported. \n Install with pip install psutil"
)
self._thread = threading.Thread(target=self._thread_body, daemon=True)
self._pid = os.getpid()
self._id = util.generate_id()
self._start_time = time.time()
self.sampler = {}
self.samples = 0
self._sample_rate_seconds = 2
self._samples_to_average = 3
self._shutdown = False
self._dir = Path(
f"output/run_{time.strftime('%Y%m%d', time.localtime())}_{int(self._start_time // 10000)}-{self._id}"
)
self.events = jsonlfile.JsonlEventsFile(self._start_time, EVENTS_FNAME,
self._dir)
def add_question():
if request.method == 'POST':
message = request.form['message']
title = request.form['title']
question_id = util.generate_id('sample_data/question.csv')
data_manager.write_to_questions("sample_data/question.csv", message,
title)
return redirect(url_for('show_questions', question_id=question_id))
return render_template('add-question.html')
def create():
# gen id
url_id = util.generate_id(8)
# save to DB
Link.insert_url(url_id, request.form['url'])
# return link to page
return redirect(url_for('share', url_id=url_id))
def monster_placement(validplace, monsters, board,
monsters_alive): # place the chosen monsters
monster_left_to_place = monsters
used = []
for monster in monster_left_to_place:
monster['ID'] = util.generate_id()
for monster_dict in monster_left_to_place:
if monster_dict not in used:
place = random.choice(validplace)
monster_dict['X'], monster_dict['Y'] = place[0], place[1]
board[place[0]][place[1]] = monster_dict['Icon']
monsters_alive.append(monster_dict)
used.append(monster_dict)
def run(self):
self._running = True
ip = [self._sock, self.pr]
self.evtShutdown.set()
while self._running:
irdy, ordy, xptrdy = select.select(ip, [], [])
for sck in irdy:
if sck == self._sock:
try:
(s, address) = self._sock.accept()
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# TODO: we should check here that tid doesn't already exist in thread list keys
tid = util.generate_id()
thd = DTPTThread(s, self, tid)
ix = self.AddServer(tid, thd)
thd.start()
except:
self._running = 0
return
elif sck == self.pr:
dump = os.read(self.pr, 4)
logger.info("DTPT server shutdown request actioned")
self._running = 0
else:
continue
self._sock.close()
logger.info("shutting down DTPT threads")
self._CloseAllThreads()
logger.info("waiting for final shutdown")
self.evtShutdown.wait()
os.close(self.pr)
os.close(self.pw)
def start_evaluation(args):
device = util.setup_torch()
num_classes = 100 if args.dataset == "cifar100" else 10
train_loader, test_loader = get_cifar(num_classes,
batch_size=args.batch_size)
# for benchmarking, decided whether we want to use unique test folders
if USE_ID:
test_id = util.generate_id()
else:
test_id = ""
results_dir = Path(args.results_dir).joinpath(test_id)
results_dir = Path(results_dir).joinpath(args.dataset)
util.check_dir(results_dir)
# Parsing arguments and prepare settings for training
params = {
"epochs": args.epochs,
"modes": args.modes,
"t_checkpoint": args.t_checkpoint,
"results_dir": results_dir,
"train_loader": train_loader,
"test_loader": test_loader,
"batch_size": args.batch_size,
# model configuration
"device": device,
"teacher_name": args.t_name,
"student_name": args.s_name,
"num_classes": num_classes,
# hyperparameters
"weight_decay": args.weight_decay,
"learning_rate": args.learning_rate,
"momentum": args.momentum,
"sched": "multisteplr",
"optim": "SGD",
# fixed knowledge distillation parameters
"lambda_student": 0.5,
"T_student": 5,
}
test_conf_name = results_dir.joinpath("test_config.json")
util.dump_json_config(test_conf_name, params)
run_benchmarks(args.modes, params, args.s_name, args.t_name)
plot_results(results_dir, test_id=test_id)
def run(self):
self._running = True
ip = [self._sock, self.pr]
self.evtShutdown.set()
while self._running:
irdy,ordy,xptrdy = select.select(ip,[],[])
for sck in irdy:
if sck == self._sock:
try:
(s, address) = self._sock.accept()
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# TODO: we should check here that tid doesn't already exist in thread list keys
tid = util.generate_id()
thd=DTPTThread(s,self,tid)
ix = self.AddServer(tid,thd)
thd.start()
except:
self._running=0
return
elif sck == self.pr:
dump=os.read(self.pr,4)
logger.info("DTPT server shutdown request actioned")
self._running = 0
else:
continue
self._sock.close()
logger.info("shutting down DTPT threads")
self._CloseAllThreads()
logger.info("waiting for final shutdown")
self.evtShutdown.wait()
os.close(self.pr)
os.close(self.pw)
def write_to_answers(filename, question_id, message):
HEADERS = []
for dictionary in read_from_file(filename):
for sub_dict in dictionary:
HEADERS.append(sub_dict)
break
with open(filename, "a+") as file:
elements = csv.DictWriter(file, fieldnames=HEADERS)
submission_time = int(time.time())
vote_number = 0
message = message
image = ""
new_id = util.generate_id(filename)
elements.writerow({"id": new_id,
"submission_time": submission_time,
"vote_number": vote_number,
"question_id" : question_id,
"message" : message,
"image": image}
)
def write_to_questions(filename,message,title):
HEADERS = []
for dictionary in read_from_file(filename):
for sub_dict in dictionary:
HEADERS.append(sub_dict)
break
with open(filename, "a+") as file:
elements = csv.DictWriter(file, fieldnames=HEADERS)
submission_time = int(time.time())
submission_time = datetime.utcfromtimestamp(submission_time).strftime('%Y-%m-%d %H:%M:%S')
vote_number = 0
message = message
title=title
image = ""
view_number = 0
new_id = util.generate_id(filename)
elements.writerow({"id": new_id,
"submission_time": submission_time,
"view_number": view_number,
"vote_number": vote_number,
"title": title,
"message" : message,
"image": image}
)