def clean_up_thread(process_num):
try:
top = load(topfile)
except FileNotFoundError:
top = set()
top.discard(process_num)
dump(topfile, top)
def add_thread(process_num):
try:
top = load(topfile)
except FileNotFoundError:
top = set()
if process_num in top:
raise OSError('GPU line {} is busy!'.format(process_num))
else:
top.add(process_num)
dump(topfile, top)
def queue_add(queues):
if not os.path.isdir(BASEDIR):
os.mkdir(BASEDIR)
if not isinstance(queues, list):
raise ValueError('Queue must be lists!')
try:
with shelf_with_locker() as shelf:
shelf['queued'].expand(queues)
except FileNotFoundError:
pep_queue = BASEDIR / 'pep_queue'
qdict = {'queued': queues, 'running': {}, 'finished': {}}
dump(pep_queue, qdict)
def shelf_with_locker():
from fcntl import flock, LOCK_EX, LOCK_UN
shelf_locker = BASEDIR / '.slf.lck'
pep_queue = BASEDIR / 'pep_queue'
try:
lck = open(shelf_locker, 'w')
flock(lck, LOCK_EX)
shelf = load(pep_queue)
yield shelf
dump(pep_queue, shelf)
except:
raise
finally:
flock(lck, LOCK_UN)
lck.close()
def test_shelf(self):
def shelftester():
with ps.shelf_with_locker() as shelf:
return shelf['queued'].pop()
shelfpos = self.testdir / 'pep_queue'
try:
os.remove(shelfpos)
except Exception:
pass
self.assertRaises(FileNotFoundError, shelftester)
dump(shelfpos, {'queued': ['test']})
self.assertEqual('test', shelftester())
def simple_train(epoch_steps):
infos = []
start_time = time.time()
with training(restore_from=config.RESTORE_FROM) as tools:
write(tools.path + "/description", config.DISCRIPTION + "\n")
for i in range(epoch_steps):
batch_init = tf.get_collection("batch_init")
tools.sess.run(batch_init)
infos.append(epoch_train(tools))
if i > 5:
recent = [x[1] for x in infos[-5:]]
if np.std(recent) < config.STOP_THRESHOLD:
break
dump(tools.path + "/trace", infos)
duration = time.time() - start_time
append(
tools.path + "/description", "Time usage: " +
time.strftime("%M minutes, %S seconds", time.gmtime(duration)) +
"\n")
return infos