def train(hidden_dim_sweep=(5, 10, 25),
n_epochs=20,
out_dir='out',
data_dir='data',
device=util.device(),
Optimizer=optim.Adam,
seed=42):
out_dir, data_dir = map(Path, (out_dir, data_dir))
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
tracess = []
best_trainer = None
best_loss = util.INF
vocab = util.Vocab.load(data_dir / 'vocab.txt')
for hidden_dim in hidden_dim_sweep:
model = Model(hidden_dim=hidden_dim, vocab=vocab, out_dim=2)
loss_fn = nn.CrossEntropyLoss()
optimizer = Optimizer(model.parameters(), lr=1e-4)
trainer = Trainer(model, loss_fn, vocab, device)
traces, loss_cur = trainer.train_loop(data_dir=data_dir,
n_epochs=n_epochs,
optimizer=optimizer,
scheduler=None)
if loss_cur < best_loss:
best_trainer = trainer
best_loss = loss_cur
tracess.append((hidden_dim, traces))
out_dir.mkdir(exist_ok=True)
for h, traces in tracess:
plotting.plot_traces(traces,
out=out_dir / f'traces_{h}.png',
title=f'Loss,hidden_dim={h}')
util.jsondump(traces, out_dir / f'traces.dim_{h}.seed_{seed}.json')
L.info('Best model loss: %s', best_loss)
model_file = out_dir / 'model.pt'
L.info('Saving best model to %s', model_file)
torch.save(best_trainer.model.state_dict(), model_file)
def report_feedback(userid):
"""
"""
print "[statistics] Starting feedback send"
fbpath = makepath('feedback.json')
if not path_exists(fbpath):
feedback = []
else:
feedback = jsonload(fbpath)
if len(feedback) > 0:
data = {'feedback' : feedback, 'userid' : userid}
network_POST('/stats/report_feedback', data)
jsondump(makepath('feedback.json'), [])
print '[statistics] Finished feedback send'
altpath = makepath('feedback.txt')
if path_exists(altpath):
feedback = fileload(altpath)
data = {'feedback' : feedback, 'userid' : userid}
network_POST('/stats/report_legacy_feedback', data)
command('rm', altpath)
def __str__(self):
items = []
if self.msg: items.append(self.msg)
if self.context: items.append(jsondump(self.context)[:300])
return " ".join(items)
def __str__(self):
items = []
if self.msg: items.append(self.msg)
if self.context: items.append(jsondump(self.context)[:300])
return " ".join(items)
def spiceMerge(spices, username):
"""
Manages the merger of a spice into the development repository.
@param spices: list of (spicename, dirname)
"""
log('[spicemanager] In func:spiceMerge')
goto_dir(makepath('spicerackdev'))
command('git', 'pull')
goto_dir(makepath(''))
spice_to_metadata = {}
for dirname in spices:
data = jsonload(makepath('spicerackdev', dirname, 'metadata.json'))
spicename = data['name']
if 'spicename' not in data:
data['spicename'] = spicename
if 'width' not in data:
data['width'] = 0
data['height'] = 0
data['dependencies'] = []
Statsd.increment('spices.spicerackdev.%s' % spicename.replace(' ', '-'))
mid = get('spice-id:%s' % spicename.replace(' ', '-'))
if mid is None:
mid = str(uuid.uuid4())
put('spice-id:%s' % spicename.replace(' ', '-'), mid)
log('[spicemanager] Processing %s %s' % (spicename, mid))
version = 1 + int(dget('version:%s' % mid, 0))
put('version:%s' % mid, version)
log('[spicemanager] Version: %s' % version)
spice_to_metadata[mid] = {
'spicename' : data['spicename'],
'gituser' : username,
'author' : data['author'],
'tagline' : data['description'],
'description' : data['description'],
'version' : version,
'hotkey' : False,
'main' : data['main'],
'width' : data['width'],
'height' : data['height'],
'dependencies' : data['dependencies']
}
put('metadata:%s' % mid, spice_to_metadata[mid])
jsondump(makepath('spicerackdev', dirname, 'metadata.json'), spice_to_metadata[mid])
# Tar the spice directory into /spices/{mid}.tar.gz
command('tar', '-cvf', makepath('spices', '%s.tar.gz' % mid), makepath('spicerackdev', dirname))
# Copy the icon file into /icons
iconpath = makepath('spicerackdev', dirname, 'icon.png')
command('cp', '-f', iconpath, makepath('icons', '%s.png' % mid))
iconpath = makepath(dirname, 'icon.png')
if not os.path.exists(iconpath):
command('tar', '-cvf', makepath('spices', '%s.tar.gz' % mid), makepath('spicerackdev', dirname))
else:
iconpath = iconpath.strip('icon.png')
command('tar',
'-cvf',
makepath('spices', '%s.tar.gz' % mid),
makepath('spicerackdev', dirname),
'--directory=%s' % iconpath,
'icon.png')
goto_dir(makepath('spicerackdev'))
command('git', 'stash')
goto_dir(makepath(''))
heroku_metadata = {
'spicename' : data['spicename'],
'gituser' : username,
'author' : data['author'],
'tagline' : data['description'],
'description' : data['description'],
'version' : version,
'hotkey' : False,
'main' : data['main']
}
log('HEROKU: %s %s', type(heroku_metadata), heroku_metadata)
for key, val in heroku_metadata.iteritems():
log('KEY %s VAL %s %s', key, type(val), val)
resp, content = send_post({str(mid) : heroku_metadata}, '/spices/refresh')
log('[spicemanager] Heroku response: %s' % str(resp))
return 'success'