def test_model(clf, X_te, y_te):
from flor import Flog
if Flog.flagged():
flog = Flog()
Flog.flagged() and flog.write({
'file_path': '/Users/rogarcia/sandbox/examples/basic.py',
'lsn': 0
})
Flog.flagged() and flog.write({'start_function': 'test_model', 'lsn': 1})
Flog.flagged() and flog.write({
'lsn':
2,
'params': [{
'0.raw.clf': flog.serialize(clf)
}, {
'1.raw.X_te': flog.serialize(X_te)
}, {
'2.raw.y_te': flog.serialize(y_te)
}]
})
score = log.metric(clf.score(X_te, y_te))
Flog.flagged() and flog.write({
'locals': [{
'score': flog.serialize(score)
}],
'lineage': 'score = log.metric(clf.score(X_te, y_te))',
'lsn': 4
})
Flog.flagged() and flog.write({'end_function': 'test_model', 'lsn': 3})
def train_model(n_estimators, X_tr, y_tr):
from flor import Flog
if Flog.flagged():
flog = Flog()
Flog.flagged() and flog.write({
'file_path': '/Users/rogarcia/sandbox/examples/basic.py',
'lsn': 0
})
Flog.flagged() and flog.write({'start_function': 'train_model', 'lsn': 1})
Flog.flagged() and flog.write({
'lsn':
2,
'params': [{
'0.raw.n_estimators': flog.serialize(n_estimators)
}, {
'1.raw.X_tr': flog.serialize(X_tr)
}, {
'2.raw.y_tr': flog.serialize(y_tr)
}]
})
clf = RandomForestClassifier(n_estimators=n_estimators).fit(X_tr, y_tr)
Flog.flagged() and flog.write({
'locals': [{
'clf': flog.serialize(clf)
}],
'lineage':
'clf = RandomForestClassifier(n_estimators=n_estimators).fit(X_tr, y_tr)',
'lsn': 4
})
__return__ = clf
Flog.flagged() and flog.write(
{
'locals': [{
'__return__': flog.serialize(__return__)
}],
'lineage': '__return__ = clf',
'lsn': 6
})
Flog.flagged() and flog.write({'end_function': 'train_model', 'lsn': 5})
return __return__
Flog.flagged() and flog.write({'end_function': 'train_model', 'lsn': 3})
if Flog.flagged():
flog = Flog(False)
Flog.flagged() and flog.write({
'file_path': '/Users/rogarcia/sandbox/iris_loop/iris_raw.py',
'lsn': 0
})
from sklearn import datasets
from sklearn import svm
from sklearn.model_selection import train_test_split
import random
import time
start_time = time.time()
Flog.flagged() and flog.write(
{
'locals': [{
'start_time': flog.serialize(start_time)
}],
'lineage': 'start_time = time.time()',
'lsn': 1
})
iris = datasets.load_iris()
Flog.flagged() and flog.write({
'locals': [{
'iris': flog.serialize(iris)
}],
'lineage': 'iris = datasets.load_iris()',
'lsn': 2
})
X_tr, X_te, y_tr, y_te = train_test_split(iris.data,
iris.target,
test_size=0.15,
from flor import Flog
if Flog.flagged():
flog = Flog(False)
Flog.flagged() and flog.write({'file_path':
'/Users/rogarcia/sandbox/iris/iris_annotated.py', 'lsn': 0})
from sklearn import datasets
from sklearn import svm
from sklearn.model_selection import train_test_split
import time
start_time = time.time()
Flog.flagged() and flog.write({'locals': [{'start_time': flog.serialize(
start_time)}], 'lineage': 'start_time = time.time()', 'lsn': 1})
iris = datasets.load_iris()
Flog.flagged() and flog.write({'locals': [{'iris': flog.serialize(iris)}],
'lineage': 'iris = datasets.load_iris()', 'lsn': 2})
X_tr, X_te, y_tr, y_te = train_test_split(iris.data, iris.target, test_size
=FLOR_PARAM('test_size', 0.15), random_state=FLOR_PARAM('random_state',
430))
Flog.flagged() and flog.write({'locals': [{'X_tr': flog.serialize(X_tr)}, {
'X_te': flog.serialize(X_te)}, {'y_tr': flog.serialize(y_tr)}, {'y_te':
flog.serialize(y_te)}], 'lineage':
'X_tr, X_te, y_tr, y_te = train_test_split(iris.data, iris.target, test_size =FLOR_PARAM("test_size", 0.15), random_state=FLOR_PARAM("random_state", 430))'
, 'lsn': 3})
clf = svm.SVC(gamma=FLOR_PARAM('gamma', 0.001), C=FLOR_PARAM('C', 100.0))
Flog.flagged() and flog.write({'locals': [{'clf': flog.serialize(clf)}],
'lineage':
'clf = svm.SVC(gamma=FLOR_PARAM("gamma", 0.001), C=FLOR_PARAM("C", 100.0))'
, 'lsn': 4})
clf.fit(X_tr, y_tr)
score = FLOR_METRIC('score', clf.score(X_te, y_te))
Flog.flagged() and flog.write({'locals': [{'score': flog.serialize(score)}],
def train(epoch):
try:
if Flog.flagged(option='start_function'):
flog = Flog()
Flog.flagged() and flog.write({'file_path': 'train.py', 'lsn': 0})
Flog.flagged() and flog.write({'start_function': 'train', 'lsn': 1})
Flog.flagged() and flog.write({
'lsn':
2,
'params': [{
'0.raw.epoch': flog.serialize(epoch, 'epoch')
}]
})
net.train()
for batch_index, (images,
labels) in enumerate(cifar100_training_loader):
Flog.flagged() and flog.write({'start_loop': 33, 'lsn': 4})
if epoch <= args.warm:
Flog.flagged() and flog.write({
'conditional_fork': '(epoch <= args.warm)',
'lsn': 6
})
warmup_scheduler.step()
else:
Flog.flagged() and flog.write({
'conditional_fork': 'not ((epoch <= args.warm))',
'lsn': 7
})
images = Variable(images)
Flog.flagged() and flog.write(
{
'locals': [{
'images': flog.serialize(images, 'images')
}],
'lineage': 'images = Variable(images)',
'lsn': 8
})
labels = Variable(labels)
Flog.flagged() and flog.write(
{
'locals': [{
'labels': flog.serialize(labels, 'labels')
}],
'lineage': 'labels = Variable(labels)',
'lsn': 9
})
labels = labels.cuda()
Flog.flagged() and flog.write(
{
'locals': [{
'labels': flog.serialize(labels, 'labels')
}],
'lineage': 'labels = labels.cuda()',
'lsn': 10
})
images = images.cuda()
Flog.flagged() and flog.write(
{
'locals': [{
'images': flog.serialize(images, 'images')
}],
'lineage': 'images = images.cuda()',
'lsn': 11
})
optimizer.zero_grad()
outputs = net(images)
Flog.flagged() and flog.write(
{
'locals': [{
'outputs': flog.serialize(outputs, 'outputs')
}],
'lineage': 'outputs = net(images)',
'lsn': 12
})
loss = loss_function(outputs, labels)
Flog.flagged() and flog.write(
{
'locals': [{
'loss': flog.serialize(loss, 'loss')
}],
'lineage': 'loss = loss_function(outputs, labels)',
'lsn': 13
})
loss.backward()
optimizer.step()
n_iter = (epoch -
1) * len(cifar100_training_loader) + batch_index + 1
Flog.flagged() and flog.write({
'locals': [{
'n_iter': flog.serialize(n_iter, 'n_iter')
}],
'lineage':
'n_iter = (epoch - 1) * len(cifar100_training_loader) + batch_index + 1',
'lsn':
14
})
last_layer = list(net.children())[-1]
Flog.flagged() and flog.write({
'locals': [{
'last_layer':
flog.serialize(last_layer, 'last_layer')
}],
'lineage':
'last_layer = list(net.children())[-1]',
'lsn':
15
})
for name, para in last_layer.named_parameters():
Flog.flagged() and flog.write({'start_loop': 52, 'lsn': 16})
if 'weight' in name:
Flog.flagged() and flog.write({
'conditional_fork': '("weight" in name)',
'lsn': 18
})
writer.add_scalar('LastLayerGradients/grad_norm2_weights',
para.grad.norm(), n_iter)
else:
Flog.flagged() and flog.write({
'conditional_fork': 'not (("weight" in name))',
'lsn': 19
})
if 'bias' in name:
Flog.flagged() and flog.write({
'conditional_fork': '("bias" in name)',
'lsn': 20
})
writer.add_scalar('LastLayerGradients/grad_norm2_bias',
para.grad.norm(), n_iter)
else:
Flog.flagged() and flog.write({
'conditional_fork': 'not (("bias" in name))',
'lsn': 21
})
Flog.flagged() and flog.write({'end_loop': 52, 'lsn': 17})
print(
'Training Epoch: {epoch} [{trained_samples}/{total_samples}]\tLoss: {:0.4f}\tLR: {:0.6f}'
.format(loss.item(),
optimizer.param_groups[0]['lr'],
epoch=epoch,
trained_samples=batch_index * args.b + len(images),
total_samples=len(cifar100_training_loader.dataset)))
writer.add_scalar('Train/loss', loss.item(), n_iter)
Flog.flagged() and flog.write({'end_loop': 33, 'lsn': 5})
for name, param in net.named_parameters():
Flog.flagged() and flog.write({'start_loop': 69, 'lsn': 22})
layer, attr = os.path.splitext(name)
Flog.flagged() and flog.write({
'locals': [{
'layer': flog.serialize(layer, 'layer')
}, {
'attr': flog.serialize(attr, 'attr')
}],
'lineage':
'layer, attr = os.path.splitext(name)',
'lsn':
24
})
attr = attr[1:]
Flog.flagged() and flog.write(
{
'locals': [{
'attr': flog.serialize(attr, 'attr')
}],
'lineage': 'attr = attr[1:]',
'lsn': 25
})
writer.add_histogram('{}/{}'.format(layer, attr), param, epoch)
Flog.flagged() and flog.write({'end_loop': 69, 'lsn': 23})
finally:
Flog.flagged() and flog.write({'end_function': 'train', 'lsn': 3})
Flog.flagged(option='end_function') and flog.writer.close()
def eval_training(epoch):
try:
if Flog.flagged(option='start_function'):
flog = Flog()
Flog.flagged() and flog.write({'file_path': 'train.py', 'lsn': 0})
Flog.flagged() and flog.write({
'start_function': 'eval_training',
'lsn': 1
})
Flog.flagged() and flog.write({
'lsn':
2,
'params': [{
'0.raw.epoch': flog.serialize(epoch, 'epoch')
}]
})
net.eval()
test_loss = 0.0
Flog.flagged() and flog.write(
{
'locals': [{
'test_loss': flog.serialize(test_loss, 'test_loss')
}],
'lineage': 'test_loss = 0.0',
'lsn': 4
})
correct = 0.0
Flog.flagged() and flog.write(
{
'locals': [{
'correct': flog.serialize(correct, 'correct')
}],
'lineage': 'correct = 0.0',
'lsn': 5
})
for images, labels in cifar100_test_loader:
Flog.flagged() and flog.write({'start_loop': 80, 'lsn': 6})
images = Variable(images)
Flog.flagged() and flog.write(
{
'locals': [{
'images': flog.serialize(images, 'images')
}],
'lineage': 'images = Variable(images)',
'lsn': 8
})
labels = Variable(labels)
Flog.flagged() and flog.write(
{
'locals': [{
'labels': flog.serialize(labels, 'labels')
}],
'lineage': 'labels = Variable(labels)',
'lsn': 9
})
images = images.cuda()
Flog.flagged() and flog.write(
{
'locals': [{
'images': flog.serialize(images, 'images')
}],
'lineage': 'images = images.cuda()',
'lsn': 10
})
labels = labels.cuda()
Flog.flagged() and flog.write(
{
'locals': [{
'labels': flog.serialize(labels, 'labels')
}],
'lineage': 'labels = labels.cuda()',
'lsn': 11
})
outputs = net(images)
Flog.flagged() and flog.write(
{
'locals': [{
'outputs': flog.serialize(outputs, 'outputs')
}],
'lineage': 'outputs = net(images)',
'lsn': 12
})
loss = loss_function(outputs, labels)
Flog.flagged() and flog.write(
{
'locals': [{
'loss': flog.serialize(loss, 'loss')
}],
'lineage': 'loss = loss_function(outputs, labels)',
'lsn': 13
})
test_loss += loss.item()
Flog.flagged() and flog.write({
'locals': [{
'test_loss': flog.serialize(test_loss, 'test_loss')
}],
'lineage':
'test_loss += loss.item()',
'lsn':
14
})
_, preds = outputs.max(1)
Flog.flagged() and flog.write({
'locals': [{
'_': flog.serialize(_, '_')
}, {
'preds': flog.serialize(preds, 'preds')
}],
'lineage':
'_, preds = outputs.max(1)',
'lsn':
15
})
correct += preds.eq(labels).sum()
Flog.flagged() and flog.write(
{
'locals': [{
'correct': flog.serialize(correct, 'correct')
}],
'lineage': 'correct += preds.eq(labels).sum()',
'lsn': 16
})
Flog.flagged() and flog.write({'end_loop': 80, 'lsn': 7})
print('Test set: Average loss: {:.4f}, Accuracy: {:.4f}'.format(
test_loss / len(cifar100_test_loader.dataset),
correct.float() / len(cifar100_test_loader.dataset)))
print()
writer.add_scalar('Test/Average loss',
test_loss / len(cifar100_test_loader.dataset), epoch)
writer.add_scalar('Test/Accuracy',
correct.float() / len(cifar100_test_loader.dataset),
epoch)
__return__ = correct.float() / len(cifar100_test_loader.dataset)
Flog.flagged() and flog.write({
'locals': [{
'__return__': flog.serialize(__return__, '__return__')
}],
'lineage':
'__return__ = correct.float() / len(cifar100_test_loader.dataset)',
'lsn':
17
})
return __return__
finally:
Flog.flagged() and flog.write({
'end_function': 'eval_training',
'lsn': 3
})
Flog.flagged(option='end_function') and flog.writer.close()
'end_function': 'eval_training',
'lsn': 3
})
Flog.flagged(option='end_function') and flog.writer.close()
if __name__ == '__main__':
Flog.flagged() and flog.write({
'conditional_fork': '(__name__ == "__main__")',
'lsn': 0
})
parser = argparse.ArgumentParser()
Flog.flagged() and flog.write(
{
'locals': [{
'parser': flog.serialize(parser, 'parser')
}],
'lineage': 'parser = argparse.ArgumentParser()',
'lsn': 2
})
parser.add_argument('-net', type=str, default='vgg16', help='net type')
parser.add_argument('-gpu', type=bool, default=True, help='use gpu or not')
parser.add_argument('-w',
type=int,
default=2,
help='number of workers for dataloader')
parser.add_argument('-b',
type=int,
default=128,
help='batch size for dataloader')
parser.add_argument('-s',
score = log.metric(clf.score(X_te, y_te))
Flog.flagged() and flog.write({
'locals': [{
'score': flog.serialize(score)
}],
'lineage': 'score = log.metric(clf.score(X_te, y_te))',
'lsn': 4
})
Flog.flagged() and flog.write({'end_function': 'test_model', 'lsn': 3})
movie_reviews = pd.read_json('data.json')
Flog.flagged() and flog.write(
{
'locals': [{
'movie_reviews': flog.serialize(movie_reviews)
}],
'lineage': 'movie_reviews = pd.read_json("data.json")',
'lsn': 1
})
movie_reviews['rating'] = movie_reviews['rating'].map(lambda x: 0
if x < 5 else 1)
Flog.flagged() and flog.write({
'locals': [{
'movie_reviews["rating"]':
flog.serialize(movie_reviews['rating'])
}],
'lineage':
'movie_reviews["rating"] = movie_reviews["rating"].map(lambda x: 0 if x < 5 else 1)',
'lsn':
2