def test_validate_fields_is_none(self):
"""Test: validate(self, event, result, fields)
Exception: Parameter "fields" is None."""
validator = Validator()
with self.assertRaises(Exception) as ctx_mng:
validator.validate({}, {}, None)
ex = ctx_mng.exception
self.assertEqual('Parameter "fields" is None', str(ex))
def test_validate_body_is_empty(self):
"""Test: validate(self, event, result, fields)
Error: body is empty."""
event = {'body': "{}"}
validator = Validator()
errors = validator.validate(event, {}, {})
self.assertEqual('body is empty', errors['body'])
def test_validate_body_is_none(self):
"""Test: validate(self, event, result, fields)
Error: body is not set."""
event = {'body': None}
validator = Validator()
errors = validator.validate(event, {}, {})
self.assertEqual('body is not set', errors['body'])
def test_validate_fields_success(self):
"""Test: validate(self, event, result, fields)"""
event = {'body': '{"title":"test-title"}'}
fields = {
'title': ['is_set', 'is_empty']
}
validator = Validator()
errors = validator.validate(event, {}, fields)
self.assertEqual({}, errors)
def test_validate_fields_is_empty(self):
"""Test: validate(self, event, result, fields)
Error: field + ' is empty'"""
event = {'body': '{"title":""}'}
fields = {
'title': ['is_empty']
}
validator = Validator()
errors = validator.validate(event, {}, fields)
self.assertEqual('title is empty', errors['title'])
def test_validate_fields_is_not_set(self):
"""Test: validate(self, event, result, fields)
Error: field + ' is not set'"""
event = {'body': '{"foo":"bar"}'}
fields = {
'title': ['is_set']
}
validator = Validator()
errors = validator.validate(event, {}, fields)
self.assertEqual('title is not set', errors['title'])
class SessionTrainer:
def __init__(self, sess, components, params):
self.sess = sess
self.params = params
self.trainer = Trainer(self.sess, components, self.params)
self.__init_session(components)
self.validator = Validator(self.sess, components, params)
self.validation_loss = MAX_LOSS
self.validation_rate_step = params['validation_rate_step']
def train(self):
index = 0
while self.trainer.should_continue():
self.__train(index)
index += 1
self.__save_state()
def __init_session(self, components):
self.saver = components['saver']
self.sess.run(tf.global_variables_initializer())
if self.params['state_fname']:
session_path = os.path.join(self.params['state_folder'],
self.params['state_fname'])
self.saver.restore(self.sess, session_path)
def __train(self, index):
check = index % self.validation_rate_step == 0
self.trainer.train_batch(index, check)
if check:
self.__validate_loss(index)
def __validate_loss(self, index):
va = self.validator.validate(index)
if va > self.validation_loss:
self.trainer.reduce_learning_rate()
self.validation_loss = va
def __save_state(self):
time_str = strftime("%d_%b_%Y_%H_%M", gmtime())
out_state_fname = self.params['saved_state_fname'] + '_{}.ckpt'.format(
time_str)
session_path = os.path.join(self.params['state_folder'],
out_state_fname)
self.saver.save(
self.sess, os.path.join(self.params['state_folder'], session_path))
def train(model, config):
# clear_output_dir()
optimizer, lr_scheduler = init_training(model, config)
logger = Logger(config)
validator = Validator(model, logger, config)
# cos_loss_fn = torch.nn.CosineEmbeddingLoss(margin=0.33)
cos_loss_fn = ZeroCosineLoss(margin=0)
pos_loss_fn = PositiveCosineLoss(margin=0.33)
class_loss_fn = torch.nn.BCEWithLogitsLoss(reduction='none')
# Data
dataloader = setup_traindata(config)
# Init progressbar
n_batches = len(dataloader)
n_epochs = math.ceil(config.optim_steps / n_batches)
pbar = Progressbar(n_epochs, n_batches)
optim_steps = 0
val_freq = config.validation_freq
get_lr = lambda: optimizer.param_groups[0]['lr']
# Training loop
for epoch in pbar(range(1, n_epochs + 1)):
for batch_i, data in enumerate(dataloader, 1):
pbar.update(epoch, batch_i)
# Validation
if optim_steps % val_freq == 0:
validator.validate(optim_steps)
# validator.val_normal(optim_steps)
# Decrease learning rate
if optim_steps % config.lr_step_frequency == 0:
lr_scheduler.step()
optimizer.zero_grad()
cats, dogs = data
inputs = torch.cat((cats, dogs))
outputs, class_outputs = model(inputs)
cat_embs, dog_embs = outputs.chunk(2)
cat_class_embs, dog_class_embs = class_outputs.chunk(2)
catpair, dogpair, catdogpair = create_pairs(cat_embs, dog_embs)
# Cosine similarity loss
cat_loss = pos_loss_fn(catpair[0], catpair[1])
dog_loss = pos_loss_fn(dogpair[0], dogpair[1])
# y = torch.ones(catdogpair[0].size(0)).to(model.device)
cat_dog_loss = cos_loss_fn(catdogpair[0], catdogpair[1])
class_l1 = class_loss_fn(cat_class_embs, torch.zeros_like(cat_class_embs))
class_l2 = class_loss_fn(dog_class_embs, torch.ones_like(dog_class_embs))
class_loss = (class_l1 + class_l2).mean() / 3
# loss_dict = dict(cat_loss=cat_loss, dog_loss=dog_loss, cat_dog_loss=cat_dog_loss, class_loss=class_loss)
loss_dict = dict(cat_loss=cat_loss, dog_loss=dog_loss, cat_dog_loss=cat_dog_loss)
# loss_dict = dict(cat_loss=cat_loss, cat_dog_loss=cat_dog_loss)
# loss_dict = dict(cat_dog_loss=cat_dog_loss)
# loss_dict = dict(class_loss=class_loss)
loss = sum(loss_dict.values())
loss.backward()
optimizer.step()
optim_steps += 1
logger.log_cosine(catpair, dogpair, catdogpair, optim_steps)
logger.log_loss(loss, optim_steps)
logger.log_loss_percent(loss_dict, optim_steps)
logger.log_learning_rate(get_lr(), optim_steps)
# Frees up GPU memory
del data; del outputs
class KanbanBoardService:
COST_PER_HOUR = 1000
def __init__(self):
self.dbms = DBDriver()
self.validator = Validator()
def create_task(self, event):
result = {
'data': {},
'status': 201 # Created.
}
fields = {'title': ['is_set', 'is_empty']}
errors = self.validator.validate(event, result, fields)
if not errors:
body = event['body']
task = Task(title=body['title'])
session = self.dbms.get_session()
try:
session.add(task)
session.commit()
result['data'] = self.map_to_json(task)
except Exception as ex:
session.rollback()
result['status'] = 500
result['data'] = {'errors': {'db': 'session.commit error'}}
print(ex)
finally:
session.close()
else:
result['data'] = {'errors': errors}
return result
def get_tasks(self, event):
result = {
'data': {},
'status': 200 # OK.
}
session = self.dbms.get_session()
try:
tasks = session.query(Task).all()
objects = []
if tasks:
for task in tasks:
obj = self.map_to_json(task)
objects.append(obj)
result['data'] = objects
except Exception as ex:
result['status'] = 500
result['data'] = {'errors': {'db': 'session.query.all error'}}
print(ex)
finally:
session.close()
return result
def update_task(self, event):
body = dict()
errors = dict()
result = {
'data': {},
'status': 205 # Reset Content.
}
fields = {'status': ['is_set', 'is_empty']}
id = event['pathParameters']['id']
if id.isdigit():
id = int(id)
else:
errors['common'] = 'incorrect path'
result['status'] = 404 # Not Found.
if not errors:
errors = self.validator.validate(event, result, fields)
if not errors:
body = event['body']
update = [
Task.Statuses.TODO.value, Task.Statuses.IN_PROGRESS.value,
Task.Statuses.DONE.value
]
if body['status'] not in update:
errors['status'] = 'status is unknown'
result['status'] = 400 # Bad Request.
if not errors:
session = self.dbms.get_session()
try:
task = session.query(Task).get(id)
if body['status'] == task.status:
errors['status'] = 'status is not changed'
result['status'] = 400 # Bad Request.
elif body['status'] - 1 != task.status:
errors['status'] = 'status is incorrect'
result['status'] = 409 # Conflict.
if not errors:
task.status = body['status']
if task.status == task.Statuses.IN_PROGRESS.value:
task.start_time = datetime.now(tz=pytz.UTC)
elif task.status == task.Statuses.DONE.value:
task.end_time = datetime.now(tz=pytz.UTC)
delta = task.end_time - task.start_time
hours = delta.total_seconds() / 3600
task.payment = hours * KanbanBoardService.COST_PER_HOUR
task.payment = round(task.payment, 2)
session.add(task)
session.commit()
result['data'] = self.map_to_json(task)
except Exception as ex:
session.rollback()
result['status'] = 500
result['data'] = {'errors': {'db': 'session.commit error'}}
print(ex)
finally:
session.close()
if errors:
result['data'] = {'errors': errors}
return result
def map_to_json(self, task):
obj = {
"id": task.id,
"title": task.title,
"start_time": None,
"end_time": None,
"status": task.status,
"payment": task.payment
}
if task.start_time:
obj['start_time'] = task.start_time.strftime(
'%Y-%m-%dT%H:%M:%S.%f')
if task.end_time:
obj['end_time'] = task.end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')
return obj