def test_validate_fields_is_none(self): """Test: validate(self, event, result, fields) Exception: Parameter "fields" is None.""" validator = Validator() with self.assertRaises(Exception) as ctx_mng: validator.validate({}, {}, None) ex = ctx_mng.exception self.assertEqual('Parameter "fields" is None', str(ex))
def test_validate_body_is_empty(self): """Test: validate(self, event, result, fields) Error: body is empty.""" event = {'body': "{}"} validator = Validator() errors = validator.validate(event, {}, {}) self.assertEqual('body is empty', errors['body'])
def test_validate_body_is_none(self): """Test: validate(self, event, result, fields) Error: body is not set.""" event = {'body': None} validator = Validator() errors = validator.validate(event, {}, {}) self.assertEqual('body is not set', errors['body'])
def test_validate_fields_success(self): """Test: validate(self, event, result, fields)""" event = {'body': '{"title":"test-title"}'} fields = { 'title': ['is_set', 'is_empty'] } validator = Validator() errors = validator.validate(event, {}, fields) self.assertEqual({}, errors)
def test_validate_fields_is_empty(self): """Test: validate(self, event, result, fields) Error: field + ' is empty'""" event = {'body': '{"title":""}'} fields = { 'title': ['is_empty'] } validator = Validator() errors = validator.validate(event, {}, fields) self.assertEqual('title is empty', errors['title'])
def test_validate_fields_is_not_set(self): """Test: validate(self, event, result, fields) Error: field + ' is not set'""" event = {'body': '{"foo":"bar"}'} fields = { 'title': ['is_set'] } validator = Validator() errors = validator.validate(event, {}, fields) self.assertEqual('title is not set', errors['title'])
class SessionTrainer: def __init__(self, sess, components, params): self.sess = sess self.params = params self.trainer = Trainer(self.sess, components, self.params) self.__init_session(components) self.validator = Validator(self.sess, components, params) self.validation_loss = MAX_LOSS self.validation_rate_step = params['validation_rate_step'] def train(self): index = 0 while self.trainer.should_continue(): self.__train(index) index += 1 self.__save_state() def __init_session(self, components): self.saver = components['saver'] self.sess.run(tf.global_variables_initializer()) if self.params['state_fname']: session_path = os.path.join(self.params['state_folder'], self.params['state_fname']) self.saver.restore(self.sess, session_path) def __train(self, index): check = index % self.validation_rate_step == 0 self.trainer.train_batch(index, check) if check: self.__validate_loss(index) def __validate_loss(self, index): va = self.validator.validate(index) if va > self.validation_loss: self.trainer.reduce_learning_rate() self.validation_loss = va def __save_state(self): time_str = strftime("%d_%b_%Y_%H_%M", gmtime()) out_state_fname = self.params['saved_state_fname'] + '_{}.ckpt'.format( time_str) session_path = os.path.join(self.params['state_folder'], out_state_fname) self.saver.save( self.sess, os.path.join(self.params['state_folder'], session_path))
def train(model, config): # clear_output_dir() optimizer, lr_scheduler = init_training(model, config) logger = Logger(config) validator = Validator(model, logger, config) # cos_loss_fn = torch.nn.CosineEmbeddingLoss(margin=0.33) cos_loss_fn = ZeroCosineLoss(margin=0) pos_loss_fn = PositiveCosineLoss(margin=0.33) class_loss_fn = torch.nn.BCEWithLogitsLoss(reduction='none') # Data dataloader = setup_traindata(config) # Init progressbar n_batches = len(dataloader) n_epochs = math.ceil(config.optim_steps / n_batches) pbar = Progressbar(n_epochs, n_batches) optim_steps = 0 val_freq = config.validation_freq get_lr = lambda: optimizer.param_groups[0]['lr'] # Training loop for epoch in pbar(range(1, n_epochs + 1)): for batch_i, data in enumerate(dataloader, 1): pbar.update(epoch, batch_i) # Validation if optim_steps % val_freq == 0: validator.validate(optim_steps) # validator.val_normal(optim_steps) # Decrease learning rate if optim_steps % config.lr_step_frequency == 0: lr_scheduler.step() optimizer.zero_grad() cats, dogs = data inputs = torch.cat((cats, dogs)) outputs, class_outputs = model(inputs) cat_embs, dog_embs = outputs.chunk(2) cat_class_embs, dog_class_embs = class_outputs.chunk(2) catpair, dogpair, catdogpair = create_pairs(cat_embs, dog_embs) # Cosine similarity loss cat_loss = pos_loss_fn(catpair[0], catpair[1]) dog_loss = pos_loss_fn(dogpair[0], dogpair[1]) # y = torch.ones(catdogpair[0].size(0)).to(model.device) cat_dog_loss = cos_loss_fn(catdogpair[0], catdogpair[1]) class_l1 = class_loss_fn(cat_class_embs, torch.zeros_like(cat_class_embs)) class_l2 = class_loss_fn(dog_class_embs, torch.ones_like(dog_class_embs)) class_loss = (class_l1 + class_l2).mean() / 3 # loss_dict = dict(cat_loss=cat_loss, dog_loss=dog_loss, cat_dog_loss=cat_dog_loss, class_loss=class_loss) loss_dict = dict(cat_loss=cat_loss, dog_loss=dog_loss, cat_dog_loss=cat_dog_loss) # loss_dict = dict(cat_loss=cat_loss, cat_dog_loss=cat_dog_loss) # loss_dict = dict(cat_dog_loss=cat_dog_loss) # loss_dict = dict(class_loss=class_loss) loss = sum(loss_dict.values()) loss.backward() optimizer.step() optim_steps += 1 logger.log_cosine(catpair, dogpair, catdogpair, optim_steps) logger.log_loss(loss, optim_steps) logger.log_loss_percent(loss_dict, optim_steps) logger.log_learning_rate(get_lr(), optim_steps) # Frees up GPU memory del data; del outputs
class KanbanBoardService: COST_PER_HOUR = 1000 def __init__(self): self.dbms = DBDriver() self.validator = Validator() def create_task(self, event): result = { 'data': {}, 'status': 201 # Created. } fields = {'title': ['is_set', 'is_empty']} errors = self.validator.validate(event, result, fields) if not errors: body = event['body'] task = Task(title=body['title']) session = self.dbms.get_session() try: session.add(task) session.commit() result['data'] = self.map_to_json(task) except Exception as ex: session.rollback() result['status'] = 500 result['data'] = {'errors': {'db': 'session.commit error'}} print(ex) finally: session.close() else: result['data'] = {'errors': errors} return result def get_tasks(self, event): result = { 'data': {}, 'status': 200 # OK. } session = self.dbms.get_session() try: tasks = session.query(Task).all() objects = [] if tasks: for task in tasks: obj = self.map_to_json(task) objects.append(obj) result['data'] = objects except Exception as ex: result['status'] = 500 result['data'] = {'errors': {'db': 'session.query.all error'}} print(ex) finally: session.close() return result def update_task(self, event): body = dict() errors = dict() result = { 'data': {}, 'status': 205 # Reset Content. } fields = {'status': ['is_set', 'is_empty']} id = event['pathParameters']['id'] if id.isdigit(): id = int(id) else: errors['common'] = 'incorrect path' result['status'] = 404 # Not Found. if not errors: errors = self.validator.validate(event, result, fields) if not errors: body = event['body'] update = [ Task.Statuses.TODO.value, Task.Statuses.IN_PROGRESS.value, Task.Statuses.DONE.value ] if body['status'] not in update: errors['status'] = 'status is unknown' result['status'] = 400 # Bad Request. if not errors: session = self.dbms.get_session() try: task = session.query(Task).get(id) if body['status'] == task.status: errors['status'] = 'status is not changed' result['status'] = 400 # Bad Request. elif body['status'] - 1 != task.status: errors['status'] = 'status is incorrect' result['status'] = 409 # Conflict. if not errors: task.status = body['status'] if task.status == task.Statuses.IN_PROGRESS.value: task.start_time = datetime.now(tz=pytz.UTC) elif task.status == task.Statuses.DONE.value: task.end_time = datetime.now(tz=pytz.UTC) delta = task.end_time - task.start_time hours = delta.total_seconds() / 3600 task.payment = hours * KanbanBoardService.COST_PER_HOUR task.payment = round(task.payment, 2) session.add(task) session.commit() result['data'] = self.map_to_json(task) except Exception as ex: session.rollback() result['status'] = 500 result['data'] = {'errors': {'db': 'session.commit error'}} print(ex) finally: session.close() if errors: result['data'] = {'errors': errors} return result def map_to_json(self, task): obj = { "id": task.id, "title": task.title, "start_time": None, "end_time": None, "status": task.status, "payment": task.payment } if task.start_time: obj['start_time'] = task.start_time.strftime( '%Y-%m-%dT%H:%M:%S.%f') if task.end_time: obj['end_time'] = task.end_time.strftime('%Y-%m-%dT%H:%M:%S.%f') return obj