def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) validator = Validator() AssetsDirValidator(serializer.validated_data, 'assets_dir', validator) validator.is_valid() data = serializer.validated_data old_data = self.find_old_project(data.get('assets_dir')) if old_data is not None: del old_data['title'] del old_data['description'] old_data.update(data) with models.suppress_auto_now(models.Project, ['created_at', 'updated_at']): project = models.Project.objects.create(**old_data) serializer = self.get_serializer_class()(project) tasks.import_project.delay(old_data['alias']) else: self.perform_create(serializer) headers = self.get_success_headers(serializer.data) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
def test_validate_body_is_empty(self): """Test: validate(self, event, result, fields) Error: body is empty.""" event = {'body': "{}"} validator = Validator() errors = validator.validate(event, {}, {}) self.assertEqual('body is empty', errors['body'])
def test_validate_body_is_none(self): """Test: validate(self, event, result, fields) Error: body is not set.""" event = {'body': None} validator = Validator() errors = validator.validate(event, {}, {}) self.assertEqual('body is not set', errors['body'])
def test_validate_fields_is_none(self): """Test: validate(self, event, result, fields) Exception: Parameter "fields" is None.""" validator = Validator() with self.assertRaises(Exception) as ctx_mng: validator.validate({}, {}, None) ex = ctx_mng.exception self.assertEqual('Parameter "fields" is None', str(ex))
def __init__(self, sess, components, params): self.sess = sess self.params = params self.trainer = Trainer(self.sess, components, self.params) self.__init_session(components) self.validator = Validator(self.sess, components, params) self.validation_loss = MAX_LOSS self.validation_rate_step = params['validation_rate_step']
def test_validate_fields_success(self): """Test: validate(self, event, result, fields)""" event = {'body': '{"title":"test-title"}'} fields = { 'title': ['is_set', 'is_empty'] } validator = Validator() errors = validator.validate(event, {}, fields) self.assertEqual({}, errors)
def test_validate_fields_is_not_set(self): """Test: validate(self, event, result, fields) Error: field + ' is not set'""" event = {'body': '{"foo":"bar"}'} fields = { 'title': ['is_set'] } validator = Validator() errors = validator.validate(event, {}, fields) self.assertEqual('title is not set', errors['title'])
def test_validate_fields_is_empty(self): """Test: validate(self, event, result, fields) Error: field + ' is empty'""" event = {'body': '{"title":""}'} fields = { 'title': ['is_empty'] } validator = Validator() errors = validator.validate(event, {}, fields) self.assertEqual('title is empty', errors['title'])
class TestRunner: validator = Validator( os.path.join(FileUtils.get_schemadir(), "merged-syntax.xsd")) @staticmethod def testOK(mapping_sample, case_prefix): files = os.listdir(mapping_sample) ok_prefix = case_prefix + "_ok" for sample_file in files: if sample_file.startswith(ok_prefix) is True: file_path = os.path.join(mapping_sample, sample_file) if TestRunner.validator.validate_file(file_path, verbose=False) is False: TestRunner.validator.validate_file(file_path, verbose=True) logger.error(sample_file + " is not valid, it should be") return False logger.info(sample_file + " is valid: fine") return True @staticmethod def testKO(mapping_sample, case_prefix): ko_prefix = case_prefix + "_ko" files = os.listdir(mapping_sample) for sample_file in files: if sample_file.startswith(ko_prefix) is True: file_path = os.path.join(mapping_sample, sample_file) if TestRunner.validator.validate_file(file_path, verbose=False) is True: logger.error(sample_file) return False logger.info(sample_file + " is not valid: fine") return True
def __init__(self, filepath='files/budget.xlsx'): logging.basicConfig( level=logging.INFO, filename="./logs/uploader_{:%Y_%m_%d_%H_%M}.log".format(datetime.datetime.now()), datefmt="%Y-%m-%d %H:%M:%S", format='%(asctime)s %(levelname)-8s %(message)s' ) self.logger = logging.getLogger("upload") self.mysql = mysql.connector.connect( host=mysql_conn["host"], user=mysql_conn["user"], password=mysql_conn["pass"], database=mysql_conn["db"] ) self.cursor = self.mysql.cursor(dictionary=True,buffered=True) self.filepath = filepath data = pd.read_excel (self.filepath, engine='openpyxl') self.dataframe_columns = [ 'id', 'year', 'month', 'type', 'classification', 'code', 'descriptor', 'lab_code', 'lab', 'business_name', 'business_unit', 'category_set', 'macrocategory', 'category_1', 'category_2', 'client_type', 'channel', 'sale', 'contribution', 'units', 'price' ] self.df = pd.DataFrame(data) self.validator = Validator(self.logger)
def proof_of_work(self): last_block = self.__chain[-1] last_hash = hash_utils.hash_block(last_block) proof = 0 while not Validator.validate_proof(self.__open_transactions, last_hash, proof): proof += 1 return proof
def __verify_range(self, row_id): reading = self.__dao.get_sensor_reading(row_id) if not reading: print('No reading saved.') return sensor_data_range = FileAccess.get_sensor_data_range() if sensor_data_range is None: print('No range config available. Cannot verify...') return result_temp = Validator.verify_temp(reading[SensorDataCol.TEMP], sensor_data_range) result_humid = Validator.verify_humid(reading[SensorDataCol.TEMP], sensor_data_range) temp_within_range = result_temp[0] humid_within_range = result_humid[0] if not (temp_within_range and humid_within_range): self.__notifier.notify_out_of_range_reading( reading, result_temp, result_humid)
class SessionTrainer: def __init__(self, sess, components, params): self.sess = sess self.params = params self.trainer = Trainer(self.sess, components, self.params) self.__init_session(components) self.validator = Validator(self.sess, components, params) self.validation_loss = MAX_LOSS self.validation_rate_step = params['validation_rate_step'] def train(self): index = 0 while self.trainer.should_continue(): self.__train(index) index += 1 self.__save_state() def __init_session(self, components): self.saver = components['saver'] self.sess.run(tf.global_variables_initializer()) if self.params['state_fname']: session_path = os.path.join(self.params['state_folder'], self.params['state_fname']) self.saver.restore(self.sess, session_path) def __train(self, index): check = index % self.validation_rate_step == 0 self.trainer.train_batch(index, check) if check: self.__validate_loss(index) def __validate_loss(self, index): va = self.validator.validate(index) if va > self.validation_loss: self.trainer.reduce_learning_rate() self.validation_loss = va def __save_state(self): time_str = strftime("%d_%b_%Y_%H_%M", gmtime()) out_state_fname = self.params['saved_state_fname'] + '_{}.ckpt'.format( time_str) session_path = os.path.join(self.params['state_folder'], out_state_fname) self.saver.save( self.sess, os.path.join(self.params['state_folder'], session_path))
def __process_day(self, day, data_range): min_temp = self.__dao.get_min_temp_of_day(day) max_temp = self.__dao.get_max_temp_of_day(day) min_humid = self.__dao.get_min_humid_of_day(day) max_humid = self.__dao.get_max_humid_of_day(day) day_result = Validator.verify_temp_humid_of_day( min_temp, max_temp, min_humid, max_humid, data_range) if not day_result[0]: day_status = 'BAD: ' + day_result[1] else: day_status = day_result[1] day_row = [day, day_status] self.__report_data.append(day_row)
def input_listener(self): session_active = True while session_active: self.print_menu() user_choice = input('Please enter a choice: ') if user_choice == '1': tx_data = self.get_tx_info() receiver, amount = tx_data if self.blockchain.create_transaction(self.id, receiver, amount): print('Transaction added') else: print('Transaction failed') print(self.blockchain.open_transactions) elif user_choice == '2': self.blockchain.mine_block(self.id) elif user_choice == '3': self.print_blockchain() elif user_choice == '4': if Validator.validate_transaction( self.blockchain.open_transactions, self.blockchain.get_balance): print('Transactions validated') else: print('Could not validate all transactions') elif user_choice == 'q' or user_choice == 'Q': session_active = False break else: print('invalid input') if not Validator.is_valid_chain(self.blockchain.chain): self.print_blockchain() print('Corrupted Blockchain, exiting') break
def create_transaction(self, sender, receiver, amount): ''' Creates a transaction between users, this transaction is then validated. If the user has enough funds the transaction is added onto a list of open transactions. Args: sender: The user who will send funds receiver: The user who will receive the funds amount: The amount of funds to be sent Returns: True if the transaction was successful. False if the transaction fails. ''' tx = Transaction(sender, receiver, amount) if Validator.validate_transaction(tx, self.get_balance): self.__open_transactions.append(tx) return True return False
def train_with_correspondences(save_folder, startnet, args): device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") check_mkdir(save_folder) writer = SummaryWriter(save_folder) # Network and weight loading model_config = model_configs.PspnetCityscapesConfig() net = model_config.init_network().to(device) if args['snapshot'] == 'latest': args['snapshot'] = get_latest_network_name(save_folder) if len(args['snapshot']) == 0: # If start from beginning state_dict = torch.load(startnet) # needed since we slightly changed the structure of the network in # pspnet state_dict = rename_keys_to_match(state_dict) net.load_state_dict(state_dict) # load original weights start_iter = 0 args['best_record'] = { 'iter': 0, 'val_loss': 1e10, 'acc': 0, 'acc_cls': 0, 'mean_iu': 0, 'fwavacc': 0 } else: # If continue training print('training resumes from ' + args['snapshot']) net.load_state_dict( torch.load(os.path.join(save_folder, args['snapshot']))) # load weights split_snapshot = args['snapshot'].split('_') start_iter = int(split_snapshot[1]) with open(os.path.join(save_folder, 'bestval.txt')) as f: best_val_dict_str = f.read() args['best_record'] = eval(best_val_dict_str.rstrip()) net.train() freeze_bn(net) # Data loading setup if args['corr_set'] == 'rc': corr_set_config = data_configs.RobotcarConfig() elif args['corr_set'] == 'cmu': corr_set_config = data_configs.CmuConfig() sliding_crop_im = joint_transforms.SlidingCropImageOnly( 713, args['stride_rate']) input_transform = model_config.input_transform pre_validation_transform = model_config.pre_validation_transform target_transform = extended_transforms.MaskToTensor() train_joint_transform_seg = joint_transforms.Compose([ joint_transforms.Resize(1024), joint_transforms.RandomRotate(10), joint_transforms.RandomHorizontallyFlip(), joint_transforms.RandomCrop(713) ]) train_joint_transform_corr = corr_transforms.Compose([ corr_transforms.CorrResize(1024), corr_transforms.CorrRandomCrop(713) ]) # keep list of segmentation loaders and validators seg_loaders = list() validators = list() # Correspondences corr_set = correspondences.Correspondences( corr_set_config.correspondence_path, corr_set_config.correspondence_im_path, input_size=(713, 713), mean_std=model_config.mean_std, input_transform=input_transform, joint_transform=train_joint_transform_corr) corr_loader = DataLoader(corr_set, batch_size=args['train_batch_size'], num_workers=args['n_workers'], shuffle=True) # Cityscapes Training c_config = data_configs.CityscapesConfig() seg_set_cs = cityscapes.CityScapes( c_config.train_im_folder, c_config.train_seg_folder, c_config.im_file_ending, c_config.seg_file_ending, id_to_trainid=c_config.id_to_trainid, joint_transform=train_joint_transform_seg, sliding_crop=None, transform=input_transform, target_transform=target_transform) seg_loader_cs = DataLoader(seg_set_cs, batch_size=args['train_batch_size'], num_workers=args['n_workers'], shuffle=True) seg_loaders.append(seg_loader_cs) # Cityscapes Validation val_set_cs = cityscapes.CityScapes( c_config.val_im_folder, c_config.val_seg_folder, c_config.im_file_ending, c_config.seg_file_ending, id_to_trainid=c_config.id_to_trainid, sliding_crop=sliding_crop_im, transform=input_transform, target_transform=target_transform, transform_before_sliding=pre_validation_transform) val_loader_cs = DataLoader(val_set_cs, batch_size=1, num_workers=args['n_workers'], shuffle=False) validator_cs = Validator(val_loader_cs, n_classes=c_config.n_classes, save_snapshot=False, extra_name_str='Cityscapes') validators.append(validator_cs) # Vistas Training and Validation if args['include_vistas']: v_config = data_configs.VistasConfig( use_subsampled_validation_set=True, use_cityscapes_classes=True) seg_set_vis = cityscapes.CityScapes( v_config.train_im_folder, v_config.train_seg_folder, v_config.im_file_ending, v_config.seg_file_ending, id_to_trainid=v_config.id_to_trainid, joint_transform=train_joint_transform_seg, sliding_crop=None, transform=input_transform, target_transform=target_transform) seg_loader_vis = DataLoader(seg_set_vis, batch_size=args['train_batch_size'], num_workers=args['n_workers'], shuffle=True) seg_loaders.append(seg_loader_vis) val_set_vis = cityscapes.CityScapes( v_config.val_im_folder, v_config.val_seg_folder, v_config.im_file_ending, v_config.seg_file_ending, id_to_trainid=v_config.id_to_trainid, sliding_crop=sliding_crop_im, transform=input_transform, target_transform=target_transform, transform_before_sliding=pre_validation_transform) val_loader_vis = DataLoader(val_set_vis, batch_size=1, num_workers=args['n_workers'], shuffle=False) validator_vis = Validator(val_loader_vis, n_classes=v_config.n_classes, save_snapshot=False, extra_name_str='Vistas') validators.append(validator_vis) else: seg_loader_vis = None map_validator = None # Extra Training extra_seg_set = cityscapes.CityScapes( corr_set_config.train_im_folder, corr_set_config.train_seg_folder, corr_set_config.im_file_ending, corr_set_config.seg_file_ending, id_to_trainid=corr_set_config.id_to_trainid, joint_transform=train_joint_transform_seg, sliding_crop=None, transform=input_transform, target_transform=target_transform) extra_seg_loader = DataLoader(extra_seg_set, batch_size=args['train_batch_size'], num_workers=args['n_workers'], shuffle=True) seg_loaders.append(extra_seg_loader) # Extra Validation extra_val_set = cityscapes.CityScapes( corr_set_config.val_im_folder, corr_set_config.val_seg_folder, corr_set_config.im_file_ending, corr_set_config.seg_file_ending, id_to_trainid=corr_set_config.id_to_trainid, sliding_crop=sliding_crop_im, transform=input_transform, target_transform=target_transform, transform_before_sliding=pre_validation_transform) extra_val_loader = DataLoader(extra_val_set, batch_size=1, num_workers=args['n_workers'], shuffle=False) extra_validator = Validator(extra_val_loader, n_classes=corr_set_config.n_classes, save_snapshot=True, extra_name_str='Extra') validators.append(extra_validator) # Loss setup if args['corr_loss_type'] == 'class': corr_loss_fct = CorrClassLoss(input_size=[713, 713]) else: corr_loss_fct = FeatureLoss( input_size=[713, 713], loss_type=args['corr_loss_type'], feat_dist_threshold_match=args['feat_dist_threshold_match'], feat_dist_threshold_nomatch=args['feat_dist_threshold_nomatch'], n_not_matching=0) seg_loss_fct = torch.nn.CrossEntropyLoss( reduction='elementwise_mean', ignore_index=cityscapes.ignore_label).to(device) # Optimizer setup optimizer = optim.SGD([{ 'params': [ param for name, param in net.named_parameters() if name[-4:] == 'bias' and param.requires_grad ], 'lr': 2 * args['lr'] }, { 'params': [ param for name, param in net.named_parameters() if name[-4:] != 'bias' and param.requires_grad ], 'lr': args['lr'], 'weight_decay': args['weight_decay'] }], momentum=args['momentum'], nesterov=True) if len(args['snapshot']) > 0: optimizer.load_state_dict( torch.load(os.path.join(save_folder, 'opt_' + args['snapshot']))) optimizer.param_groups[0]['lr'] = 2 * args['lr'] optimizer.param_groups[1]['lr'] = args['lr'] open(os.path.join(save_folder, str(datetime.datetime.now()) + '.txt'), 'w').write(str(args) + '\n\n') if len(args['snapshot']) == 0: f_handle = open(os.path.join(save_folder, 'log.log'), 'w', buffering=1) else: clean_log_before_continuing(os.path.join(save_folder, 'log.log'), start_iter) f_handle = open(os.path.join(save_folder, 'log.log'), 'a', buffering=1) ########################################################################## # # MAIN TRAINING CONSISTS OF ALL SEGMENTATION LOSSES AND A CORRESPONDENCE LOSS # ########################################################################## softm = torch.nn.Softmax2d() val_iter = 0 train_corr_loss = AverageMeter() train_seg_cs_loss = AverageMeter() train_seg_extra_loss = AverageMeter() train_seg_vis_loss = AverageMeter() seg_loss_meters = list() seg_loss_meters.append(train_seg_cs_loss) if args['include_vistas']: seg_loss_meters.append(train_seg_vis_loss) seg_loss_meters.append(train_seg_extra_loss) curr_iter = start_iter for i in range(args['max_iter']): optimizer.param_groups[0]['lr'] = 2 * args['lr'] * ( 1 - float(curr_iter) / args['max_iter'])**args['lr_decay'] optimizer.param_groups[1]['lr'] = args['lr'] * ( 1 - float(curr_iter) / args['max_iter'])**args['lr_decay'] ####################################################################### # SEGMENTATION UPDATE STEP ####################################################################### # for si, seg_loader in enumerate(seg_loaders): # get segmentation training sample inputs, gts = next(iter(seg_loader)) slice_batch_pixel_size = inputs.size(0) * inputs.size( 2) * inputs.size(3) inputs = inputs.to(device) gts = gts.to(device) optimizer.zero_grad() outputs, aux = net(inputs) main_loss = args['seg_loss_weight'] * seg_loss_fct(outputs, gts) aux_loss = args['seg_loss_weight'] * seg_loss_fct(aux, gts) loss = main_loss + 0.4 * aux_loss loss.backward() optimizer.step() seg_loss_meters[si].update(main_loss.item(), slice_batch_pixel_size) ####################################################################### # CORRESPONDENCE UPDATE STEP ####################################################################### if args['corr_loss_weight'] > 0 and args[ 'n_iterations_before_corr_loss'] < curr_iter: img_ref, img_other, pts_ref, pts_other, weights = next( iter(corr_loader)) # Transfer data to device # img_ref is from the "good" sequence with generally better # segmentation results img_ref = img_ref.to(device) img_other = img_other.to(device) pts_ref = [p.to(device) for p in pts_ref] pts_other = [p.to(device) for p in pts_other] weights = [w.to(device) for w in weights] # Forward pass if args['corr_loss_type'] == 'hingeF': # Works on features net.output_all = True with torch.no_grad(): output_feat_ref, aux_feat_ref, output_ref, aux_ref = net( img_ref) output_feat_other, aux_feat_other, output_other, aux_other = net( img_other ) # output1 must be last to backpropagate derivative correctly net.output_all = False else: # Works on class probs with torch.no_grad(): output_ref, aux_ref = net(img_ref) if args['corr_loss_type'] != 'hingeF' and args[ 'corr_loss_type'] != 'hingeC': output_ref = softm(output_ref) aux_ref = softm(aux_ref) # output1 must be last to backpropagate derivative correctly output_other, aux_other = net(img_other) if args['corr_loss_type'] != 'hingeF' and args[ 'corr_loss_type'] != 'hingeC': output_other = softm(output_other) aux_other = softm(aux_other) # Correspondence filtering pts_ref_orig, pts_other_orig, weights_orig, batch_inds_to_keep_orig = correspondences.refine_correspondence_sample( output_ref, output_other, pts_ref, pts_other, weights, remove_same_class=args['remove_same_class'], remove_classes=args['classes_to_ignore']) pts_ref_orig = [ p for b, p in zip(batch_inds_to_keep_orig, pts_ref_orig) if b.item() > 0 ] pts_other_orig = [ p for b, p in zip(batch_inds_to_keep_orig, pts_other_orig) if b.item() > 0 ] weights_orig = [ p for b, p in zip(batch_inds_to_keep_orig, weights_orig) if b.item() > 0 ] if args['corr_loss_type'] == 'hingeF': # remove entire samples if needed output_vals_ref = output_feat_ref[batch_inds_to_keep_orig] output_vals_other = output_feat_other[batch_inds_to_keep_orig] else: # remove entire samples if needed output_vals_ref = output_ref[batch_inds_to_keep_orig] output_vals_other = output_other[batch_inds_to_keep_orig] pts_ref_aux, pts_other_aux, weights_aux, batch_inds_to_keep_aux = correspondences.refine_correspondence_sample( aux_ref, aux_other, pts_ref, pts_other, weights, remove_same_class=args['remove_same_class'], remove_classes=args['classes_to_ignore']) pts_ref_aux = [ p for b, p in zip(batch_inds_to_keep_aux, pts_ref_aux) if b.item() > 0 ] pts_other_aux = [ p for b, p in zip(batch_inds_to_keep_aux, pts_other_aux) if b.item() > 0 ] weights_aux = [ p for b, p in zip(batch_inds_to_keep_aux, weights_aux) if b.item() > 0 ] if args['corr_loss_type'] == 'hingeF': # remove entire samples if needed aux_vals_ref = aux_feat_ref[batch_inds_to_keep_orig] aux_vals_other = aux_feat_other[batch_inds_to_keep_orig] else: # remove entire samples if needed aux_vals_ref = aux_ref[batch_inds_to_keep_aux] aux_vals_other = aux_other[batch_inds_to_keep_aux] optimizer.zero_grad() # correspondence loss if output_vals_ref.size(0) > 0: loss_corr_hr = corr_loss_fct(output_vals_ref, output_vals_other, pts_ref_orig, pts_other_orig, weights_orig) else: loss_corr_hr = 0 * output_vals_other.sum() if aux_vals_ref.size(0) > 0: loss_corr_aux = corr_loss_fct( aux_vals_ref, aux_vals_other, pts_ref_aux, pts_other_aux, weights_aux) # use output from img1 as "reference" else: loss_corr_aux = 0 * aux_vals_other.sum() loss_corr = args['corr_loss_weight'] * \ (loss_corr_hr + 0.4 * loss_corr_aux) loss_corr.backward() optimizer.step() train_corr_loss.update(loss_corr.item()) ####################################################################### # LOGGING ETC ####################################################################### curr_iter += 1 val_iter += 1 writer.add_scalar('train_seg_loss_cs', train_seg_cs_loss.avg, curr_iter) writer.add_scalar('train_seg_loss_extra', train_seg_extra_loss.avg, curr_iter) writer.add_scalar('train_seg_loss_vis', train_seg_vis_loss.avg, curr_iter) writer.add_scalar('train_corr_loss', train_corr_loss.avg, curr_iter) writer.add_scalar('lr', optimizer.param_groups[1]['lr'], curr_iter) if (i + 1) % args['print_freq'] == 0: str2write = '[iter %d / %d], [train corr loss %.5f] , [seg cs loss %.5f], [seg vis loss %.5f], [seg extra loss %.5f]. [lr %.10f]' % ( curr_iter, len(corr_loader), train_corr_loss.avg, train_seg_cs_loss.avg, train_seg_vis_loss.avg, train_seg_extra_loss.avg, optimizer.param_groups[1]['lr']) print(str2write) f_handle.write(str2write + "\n") if val_iter >= args['val_interval']: val_iter = 0 for validator in validators: validator.run(net, optimizer, args, curr_iter, save_folder, f_handle, writer=writer) # Post training f_handle.close() writer.close()
def __init__(self): self.dbms = DBDriver() self.validator = Validator()
def train(model, config): # clear_output_dir() optimizer, lr_scheduler = init_training(model, config) logger = Logger(config) validator = Validator(model, logger, config) # cos_loss_fn = torch.nn.CosineEmbeddingLoss(margin=0.33) cos_loss_fn = ZeroCosineLoss(margin=0) pos_loss_fn = PositiveCosineLoss(margin=0.33) class_loss_fn = torch.nn.BCEWithLogitsLoss(reduction='none') # Data dataloader = setup_traindata(config) # Init progressbar n_batches = len(dataloader) n_epochs = math.ceil(config.optim_steps / n_batches) pbar = Progressbar(n_epochs, n_batches) optim_steps = 0 val_freq = config.validation_freq get_lr = lambda: optimizer.param_groups[0]['lr'] # Training loop for epoch in pbar(range(1, n_epochs + 1)): for batch_i, data in enumerate(dataloader, 1): pbar.update(epoch, batch_i) # Validation if optim_steps % val_freq == 0: validator.validate(optim_steps) # validator.val_normal(optim_steps) # Decrease learning rate if optim_steps % config.lr_step_frequency == 0: lr_scheduler.step() optimizer.zero_grad() cats, dogs = data inputs = torch.cat((cats, dogs)) outputs, class_outputs = model(inputs) cat_embs, dog_embs = outputs.chunk(2) cat_class_embs, dog_class_embs = class_outputs.chunk(2) catpair, dogpair, catdogpair = create_pairs(cat_embs, dog_embs) # Cosine similarity loss cat_loss = pos_loss_fn(catpair[0], catpair[1]) dog_loss = pos_loss_fn(dogpair[0], dogpair[1]) # y = torch.ones(catdogpair[0].size(0)).to(model.device) cat_dog_loss = cos_loss_fn(catdogpair[0], catdogpair[1]) class_l1 = class_loss_fn(cat_class_embs, torch.zeros_like(cat_class_embs)) class_l2 = class_loss_fn(dog_class_embs, torch.ones_like(dog_class_embs)) class_loss = (class_l1 + class_l2).mean() / 3 # loss_dict = dict(cat_loss=cat_loss, dog_loss=dog_loss, cat_dog_loss=cat_dog_loss, class_loss=class_loss) loss_dict = dict(cat_loss=cat_loss, dog_loss=dog_loss, cat_dog_loss=cat_dog_loss) # loss_dict = dict(cat_loss=cat_loss, cat_dog_loss=cat_dog_loss) # loss_dict = dict(cat_dog_loss=cat_dog_loss) # loss_dict = dict(class_loss=class_loss) loss = sum(loss_dict.values()) loss.backward() optimizer.step() optim_steps += 1 logger.log_cosine(catpair, dogpair, catdogpair, optim_steps) logger.log_loss(loss, optim_steps) logger.log_loss_percent(loss_dict, optim_steps) logger.log_learning_rate(get_lr(), optim_steps) # Frees up GPU memory del data; del outputs
class KanbanBoardService: COST_PER_HOUR = 1000 def __init__(self): self.dbms = DBDriver() self.validator = Validator() def create_task(self, event): result = { 'data': {}, 'status': 201 # Created. } fields = {'title': ['is_set', 'is_empty']} errors = self.validator.validate(event, result, fields) if not errors: body = event['body'] task = Task(title=body['title']) session = self.dbms.get_session() try: session.add(task) session.commit() result['data'] = self.map_to_json(task) except Exception as ex: session.rollback() result['status'] = 500 result['data'] = {'errors': {'db': 'session.commit error'}} print(ex) finally: session.close() else: result['data'] = {'errors': errors} return result def get_tasks(self, event): result = { 'data': {}, 'status': 200 # OK. } session = self.dbms.get_session() try: tasks = session.query(Task).all() objects = [] if tasks: for task in tasks: obj = self.map_to_json(task) objects.append(obj) result['data'] = objects except Exception as ex: result['status'] = 500 result['data'] = {'errors': {'db': 'session.query.all error'}} print(ex) finally: session.close() return result def update_task(self, event): body = dict() errors = dict() result = { 'data': {}, 'status': 205 # Reset Content. } fields = {'status': ['is_set', 'is_empty']} id = event['pathParameters']['id'] if id.isdigit(): id = int(id) else: errors['common'] = 'incorrect path' result['status'] = 404 # Not Found. if not errors: errors = self.validator.validate(event, result, fields) if not errors: body = event['body'] update = [ Task.Statuses.TODO.value, Task.Statuses.IN_PROGRESS.value, Task.Statuses.DONE.value ] if body['status'] not in update: errors['status'] = 'status is unknown' result['status'] = 400 # Bad Request. if not errors: session = self.dbms.get_session() try: task = session.query(Task).get(id) if body['status'] == task.status: errors['status'] = 'status is not changed' result['status'] = 400 # Bad Request. elif body['status'] - 1 != task.status: errors['status'] = 'status is incorrect' result['status'] = 409 # Conflict. if not errors: task.status = body['status'] if task.status == task.Statuses.IN_PROGRESS.value: task.start_time = datetime.now(tz=pytz.UTC) elif task.status == task.Statuses.DONE.value: task.end_time = datetime.now(tz=pytz.UTC) delta = task.end_time - task.start_time hours = delta.total_seconds() / 3600 task.payment = hours * KanbanBoardService.COST_PER_HOUR task.payment = round(task.payment, 2) session.add(task) session.commit() result['data'] = self.map_to_json(task) except Exception as ex: session.rollback() result['status'] = 500 result['data'] = {'errors': {'db': 'session.commit error'}} print(ex) finally: session.close() if errors: result['data'] = {'errors': errors} return result def map_to_json(self, task): obj = { "id": task.id, "title": task.title, "start_time": None, "end_time": None, "status": task.status, "payment": task.payment } if task.start_time: obj['start_time'] = task.start_time.strftime( '%Y-%m-%dT%H:%M:%S.%f') if task.end_time: obj['end_time'] = task.end_time.strftime('%Y-%m-%dT%H:%M:%S.%f') return obj
def train(model, config): # clear_output_dir() optimizer, lr_scheduler = init_training(model, config) logger = Logger(config) # TODO: Check which images it thinks are similar from e.g. copydays. # transformer = AllTransformer() # transformer = JpgTransformer() # transformer = RotateTransformer() # transformer = FlipTransformer() # transformer = RotateCropTransformer() transformer = CropTransformer() validator = Validator(model, logger, config, transformer) margin = 5 triplet_loss_fn = TripletMarginLoss(margin, p=config.distance_norm, swap=True) neg_cos_loss_fn = ZeroCosineLoss(margin=0.1) pos_cos_loss_fn = PositiveCosineLoss(margin=0.1) similarity_loss_fn = torch.nn.BCELoss() # Data dataloader = setup_traindata(config, transformer) # Init progressbar n_batches = len(dataloader) n_epochs = math.ceil(config.optim_steps / n_batches) pbar = Progressbar(n_epochs, n_batches) optim_steps = 0 val_freq = config.validation_freq # Training loop for epoch in pbar(range(1, n_epochs + 1)): for batch_i, data in enumerate(dataloader, 1): pbar.update(epoch, batch_i) # Validation # if optim_steps % val_freq == 0: # validator.validate(optim_steps) print("START") # Decrease learning rate if optim_steps % config.lr_step_frequency == 0: lr_scheduler.step() optimizer.zero_grad() original, transformed = data inputs = torch.cat((original, transformed)) outputs = model(inputs) original_emb, transf_emb = outputs anchors, positives, negatives = create_triplets(original_emb, transf_emb) print(anchors.shape) # Triplet loss triplet_loss = triplet_loss_fn(anchors, positives, negatives) # anchors, positives = scale_embeddings(anchors, positives, model) # anchors, negatives = scale_embeddings(anchors, negatives, model) # Cosine similarity loss # cos_match_loss = pos_cos_loss_fn(anchors, positives) # cos_not_match_loss = neg_cos_loss_fn(anchors, negatives) # Direct net loss # a_p, a_n = model.cc_similarity_net(anchors, positives, negatives) # net_match_loss = similarity_loss_fn(a_p, torch.ones_like(a_p)) # net_not_match_loss = similarity_loss_fn(a_n, torch.zeros_like(a_n)) # net_loss = net_match_loss + net_not_match_loss # loss_dict = dict(triplet=triplet_loss, cos_pos=cos_match_loss, cos_neg=cos_not_match_loss) # loss_dict = dict(cos_pos=cos_match_loss, cos_neg=cos_not_match_loss) loss_dict = dict(triplet_loss=triplet_loss) # loss_dict = dict(direct_match=net_match_loss, # direct_not_match=net_not_match_loss) loss = sum(loss_dict.values()) loss.backward() optimizer.step() optim_steps += 1 corrects = model.corrects(transf_emb, original_emb) logger.easy_or_hard(anchors, positives, negatives, margin, optim_steps) logger.log_loss(loss, optim_steps) logger.log_loss_percent(loss_dict, optim_steps) logger.log_corrects(corrects, optim_steps) logger.log_cosine(anchors, positives, negatives) # logger.log_p(model.feature_extractor.pool.p, optim_steps) # logger.log_weights(model.feature_extractor.sim_weights) # Frees up GPU memory del data del outputs
class UploadBudget(): def __init__(self, filepath='files/budget.xlsx'): logging.basicConfig( level=logging.INFO, filename="./logs/uploader_{:%Y_%m_%d_%H_%M}.log".format(datetime.datetime.now()), datefmt="%Y-%m-%d %H:%M:%S", format='%(asctime)s %(levelname)-8s %(message)s' ) self.logger = logging.getLogger("upload") self.mysql = mysql.connector.connect( host=mysql_conn["host"], user=mysql_conn["user"], password=mysql_conn["pass"], database=mysql_conn["db"] ) self.cursor = self.mysql.cursor(dictionary=True,buffered=True) self.filepath = filepath data = pd.read_excel (self.filepath, engine='openpyxl') self.dataframe_columns = [ 'id', 'year', 'month', 'type', 'classification', 'code', 'descriptor', 'lab_code', 'lab', 'business_name', 'business_unit', 'category_set', 'macrocategory', 'category_1', 'category_2', 'client_type', 'channel', 'sale', 'contribution', 'units', 'price' ] self.df = pd.DataFrame(data) self.validator = Validator(self.logger) def execute(self): try: with open('lockfile.txt', 'r') as f: lock = f.readline().strip().split() if lock[0] == 'locked': self.logger.info(f'Process already running!') sys.exit(0) except FileNotFoundError: with open('lockfile.txt', 'w') as f: f.write('locked') if set(self.dataframe_columns).issubset(self.df.columns) is False: self.logger.error('Invalid file') os.remove("lockfile.txt") os.remove(self.filepath) sys.exit(0) self.logger.info(f'Process started') for index, row in self.df.iterrows(): id = row["id"] year = self.validator.validate_data_number(row["year"], id, 'year') month = self.validator.validate_data_number(row["month"], id, 'month') type = self.validator.validate_data_number(row["type"], id, 'type') classification = self.validator.validate_data_string(row['classification'], id, 'classification') code = self.validator.validate_data_number(row["code"], id, 'code') descriptor = self.validator.validate_data_string(row["descriptor"], id, 'descriptor') lab_code = self.validator.validate_data_number(row['lab_code'], id, 'lab_code') lab = self.validator.validate_data_string(row['lab'], id, 'lab') business_name = self.validator.validate_data_string(row['business_name'], id, 'business_name') business_unit = self.validator.validate_data_string(row['business_unit'], id, 'business_unit') category_set = self.validator.validate_data_string(row['category_set'], id, 'category_set') macrocategory = self.validator.validate_data_string(row['macrocategory'], id, 'macrocategory') category_1 = self.validator.validate_data_string(row['category_1'], id, 'category_1') category_2 = self.validator.validate_data_string(row['category_2'], id, 'category_2') client_type = self.validator.validate_data_string(row['client_type'], id, 'client_type') channel = self.validator.validate_data_string(row['channel'], id, 'channel') sale = self.validator.validate_data_number(row['sale'], id, 'sale') contribution = self.validator.validate_data_number(row['contribution'], id, 'contribution') units = self.validator.validate_data_number(row['units'], id, 'units') price = self.validator.validate_data_number(row['price'], id, 'price') classification_id = search_by_name(self.cursor, 'classification', classification) if classification_id == 0: self.cursor.execute('INSERT INTO classification (name) VALUES (%s)' % (classification)) self.mysql.commit() classification_id = self.cursor.lastrowid lab_id = search_by_name(self.cursor, 'laboratory', lab) if lab_id == 0: self.cursor.execute('INSERT INTO laboratory (name, code) VALUES (%s, %s)' % (lab, lab_code)) self.mysql.commit() lab_id = self.cursor.lastrowid business_id = search_by_name(self.cursor, 'business', business_name) if business_id == 0: self.cursor.execute('INSERT INTO business (name) VALUES (%s)' % (business_name)) self.mysql.commit() business_id = self.cursor.lastrowid business_unit_id = search_by_name(self.cursor, 'business_unit', business_unit) if business_unit_id == 0: self.cursor.execute('INSERT INTO business_unit (name) VALUES (%s)' % (business_unit)) self.mysql.commit() business_unit_id = self.cursor.lastrowid category_set_id = search_by_name(self.cursor, 'category_set', category_set) if category_set_id == 0: self.cursor.execute('INSERT INTO category_set (name, businessUnitId) VALUES (%s, %s)' % (category_set, business_unit_id)) self.mysql.commit() category_set_id = self.cursor.lastrowid macrocategory_id = search_by_name(self.cursor, 'macrocategory', macrocategory) if macrocategory_id == 0: self.cursor.execute('INSERT INTO macrocategory (name, categorySetId) VALUES (%s, %s)' % (macrocategory, category_set_id)) self.mysql.commit() macrocategory_id = self.cursor.lastrowid category_1_id = search_by_name(self.cursor, 'category_1', category_1) if category_1_id == 0: self.cursor.execute('INSERT INTO category_1 (name, macrocategoryId) VALUES (%s, %s)' % (category_1, macrocategory_id)) self.mysql.commit() category_1_id = self.cursor.lastrowid category_2_id = search_by_name(self.cursor, 'category_2', category_2) if category_2_id == 0: self.cursor.execute('INSERT INTO category_2 (name, category1Id) VALUES (%s, %s)' % (category_2, category_1_id)) self.mysql.commit() category_2_id = self.cursor.lastrowid client_type_id = search_by_name(self.cursor, 'client_type', client_type) if client_type_id == 0: self.cursor.execute('INSERT INTO client_type (name) VALUES (%s)' % (client_type)) self.mysql.commit() client_type_id = self.cursor.lastrowid channel_id = search_by_name(self.cursor, 'channel', channel) if channel_id == 0: self.cursor.execute('INSERT INTO channel (name) VALUES (%s)' % (channel)) self.mysql.commit() channel_id = self.cursor.lastrowid try: self.cursor.execute(""" INSERT INTO budget ( id, year, month, type, code, descriptor, sale, contribution, units, price, classificationId, laboratoryId, businessId, businessUnitId, categorySetId, macrocategoryId, category1Id, category2Id, clientTypeId, channelId ) VALUES ( %s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s ) """ % ( id, year, month, type, code, descriptor, sale, contribution, units, price, classification_id, lab_id, business_id, business_unit_id, category_set_id, macrocategory_id, category_1_id, category_2_id, client_type_id, channel_id )) except Exception as e: self.logger.error(f'Error inserting row, id: {row["id"]}') self.logger.exception(e) continue else: self.mysql.commit() self.logger.info(f'Process finished') os.remove("lockfile.txt") os.remove(self.filepath)