def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
validator = Validator()
AssetsDirValidator(serializer.validated_data, 'assets_dir', validator)
validator.is_valid()
data = serializer.validated_data
old_data = self.find_old_project(data.get('assets_dir'))
if old_data is not None:
del old_data['title']
del old_data['description']
old_data.update(data)
with models.suppress_auto_now(models.Project,
['created_at', 'updated_at']):
project = models.Project.objects.create(**old_data)
serializer = self.get_serializer_class()(project)
tasks.import_project.delay(old_data['alias'])
else:
self.perform_create(serializer)
headers = self.get_success_headers(serializer.data)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=headers)
def test_validate_body_is_empty(self):
"""Test: validate(self, event, result, fields)
Error: body is empty."""
event = {'body': "{}"}
validator = Validator()
errors = validator.validate(event, {}, {})
self.assertEqual('body is empty', errors['body'])
def test_validate_body_is_none(self):
"""Test: validate(self, event, result, fields)
Error: body is not set."""
event = {'body': None}
validator = Validator()
errors = validator.validate(event, {}, {})
self.assertEqual('body is not set', errors['body'])
def test_validate_fields_is_none(self):
"""Test: validate(self, event, result, fields)
Exception: Parameter "fields" is None."""
validator = Validator()
with self.assertRaises(Exception) as ctx_mng:
validator.validate({}, {}, None)
ex = ctx_mng.exception
self.assertEqual('Parameter "fields" is None', str(ex))
def __init__(self, sess, components, params):
self.sess = sess
self.params = params
self.trainer = Trainer(self.sess, components, self.params)
self.__init_session(components)
self.validator = Validator(self.sess, components, params)
self.validation_loss = MAX_LOSS
self.validation_rate_step = params['validation_rate_step']
def test_validate_fields_success(self):
"""Test: validate(self, event, result, fields)"""
event = {'body': '{"title":"test-title"}'}
fields = {
'title': ['is_set', 'is_empty']
}
validator = Validator()
errors = validator.validate(event, {}, fields)
self.assertEqual({}, errors)
def test_validate_fields_is_not_set(self):
"""Test: validate(self, event, result, fields)
Error: field + ' is not set'"""
event = {'body': '{"foo":"bar"}'}
fields = {
'title': ['is_set']
}
validator = Validator()
errors = validator.validate(event, {}, fields)
self.assertEqual('title is not set', errors['title'])
def test_validate_fields_is_empty(self):
"""Test: validate(self, event, result, fields)
Error: field + ' is empty'"""
event = {'body': '{"title":""}'}
fields = {
'title': ['is_empty']
}
validator = Validator()
errors = validator.validate(event, {}, fields)
self.assertEqual('title is empty', errors['title'])
class TestRunner:
validator = Validator(
os.path.join(FileUtils.get_schemadir(), "merged-syntax.xsd"))
@staticmethod
def testOK(mapping_sample, case_prefix):
files = os.listdir(mapping_sample)
ok_prefix = case_prefix + "_ok"
for sample_file in files:
if sample_file.startswith(ok_prefix) is True:
file_path = os.path.join(mapping_sample, sample_file)
if TestRunner.validator.validate_file(file_path,
verbose=False) is False:
TestRunner.validator.validate_file(file_path, verbose=True)
logger.error(sample_file + " is not valid, it should be")
return False
logger.info(sample_file + " is valid: fine")
return True
@staticmethod
def testKO(mapping_sample, case_prefix):
ko_prefix = case_prefix + "_ko"
files = os.listdir(mapping_sample)
for sample_file in files:
if sample_file.startswith(ko_prefix) is True:
file_path = os.path.join(mapping_sample, sample_file)
if TestRunner.validator.validate_file(file_path,
verbose=False) is True:
logger.error(sample_file)
return False
logger.info(sample_file + " is not valid: fine")
return True
def __init__(self, filepath='files/budget.xlsx'):
logging.basicConfig(
level=logging.INFO,
filename="./logs/uploader_{:%Y_%m_%d_%H_%M}.log".format(datetime.datetime.now()),
datefmt="%Y-%m-%d %H:%M:%S",
format='%(asctime)s %(levelname)-8s %(message)s'
)
self.logger = logging.getLogger("upload")
self.mysql = mysql.connector.connect(
host=mysql_conn["host"],
user=mysql_conn["user"],
password=mysql_conn["pass"],
database=mysql_conn["db"]
)
self.cursor = self.mysql.cursor(dictionary=True,buffered=True)
self.filepath = filepath
data = pd.read_excel (self.filepath, engine='openpyxl')
self.dataframe_columns = [
'id',
'year',
'month',
'type',
'classification',
'code',
'descriptor',
'lab_code',
'lab',
'business_name',
'business_unit',
'category_set',
'macrocategory',
'category_1',
'category_2',
'client_type',
'channel',
'sale',
'contribution',
'units',
'price'
]
self.df = pd.DataFrame(data)
self.validator = Validator(self.logger)
def proof_of_work(self):
last_block = self.__chain[-1]
last_hash = hash_utils.hash_block(last_block)
proof = 0
while not Validator.validate_proof(self.__open_transactions, last_hash, proof):
proof += 1
return proof
def __verify_range(self, row_id):
reading = self.__dao.get_sensor_reading(row_id)
if not reading:
print('No reading saved.')
return
sensor_data_range = FileAccess.get_sensor_data_range()
if sensor_data_range is None:
print('No range config available. Cannot verify...')
return
result_temp = Validator.verify_temp(reading[SensorDataCol.TEMP],
sensor_data_range)
result_humid = Validator.verify_humid(reading[SensorDataCol.TEMP],
sensor_data_range)
temp_within_range = result_temp[0]
humid_within_range = result_humid[0]
if not (temp_within_range and humid_within_range):
self.__notifier.notify_out_of_range_reading(
reading, result_temp, result_humid)
class SessionTrainer:
def __init__(self, sess, components, params):
self.sess = sess
self.params = params
self.trainer = Trainer(self.sess, components, self.params)
self.__init_session(components)
self.validator = Validator(self.sess, components, params)
self.validation_loss = MAX_LOSS
self.validation_rate_step = params['validation_rate_step']
def train(self):
index = 0
while self.trainer.should_continue():
self.__train(index)
index += 1
self.__save_state()
def __init_session(self, components):
self.saver = components['saver']
self.sess.run(tf.global_variables_initializer())
if self.params['state_fname']:
session_path = os.path.join(self.params['state_folder'],
self.params['state_fname'])
self.saver.restore(self.sess, session_path)
def __train(self, index):
check = index % self.validation_rate_step == 0
self.trainer.train_batch(index, check)
if check:
self.__validate_loss(index)
def __validate_loss(self, index):
va = self.validator.validate(index)
if va > self.validation_loss:
self.trainer.reduce_learning_rate()
self.validation_loss = va
def __save_state(self):
time_str = strftime("%d_%b_%Y_%H_%M", gmtime())
out_state_fname = self.params['saved_state_fname'] + '_{}.ckpt'.format(
time_str)
session_path = os.path.join(self.params['state_folder'],
out_state_fname)
self.saver.save(
self.sess, os.path.join(self.params['state_folder'], session_path))
def __process_day(self, day, data_range):
min_temp = self.__dao.get_min_temp_of_day(day)
max_temp = self.__dao.get_max_temp_of_day(day)
min_humid = self.__dao.get_min_humid_of_day(day)
max_humid = self.__dao.get_max_humid_of_day(day)
day_result = Validator.verify_temp_humid_of_day(
min_temp, max_temp, min_humid, max_humid, data_range)
if not day_result[0]:
day_status = 'BAD: ' + day_result[1]
else:
day_status = day_result[1]
day_row = [day, day_status]
self.__report_data.append(day_row)
def input_listener(self):
session_active = True
while session_active:
self.print_menu()
user_choice = input('Please enter a choice: ')
if user_choice == '1':
tx_data = self.get_tx_info()
receiver, amount = tx_data
if self.blockchain.create_transaction(self.id, receiver,
amount):
print('Transaction added')
else:
print('Transaction failed')
print(self.blockchain.open_transactions)
elif user_choice == '2':
self.blockchain.mine_block(self.id)
elif user_choice == '3':
self.print_blockchain()
elif user_choice == '4':
if Validator.validate_transaction(
self.blockchain.open_transactions,
self.blockchain.get_balance):
print('Transactions validated')
else:
print('Could not validate all transactions')
elif user_choice == 'q' or user_choice == 'Q':
session_active = False
break
else:
print('invalid input')
if not Validator.is_valid_chain(self.blockchain.chain):
self.print_blockchain()
print('Corrupted Blockchain, exiting')
break
def create_transaction(self, sender, receiver, amount):
'''
Creates a transaction between users, this transaction is then
validated. If the user has enough funds the transaction is added
onto a list of open transactions.
Args:
sender: The user who will send funds
receiver: The user who will receive the funds
amount: The amount of funds to be sent
Returns:
True if the transaction was successful.
False if the transaction fails.
'''
tx = Transaction(sender, receiver, amount)
if Validator.validate_transaction(tx, self.get_balance):
self.__open_transactions.append(tx)
return True
return False
def train_with_correspondences(save_folder, startnet, args):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
check_mkdir(save_folder)
writer = SummaryWriter(save_folder)
# Network and weight loading
model_config = model_configs.PspnetCityscapesConfig()
net = model_config.init_network().to(device)
if args['snapshot'] == 'latest':
args['snapshot'] = get_latest_network_name(save_folder)
if len(args['snapshot']) == 0: # If start from beginning
state_dict = torch.load(startnet)
# needed since we slightly changed the structure of the network in
# pspnet
state_dict = rename_keys_to_match(state_dict)
net.load_state_dict(state_dict) # load original weights
start_iter = 0
args['best_record'] = {
'iter': 0,
'val_loss': 1e10,
'acc': 0,
'acc_cls': 0,
'mean_iu': 0,
'fwavacc': 0
}
else: # If continue training
print('training resumes from ' + args['snapshot'])
net.load_state_dict(
torch.load(os.path.join(save_folder,
args['snapshot']))) # load weights
split_snapshot = args['snapshot'].split('_')
start_iter = int(split_snapshot[1])
with open(os.path.join(save_folder, 'bestval.txt')) as f:
best_val_dict_str = f.read()
args['best_record'] = eval(best_val_dict_str.rstrip())
net.train()
freeze_bn(net)
# Data loading setup
if args['corr_set'] == 'rc':
corr_set_config = data_configs.RobotcarConfig()
elif args['corr_set'] == 'cmu':
corr_set_config = data_configs.CmuConfig()
sliding_crop_im = joint_transforms.SlidingCropImageOnly(
713, args['stride_rate'])
input_transform = model_config.input_transform
pre_validation_transform = model_config.pre_validation_transform
target_transform = extended_transforms.MaskToTensor()
train_joint_transform_seg = joint_transforms.Compose([
joint_transforms.Resize(1024),
joint_transforms.RandomRotate(10),
joint_transforms.RandomHorizontallyFlip(),
joint_transforms.RandomCrop(713)
])
train_joint_transform_corr = corr_transforms.Compose([
corr_transforms.CorrResize(1024),
corr_transforms.CorrRandomCrop(713)
])
# keep list of segmentation loaders and validators
seg_loaders = list()
validators = list()
# Correspondences
corr_set = correspondences.Correspondences(
corr_set_config.correspondence_path,
corr_set_config.correspondence_im_path,
input_size=(713, 713),
mean_std=model_config.mean_std,
input_transform=input_transform,
joint_transform=train_joint_transform_corr)
corr_loader = DataLoader(corr_set,
batch_size=args['train_batch_size'],
num_workers=args['n_workers'],
shuffle=True)
# Cityscapes Training
c_config = data_configs.CityscapesConfig()
seg_set_cs = cityscapes.CityScapes(
c_config.train_im_folder,
c_config.train_seg_folder,
c_config.im_file_ending,
c_config.seg_file_ending,
id_to_trainid=c_config.id_to_trainid,
joint_transform=train_joint_transform_seg,
sliding_crop=None,
transform=input_transform,
target_transform=target_transform)
seg_loader_cs = DataLoader(seg_set_cs,
batch_size=args['train_batch_size'],
num_workers=args['n_workers'],
shuffle=True)
seg_loaders.append(seg_loader_cs)
# Cityscapes Validation
val_set_cs = cityscapes.CityScapes(
c_config.val_im_folder,
c_config.val_seg_folder,
c_config.im_file_ending,
c_config.seg_file_ending,
id_to_trainid=c_config.id_to_trainid,
sliding_crop=sliding_crop_im,
transform=input_transform,
target_transform=target_transform,
transform_before_sliding=pre_validation_transform)
val_loader_cs = DataLoader(val_set_cs,
batch_size=1,
num_workers=args['n_workers'],
shuffle=False)
validator_cs = Validator(val_loader_cs,
n_classes=c_config.n_classes,
save_snapshot=False,
extra_name_str='Cityscapes')
validators.append(validator_cs)
# Vistas Training and Validation
if args['include_vistas']:
v_config = data_configs.VistasConfig(
use_subsampled_validation_set=True, use_cityscapes_classes=True)
seg_set_vis = cityscapes.CityScapes(
v_config.train_im_folder,
v_config.train_seg_folder,
v_config.im_file_ending,
v_config.seg_file_ending,
id_to_trainid=v_config.id_to_trainid,
joint_transform=train_joint_transform_seg,
sliding_crop=None,
transform=input_transform,
target_transform=target_transform)
seg_loader_vis = DataLoader(seg_set_vis,
batch_size=args['train_batch_size'],
num_workers=args['n_workers'],
shuffle=True)
seg_loaders.append(seg_loader_vis)
val_set_vis = cityscapes.CityScapes(
v_config.val_im_folder,
v_config.val_seg_folder,
v_config.im_file_ending,
v_config.seg_file_ending,
id_to_trainid=v_config.id_to_trainid,
sliding_crop=sliding_crop_im,
transform=input_transform,
target_transform=target_transform,
transform_before_sliding=pre_validation_transform)
val_loader_vis = DataLoader(val_set_vis,
batch_size=1,
num_workers=args['n_workers'],
shuffle=False)
validator_vis = Validator(val_loader_vis,
n_classes=v_config.n_classes,
save_snapshot=False,
extra_name_str='Vistas')
validators.append(validator_vis)
else:
seg_loader_vis = None
map_validator = None
# Extra Training
extra_seg_set = cityscapes.CityScapes(
corr_set_config.train_im_folder,
corr_set_config.train_seg_folder,
corr_set_config.im_file_ending,
corr_set_config.seg_file_ending,
id_to_trainid=corr_set_config.id_to_trainid,
joint_transform=train_joint_transform_seg,
sliding_crop=None,
transform=input_transform,
target_transform=target_transform)
extra_seg_loader = DataLoader(extra_seg_set,
batch_size=args['train_batch_size'],
num_workers=args['n_workers'],
shuffle=True)
seg_loaders.append(extra_seg_loader)
# Extra Validation
extra_val_set = cityscapes.CityScapes(
corr_set_config.val_im_folder,
corr_set_config.val_seg_folder,
corr_set_config.im_file_ending,
corr_set_config.seg_file_ending,
id_to_trainid=corr_set_config.id_to_trainid,
sliding_crop=sliding_crop_im,
transform=input_transform,
target_transform=target_transform,
transform_before_sliding=pre_validation_transform)
extra_val_loader = DataLoader(extra_val_set,
batch_size=1,
num_workers=args['n_workers'],
shuffle=False)
extra_validator = Validator(extra_val_loader,
n_classes=corr_set_config.n_classes,
save_snapshot=True,
extra_name_str='Extra')
validators.append(extra_validator)
# Loss setup
if args['corr_loss_type'] == 'class':
corr_loss_fct = CorrClassLoss(input_size=[713, 713])
else:
corr_loss_fct = FeatureLoss(
input_size=[713, 713],
loss_type=args['corr_loss_type'],
feat_dist_threshold_match=args['feat_dist_threshold_match'],
feat_dist_threshold_nomatch=args['feat_dist_threshold_nomatch'],
n_not_matching=0)
seg_loss_fct = torch.nn.CrossEntropyLoss(
reduction='elementwise_mean',
ignore_index=cityscapes.ignore_label).to(device)
# Optimizer setup
optimizer = optim.SGD([{
'params': [
param for name, param in net.named_parameters()
if name[-4:] == 'bias' and param.requires_grad
],
'lr':
2 * args['lr']
}, {
'params': [
param for name, param in net.named_parameters()
if name[-4:] != 'bias' and param.requires_grad
],
'lr':
args['lr'],
'weight_decay':
args['weight_decay']
}],
momentum=args['momentum'],
nesterov=True)
if len(args['snapshot']) > 0:
optimizer.load_state_dict(
torch.load(os.path.join(save_folder, 'opt_' + args['snapshot'])))
optimizer.param_groups[0]['lr'] = 2 * args['lr']
optimizer.param_groups[1]['lr'] = args['lr']
open(os.path.join(save_folder,
str(datetime.datetime.now()) + '.txt'),
'w').write(str(args) + '\n\n')
if len(args['snapshot']) == 0:
f_handle = open(os.path.join(save_folder, 'log.log'), 'w', buffering=1)
else:
clean_log_before_continuing(os.path.join(save_folder, 'log.log'),
start_iter)
f_handle = open(os.path.join(save_folder, 'log.log'), 'a', buffering=1)
##########################################################################
#
# MAIN TRAINING CONSISTS OF ALL SEGMENTATION LOSSES AND A CORRESPONDENCE LOSS
#
##########################################################################
softm = torch.nn.Softmax2d()
val_iter = 0
train_corr_loss = AverageMeter()
train_seg_cs_loss = AverageMeter()
train_seg_extra_loss = AverageMeter()
train_seg_vis_loss = AverageMeter()
seg_loss_meters = list()
seg_loss_meters.append(train_seg_cs_loss)
if args['include_vistas']:
seg_loss_meters.append(train_seg_vis_loss)
seg_loss_meters.append(train_seg_extra_loss)
curr_iter = start_iter
for i in range(args['max_iter']):
optimizer.param_groups[0]['lr'] = 2 * args['lr'] * (
1 - float(curr_iter) / args['max_iter'])**args['lr_decay']
optimizer.param_groups[1]['lr'] = args['lr'] * (
1 - float(curr_iter) / args['max_iter'])**args['lr_decay']
#######################################################################
# SEGMENTATION UPDATE STEP
#######################################################################
#
for si, seg_loader in enumerate(seg_loaders):
# get segmentation training sample
inputs, gts = next(iter(seg_loader))
slice_batch_pixel_size = inputs.size(0) * inputs.size(
2) * inputs.size(3)
inputs = inputs.to(device)
gts = gts.to(device)
optimizer.zero_grad()
outputs, aux = net(inputs)
main_loss = args['seg_loss_weight'] * seg_loss_fct(outputs, gts)
aux_loss = args['seg_loss_weight'] * seg_loss_fct(aux, gts)
loss = main_loss + 0.4 * aux_loss
loss.backward()
optimizer.step()
seg_loss_meters[si].update(main_loss.item(),
slice_batch_pixel_size)
#######################################################################
# CORRESPONDENCE UPDATE STEP
#######################################################################
if args['corr_loss_weight'] > 0 and args[
'n_iterations_before_corr_loss'] < curr_iter:
img_ref, img_other, pts_ref, pts_other, weights = next(
iter(corr_loader))
# Transfer data to device
# img_ref is from the "good" sequence with generally better
# segmentation results
img_ref = img_ref.to(device)
img_other = img_other.to(device)
pts_ref = [p.to(device) for p in pts_ref]
pts_other = [p.to(device) for p in pts_other]
weights = [w.to(device) for w in weights]
# Forward pass
if args['corr_loss_type'] == 'hingeF': # Works on features
net.output_all = True
with torch.no_grad():
output_feat_ref, aux_feat_ref, output_ref, aux_ref = net(
img_ref)
output_feat_other, aux_feat_other, output_other, aux_other = net(
img_other
) # output1 must be last to backpropagate derivative correctly
net.output_all = False
else: # Works on class probs
with torch.no_grad():
output_ref, aux_ref = net(img_ref)
if args['corr_loss_type'] != 'hingeF' and args[
'corr_loss_type'] != 'hingeC':
output_ref = softm(output_ref)
aux_ref = softm(aux_ref)
# output1 must be last to backpropagate derivative correctly
output_other, aux_other = net(img_other)
if args['corr_loss_type'] != 'hingeF' and args[
'corr_loss_type'] != 'hingeC':
output_other = softm(output_other)
aux_other = softm(aux_other)
# Correspondence filtering
pts_ref_orig, pts_other_orig, weights_orig, batch_inds_to_keep_orig = correspondences.refine_correspondence_sample(
output_ref,
output_other,
pts_ref,
pts_other,
weights,
remove_same_class=args['remove_same_class'],
remove_classes=args['classes_to_ignore'])
pts_ref_orig = [
p for b, p in zip(batch_inds_to_keep_orig, pts_ref_orig)
if b.item() > 0
]
pts_other_orig = [
p for b, p in zip(batch_inds_to_keep_orig, pts_other_orig)
if b.item() > 0
]
weights_orig = [
p for b, p in zip(batch_inds_to_keep_orig, weights_orig)
if b.item() > 0
]
if args['corr_loss_type'] == 'hingeF':
# remove entire samples if needed
output_vals_ref = output_feat_ref[batch_inds_to_keep_orig]
output_vals_other = output_feat_other[batch_inds_to_keep_orig]
else:
# remove entire samples if needed
output_vals_ref = output_ref[batch_inds_to_keep_orig]
output_vals_other = output_other[batch_inds_to_keep_orig]
pts_ref_aux, pts_other_aux, weights_aux, batch_inds_to_keep_aux = correspondences.refine_correspondence_sample(
aux_ref,
aux_other,
pts_ref,
pts_other,
weights,
remove_same_class=args['remove_same_class'],
remove_classes=args['classes_to_ignore'])
pts_ref_aux = [
p for b, p in zip(batch_inds_to_keep_aux, pts_ref_aux)
if b.item() > 0
]
pts_other_aux = [
p for b, p in zip(batch_inds_to_keep_aux, pts_other_aux)
if b.item() > 0
]
weights_aux = [
p for b, p in zip(batch_inds_to_keep_aux, weights_aux)
if b.item() > 0
]
if args['corr_loss_type'] == 'hingeF':
# remove entire samples if needed
aux_vals_ref = aux_feat_ref[batch_inds_to_keep_orig]
aux_vals_other = aux_feat_other[batch_inds_to_keep_orig]
else:
# remove entire samples if needed
aux_vals_ref = aux_ref[batch_inds_to_keep_aux]
aux_vals_other = aux_other[batch_inds_to_keep_aux]
optimizer.zero_grad()
# correspondence loss
if output_vals_ref.size(0) > 0:
loss_corr_hr = corr_loss_fct(output_vals_ref,
output_vals_other, pts_ref_orig,
pts_other_orig, weights_orig)
else:
loss_corr_hr = 0 * output_vals_other.sum()
if aux_vals_ref.size(0) > 0:
loss_corr_aux = corr_loss_fct(
aux_vals_ref, aux_vals_other, pts_ref_aux, pts_other_aux,
weights_aux) # use output from img1 as "reference"
else:
loss_corr_aux = 0 * aux_vals_other.sum()
loss_corr = args['corr_loss_weight'] * \
(loss_corr_hr + 0.4 * loss_corr_aux)
loss_corr.backward()
optimizer.step()
train_corr_loss.update(loss_corr.item())
#######################################################################
# LOGGING ETC
#######################################################################
curr_iter += 1
val_iter += 1
writer.add_scalar('train_seg_loss_cs', train_seg_cs_loss.avg,
curr_iter)
writer.add_scalar('train_seg_loss_extra', train_seg_extra_loss.avg,
curr_iter)
writer.add_scalar('train_seg_loss_vis', train_seg_vis_loss.avg,
curr_iter)
writer.add_scalar('train_corr_loss', train_corr_loss.avg, curr_iter)
writer.add_scalar('lr', optimizer.param_groups[1]['lr'], curr_iter)
if (i + 1) % args['print_freq'] == 0:
str2write = '[iter %d / %d], [train corr loss %.5f] , [seg cs loss %.5f], [seg vis loss %.5f], [seg extra loss %.5f]. [lr %.10f]' % (
curr_iter, len(corr_loader), train_corr_loss.avg,
train_seg_cs_loss.avg, train_seg_vis_loss.avg,
train_seg_extra_loss.avg, optimizer.param_groups[1]['lr'])
print(str2write)
f_handle.write(str2write + "\n")
if val_iter >= args['val_interval']:
val_iter = 0
for validator in validators:
validator.run(net,
optimizer,
args,
curr_iter,
save_folder,
f_handle,
writer=writer)
# Post training
f_handle.close()
writer.close()
def __init__(self):
self.dbms = DBDriver()
self.validator = Validator()
def train(model, config):
# clear_output_dir()
optimizer, lr_scheduler = init_training(model, config)
logger = Logger(config)
validator = Validator(model, logger, config)
# cos_loss_fn = torch.nn.CosineEmbeddingLoss(margin=0.33)
cos_loss_fn = ZeroCosineLoss(margin=0)
pos_loss_fn = PositiveCosineLoss(margin=0.33)
class_loss_fn = torch.nn.BCEWithLogitsLoss(reduction='none')
# Data
dataloader = setup_traindata(config)
# Init progressbar
n_batches = len(dataloader)
n_epochs = math.ceil(config.optim_steps / n_batches)
pbar = Progressbar(n_epochs, n_batches)
optim_steps = 0
val_freq = config.validation_freq
get_lr = lambda: optimizer.param_groups[0]['lr']
# Training loop
for epoch in pbar(range(1, n_epochs + 1)):
for batch_i, data in enumerate(dataloader, 1):
pbar.update(epoch, batch_i)
# Validation
if optim_steps % val_freq == 0:
validator.validate(optim_steps)
# validator.val_normal(optim_steps)
# Decrease learning rate
if optim_steps % config.lr_step_frequency == 0:
lr_scheduler.step()
optimizer.zero_grad()
cats, dogs = data
inputs = torch.cat((cats, dogs))
outputs, class_outputs = model(inputs)
cat_embs, dog_embs = outputs.chunk(2)
cat_class_embs, dog_class_embs = class_outputs.chunk(2)
catpair, dogpair, catdogpair = create_pairs(cat_embs, dog_embs)
# Cosine similarity loss
cat_loss = pos_loss_fn(catpair[0], catpair[1])
dog_loss = pos_loss_fn(dogpair[0], dogpair[1])
# y = torch.ones(catdogpair[0].size(0)).to(model.device)
cat_dog_loss = cos_loss_fn(catdogpair[0], catdogpair[1])
class_l1 = class_loss_fn(cat_class_embs, torch.zeros_like(cat_class_embs))
class_l2 = class_loss_fn(dog_class_embs, torch.ones_like(dog_class_embs))
class_loss = (class_l1 + class_l2).mean() / 3
# loss_dict = dict(cat_loss=cat_loss, dog_loss=dog_loss, cat_dog_loss=cat_dog_loss, class_loss=class_loss)
loss_dict = dict(cat_loss=cat_loss, dog_loss=dog_loss, cat_dog_loss=cat_dog_loss)
# loss_dict = dict(cat_loss=cat_loss, cat_dog_loss=cat_dog_loss)
# loss_dict = dict(cat_dog_loss=cat_dog_loss)
# loss_dict = dict(class_loss=class_loss)
loss = sum(loss_dict.values())
loss.backward()
optimizer.step()
optim_steps += 1
logger.log_cosine(catpair, dogpair, catdogpair, optim_steps)
logger.log_loss(loss, optim_steps)
logger.log_loss_percent(loss_dict, optim_steps)
logger.log_learning_rate(get_lr(), optim_steps)
# Frees up GPU memory
del data; del outputs
class KanbanBoardService:
COST_PER_HOUR = 1000
def __init__(self):
self.dbms = DBDriver()
self.validator = Validator()
def create_task(self, event):
result = {
'data': {},
'status': 201 # Created.
}
fields = {'title': ['is_set', 'is_empty']}
errors = self.validator.validate(event, result, fields)
if not errors:
body = event['body']
task = Task(title=body['title'])
session = self.dbms.get_session()
try:
session.add(task)
session.commit()
result['data'] = self.map_to_json(task)
except Exception as ex:
session.rollback()
result['status'] = 500
result['data'] = {'errors': {'db': 'session.commit error'}}
print(ex)
finally:
session.close()
else:
result['data'] = {'errors': errors}
return result
def get_tasks(self, event):
result = {
'data': {},
'status': 200 # OK.
}
session = self.dbms.get_session()
try:
tasks = session.query(Task).all()
objects = []
if tasks:
for task in tasks:
obj = self.map_to_json(task)
objects.append(obj)
result['data'] = objects
except Exception as ex:
result['status'] = 500
result['data'] = {'errors': {'db': 'session.query.all error'}}
print(ex)
finally:
session.close()
return result
def update_task(self, event):
body = dict()
errors = dict()
result = {
'data': {},
'status': 205 # Reset Content.
}
fields = {'status': ['is_set', 'is_empty']}
id = event['pathParameters']['id']
if id.isdigit():
id = int(id)
else:
errors['common'] = 'incorrect path'
result['status'] = 404 # Not Found.
if not errors:
errors = self.validator.validate(event, result, fields)
if not errors:
body = event['body']
update = [
Task.Statuses.TODO.value, Task.Statuses.IN_PROGRESS.value,
Task.Statuses.DONE.value
]
if body['status'] not in update:
errors['status'] = 'status is unknown'
result['status'] = 400 # Bad Request.
if not errors:
session = self.dbms.get_session()
try:
task = session.query(Task).get(id)
if body['status'] == task.status:
errors['status'] = 'status is not changed'
result['status'] = 400 # Bad Request.
elif body['status'] - 1 != task.status:
errors['status'] = 'status is incorrect'
result['status'] = 409 # Conflict.
if not errors:
task.status = body['status']
if task.status == task.Statuses.IN_PROGRESS.value:
task.start_time = datetime.now(tz=pytz.UTC)
elif task.status == task.Statuses.DONE.value:
task.end_time = datetime.now(tz=pytz.UTC)
delta = task.end_time - task.start_time
hours = delta.total_seconds() / 3600
task.payment = hours * KanbanBoardService.COST_PER_HOUR
task.payment = round(task.payment, 2)
session.add(task)
session.commit()
result['data'] = self.map_to_json(task)
except Exception as ex:
session.rollback()
result['status'] = 500
result['data'] = {'errors': {'db': 'session.commit error'}}
print(ex)
finally:
session.close()
if errors:
result['data'] = {'errors': errors}
return result
def map_to_json(self, task):
obj = {
"id": task.id,
"title": task.title,
"start_time": None,
"end_time": None,
"status": task.status,
"payment": task.payment
}
if task.start_time:
obj['start_time'] = task.start_time.strftime(
'%Y-%m-%dT%H:%M:%S.%f')
if task.end_time:
obj['end_time'] = task.end_time.strftime('%Y-%m-%dT%H:%M:%S.%f')
return obj
def train(model, config):
# clear_output_dir()
optimizer, lr_scheduler = init_training(model, config)
logger = Logger(config)
# TODO: Check which images it thinks are similar from e.g. copydays.
# transformer = AllTransformer()
# transformer = JpgTransformer()
# transformer = RotateTransformer()
# transformer = FlipTransformer()
# transformer = RotateCropTransformer()
transformer = CropTransformer()
validator = Validator(model, logger, config, transformer)
margin = 5
triplet_loss_fn = TripletMarginLoss(margin,
p=config.distance_norm,
swap=True)
neg_cos_loss_fn = ZeroCosineLoss(margin=0.1)
pos_cos_loss_fn = PositiveCosineLoss(margin=0.1)
similarity_loss_fn = torch.nn.BCELoss()
# Data
dataloader = setup_traindata(config, transformer)
# Init progressbar
n_batches = len(dataloader)
n_epochs = math.ceil(config.optim_steps / n_batches)
pbar = Progressbar(n_epochs, n_batches)
optim_steps = 0
val_freq = config.validation_freq
# Training loop
for epoch in pbar(range(1, n_epochs + 1)):
for batch_i, data in enumerate(dataloader, 1):
pbar.update(epoch, batch_i)
# Validation
# if optim_steps % val_freq == 0:
# validator.validate(optim_steps)
print("START")
# Decrease learning rate
if optim_steps % config.lr_step_frequency == 0:
lr_scheduler.step()
optimizer.zero_grad()
original, transformed = data
inputs = torch.cat((original, transformed))
outputs = model(inputs)
original_emb, transf_emb = outputs
anchors, positives, negatives = create_triplets(original_emb, transf_emb)
print(anchors.shape)
# Triplet loss
triplet_loss = triplet_loss_fn(anchors, positives, negatives)
# anchors, positives = scale_embeddings(anchors, positives, model)
# anchors, negatives = scale_embeddings(anchors, negatives, model)
# Cosine similarity loss
# cos_match_loss = pos_cos_loss_fn(anchors, positives)
# cos_not_match_loss = neg_cos_loss_fn(anchors, negatives)
# Direct net loss
# a_p, a_n = model.cc_similarity_net(anchors, positives, negatives)
# net_match_loss = similarity_loss_fn(a_p, torch.ones_like(a_p))
# net_not_match_loss = similarity_loss_fn(a_n, torch.zeros_like(a_n))
# net_loss = net_match_loss + net_not_match_loss
# loss_dict = dict(triplet=triplet_loss, cos_pos=cos_match_loss, cos_neg=cos_not_match_loss)
# loss_dict = dict(cos_pos=cos_match_loss, cos_neg=cos_not_match_loss)
loss_dict = dict(triplet_loss=triplet_loss)
# loss_dict = dict(direct_match=net_match_loss,
# direct_not_match=net_not_match_loss)
loss = sum(loss_dict.values())
loss.backward()
optimizer.step()
optim_steps += 1
corrects = model.corrects(transf_emb, original_emb)
logger.easy_or_hard(anchors, positives, negatives, margin, optim_steps)
logger.log_loss(loss, optim_steps)
logger.log_loss_percent(loss_dict, optim_steps)
logger.log_corrects(corrects, optim_steps)
logger.log_cosine(anchors, positives, negatives)
# logger.log_p(model.feature_extractor.pool.p, optim_steps)
# logger.log_weights(model.feature_extractor.sim_weights)
# Frees up GPU memory
del data
del outputs
class UploadBudget():
def __init__(self, filepath='files/budget.xlsx'):
logging.basicConfig(
level=logging.INFO,
filename="./logs/uploader_{:%Y_%m_%d_%H_%M}.log".format(datetime.datetime.now()),
datefmt="%Y-%m-%d %H:%M:%S",
format='%(asctime)s %(levelname)-8s %(message)s'
)
self.logger = logging.getLogger("upload")
self.mysql = mysql.connector.connect(
host=mysql_conn["host"],
user=mysql_conn["user"],
password=mysql_conn["pass"],
database=mysql_conn["db"]
)
self.cursor = self.mysql.cursor(dictionary=True,buffered=True)
self.filepath = filepath
data = pd.read_excel (self.filepath, engine='openpyxl')
self.dataframe_columns = [
'id',
'year',
'month',
'type',
'classification',
'code',
'descriptor',
'lab_code',
'lab',
'business_name',
'business_unit',
'category_set',
'macrocategory',
'category_1',
'category_2',
'client_type',
'channel',
'sale',
'contribution',
'units',
'price'
]
self.df = pd.DataFrame(data)
self.validator = Validator(self.logger)
def execute(self):
try:
with open('lockfile.txt', 'r') as f:
lock = f.readline().strip().split()
if lock[0] == 'locked':
self.logger.info(f'Process already running!')
sys.exit(0)
except FileNotFoundError:
with open('lockfile.txt', 'w') as f:
f.write('locked')
if set(self.dataframe_columns).issubset(self.df.columns) is False:
self.logger.error('Invalid file')
os.remove("lockfile.txt")
os.remove(self.filepath)
sys.exit(0)
self.logger.info(f'Process started')
for index, row in self.df.iterrows():
id = row["id"]
year = self.validator.validate_data_number(row["year"], id, 'year')
month = self.validator.validate_data_number(row["month"], id, 'month')
type = self.validator.validate_data_number(row["type"], id, 'type')
classification = self.validator.validate_data_string(row['classification'], id, 'classification')
code = self.validator.validate_data_number(row["code"], id, 'code')
descriptor = self.validator.validate_data_string(row["descriptor"], id, 'descriptor')
lab_code = self.validator.validate_data_number(row['lab_code'], id, 'lab_code')
lab = self.validator.validate_data_string(row['lab'], id, 'lab')
business_name = self.validator.validate_data_string(row['business_name'], id, 'business_name')
business_unit = self.validator.validate_data_string(row['business_unit'], id, 'business_unit')
category_set = self.validator.validate_data_string(row['category_set'], id, 'category_set')
macrocategory = self.validator.validate_data_string(row['macrocategory'], id, 'macrocategory')
category_1 = self.validator.validate_data_string(row['category_1'], id, 'category_1')
category_2 = self.validator.validate_data_string(row['category_2'], id, 'category_2')
client_type = self.validator.validate_data_string(row['client_type'], id, 'client_type')
channel = self.validator.validate_data_string(row['channel'], id, 'channel')
sale = self.validator.validate_data_number(row['sale'], id, 'sale')
contribution = self.validator.validate_data_number(row['contribution'], id, 'contribution')
units = self.validator.validate_data_number(row['units'], id, 'units')
price = self.validator.validate_data_number(row['price'], id, 'price')
classification_id = search_by_name(self.cursor, 'classification', classification)
if classification_id == 0:
self.cursor.execute('INSERT INTO classification (name) VALUES (%s)' % (classification))
self.mysql.commit()
classification_id = self.cursor.lastrowid
lab_id = search_by_name(self.cursor, 'laboratory', lab)
if lab_id == 0:
self.cursor.execute('INSERT INTO laboratory (name, code) VALUES (%s, %s)' % (lab, lab_code))
self.mysql.commit()
lab_id = self.cursor.lastrowid
business_id = search_by_name(self.cursor, 'business', business_name)
if business_id == 0:
self.cursor.execute('INSERT INTO business (name) VALUES (%s)' % (business_name))
self.mysql.commit()
business_id = self.cursor.lastrowid
business_unit_id = search_by_name(self.cursor, 'business_unit', business_unit)
if business_unit_id == 0:
self.cursor.execute('INSERT INTO business_unit (name) VALUES (%s)' % (business_unit))
self.mysql.commit()
business_unit_id = self.cursor.lastrowid
category_set_id = search_by_name(self.cursor, 'category_set', category_set)
if category_set_id == 0:
self.cursor.execute('INSERT INTO category_set (name, businessUnitId) VALUES (%s, %s)' % (category_set, business_unit_id))
self.mysql.commit()
category_set_id = self.cursor.lastrowid
macrocategory_id = search_by_name(self.cursor, 'macrocategory', macrocategory)
if macrocategory_id == 0:
self.cursor.execute('INSERT INTO macrocategory (name, categorySetId) VALUES (%s, %s)' % (macrocategory, category_set_id))
self.mysql.commit()
macrocategory_id = self.cursor.lastrowid
category_1_id = search_by_name(self.cursor, 'category_1', category_1)
if category_1_id == 0:
self.cursor.execute('INSERT INTO category_1 (name, macrocategoryId) VALUES (%s, %s)' % (category_1, macrocategory_id))
self.mysql.commit()
category_1_id = self.cursor.lastrowid
category_2_id = search_by_name(self.cursor, 'category_2', category_2)
if category_2_id == 0:
self.cursor.execute('INSERT INTO category_2 (name, category1Id) VALUES (%s, %s)' % (category_2, category_1_id))
self.mysql.commit()
category_2_id = self.cursor.lastrowid
client_type_id = search_by_name(self.cursor, 'client_type', client_type)
if client_type_id == 0:
self.cursor.execute('INSERT INTO client_type (name) VALUES (%s)' % (client_type))
self.mysql.commit()
client_type_id = self.cursor.lastrowid
channel_id = search_by_name(self.cursor, 'channel', channel)
if channel_id == 0:
self.cursor.execute('INSERT INTO channel (name) VALUES (%s)' % (channel))
self.mysql.commit()
channel_id = self.cursor.lastrowid
try:
self.cursor.execute("""
INSERT INTO budget (
id, year, month, type, code, descriptor, sale, contribution, units, price, classificationId, laboratoryId, businessId, businessUnitId, categorySetId, macrocategoryId, category1Id, category2Id, clientTypeId, channelId
) VALUES (
%s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s
)
"""
% (
id, year, month, type, code, descriptor, sale, contribution, units, price, classification_id, lab_id, business_id, business_unit_id, category_set_id, macrocategory_id, category_1_id, category_2_id, client_type_id, channel_id
))
except Exception as e:
self.logger.error(f'Error inserting row, id: {row["id"]}')
self.logger.exception(e)
continue
else:
self.mysql.commit()
self.logger.info(f'Process finished')
os.remove("lockfile.txt")
os.remove(self.filepath)