def transform_identifier(x):
if U.starts_with_any(x, ['Dot', 'Buddy', 'Mom', 'Cleo']):
parts = x.split(',')
if len(parts) > 1:
prefix = os.path.commonprefix(parts)
n = len(prefix)
trimmed = [U.camel_to_snake(part[n:]) for part in parts]
string = '_'.join(trimmed)
else:
prefix = ''
string = U.camel_to_snake(x.replace('_', ''))
result = f'{prefix}{string}'
return result.lower()
return x
def type(self):
"""Get type compatible with Cerberus by converting class name
from camel case to snake case
:return: (str) snake case version of class name
"""
return camel_to_snake(self.__class__.__name__.replace('Field', ''))
def __init__(self, db, conf):
self.db = db
self.conf = conf
self.name = str(self.__class__).split("'")[1]
self.target_name = utils.camel_to_snake(self.name.split('.')[-1])
self.target_collection = storage.MongoCollection(
self.db, self.target_name)
self.log = logging.getLogger(self.name)
self.batch_process = True
def visit(self, node):
"""Find the corresponding method with a given node
and call it.
Args:
node (SyntaxNode): Node to be visited.
Returns:
any: Return value of the actual method.
"""
camel_case_name = camel_to_snake(node.__class__.__name__)
method = getattr(self, f"visit_{camel_case_name}")
return method(node)
def get(self, request_url, rdata, ret_class=None):
request_url = urljoin(request_url, rdata.arg)
print(request_url)
r = requests.get(request_url,
params=rdata.params,
headers=rdata.headers)
try:
ret = r.json().get('Data')
except AttributeError:
ret = r.json()
if ret_class:
yield from (ret_class(
**{camel_to_snake(k): v
for k, v in result.items()}) for result in ret)
else:
yield from ret
def role( cls ):
"""
The name of the role performed by instances of this class, or rather by the EC2 instances
they represent.
"""
return camel_to_snake( cls.__name__, '-' )
def train(net,
optimizer,
criterion,
data_loader,
epoch,
scheduler=None,
display_iter=100,
device=torch.device('cpu'),
display=None,
val_loader=None,
supervision='full'):
"""
Training loop to optimize a network for several epochs and a specified loss
Args:
net: a PyTorch model
optimizer: a PyTorch optimizer
data_loader: a PyTorch dataset loader
epoch: int specifying the number of training epochs
criterion: a PyTorch-compatible loss function, e.g. nn.CrossEntropyLoss
device (optional): torch device to use (defaults to CPU)
display_iter (optional): number of iterations before refreshing the
display (False/None to switch off).
scheduler (optional): PyTorch scheduler
val_loader (optional): validation dataset
supervision (optional): 'full' or 'semi'
"""
if criterion is None:
raise Exception("Missing criterion. You must specify a loss function.")
net.to(device)
save_epoch = epoch // 20 if epoch > 20 else 1
losses = np.zeros(1000000)
mean_losses = np.zeros(100000000)
iter_ = 1
loss_win, val_win = None, None
val_accuracies = []
for e in tqdm(range(1, epoch + 1), desc="Training the network"):
# Set the network to training mode
net.train()
avg_loss = 0.
# Run the training loop for one epoch
for batch_idx, (data, target) in tqdm(enumerate(data_loader),
total=len(data_loader)):
# Load the data into the GPU if required
data, target = data.to(device), target.to(device)
optimizer.zero_grad()
if supervision == 'full':
output = net(data)
loss = criterion(output, target)
elif supervision == 'semi':
outs = net(data)
output, rec = outs
loss = criterion[0](
output,
target) + net.aux_loss_weight * criterion[1](rec, data)
else:
raise ValueError(
"supervision mode \"{}\" is unknown.".format(supervision))
loss.backward()
optimizer.step()
avg_loss += loss.item()
losses[iter_] = loss.item()
mean_losses[iter_] = np.mean(losses[max(0, iter_ - 100):iter_ + 1])
if display_iter and iter_ % display_iter == 0:
string = 'Train (epoch {}/{}) [{}/{} ({:.0f}%)]\tLoss: {:.6f}'
string = string.format(e, epoch, batch_idx * len(data),
len(data) * len(data_loader),
100. * batch_idx / len(data_loader),
mean_losses[iter_])
update = None if loss_win is None else 'append'
loss_win = display.line(
X=np.arange(iter_ - display_iter, iter_),
Y=mean_losses[iter_ - display_iter:iter_],
win=loss_win,
update=update,
opts={
'title': "Training loss",
'xlabel': "Iterations",
'ylabel': "Loss"
})
tqdm.write(string)
if len(val_accuracies) > 0:
val_win = display.line(Y=np.array(val_accuracies),
X=np.arange(len(val_accuracies)),
win=val_win,
opts={
'title': "Validation accuracy",
'xlabel': "Epochs",
'ylabel': "Accuracy"
})
iter_ += 1
del (data, target, loss, output)
# Update the scheduler
avg_loss /= len(data_loader)
if val_loader is not None:
val_acc = val(net,
val_loader,
device=device,
supervision=supervision)
val_accuracies.append(val_acc)
metric = -val_acc
else:
metric = avg_loss
if isinstance(scheduler, optim.lr_scheduler.ReduceLROnPlateau):
scheduler.step(metric)
elif scheduler is not None:
scheduler.step()
# Save the weights
if e % save_epoch == 0:
save_model(net,
camel_to_snake(str(net.__class__.__name__)),
data_loader.dataset.name,
epoch=e,
metric=abs(metric))
def test_camel_to_snake2(self):
snake_str = 'normalizing_flow1'
camel_str = 'NormalizingFlow1'
self.assertEqual(camel_to_snake(camel_str), snake_str)
def test_camel_to_snake1(self):
snake_str = 'this_is_string'
camel_str = 'ThisIsString'
self.assertEqual(camel_to_snake(camel_str), snake_str)
def train(net,
optimizer,
criterion_labeled,
criterion_val,
train_loader,
writer,
args,
display_iter=10,
display=None,
val_loader=None):
"""
Training loop to optimize a network for several epochs and a specified loss
Args:
net: a PyTorch model
optimizer: a PyTorch optimizer
train_loader: a PyTorch dataset loader for the labeled dataset
epoch: int specifying the number of training epochs
threshold: probability thresold for pseudo labels acceptance
criterion_labeled: a PyTorch-compatible loss function, e.g. nn.CrossEntropyLoss
device (optional): torch device to use (defaults to CPU)
display_iter (optional): number of iterations before refreshing the
display (False/None to switch off).
scheduler (optional): PyTorch scheduler
val_loader (optional): validation dataset
supervision (optional): 'full' or 'semi'
"""
save_dir = args.save_dir
if criterion_labeled is None:
raise Exception("Missing criterion. You must specify a loss function.")
net.to(args.device)
save_epoch = args.epochs // 20 if args.epochs > 20 else 1
losses = np.zeros(1000000)
mean_losses = np.zeros(100000000)
iter_ = 1
loss_win, val_win = None, None
val_accuracies = []
for e in tqdm(range(1, args.epochs + 1), desc="Training the network"):
# Set the network to training mode
net.train()
avg_loss = 0.
losses_meter = AverageMeter()
train_loss = 0
correct = 0
total = 0
# Run the training loop for one epoch
for batch_idx, (data, targets) in tqdm(enumerate(train_loader),
total=len(train_loader)):
targets = targets - 1
batch_size = data.shape[0]
# Load the data into the GPU if required
data = data.to(args.device)
targets = targets.to(args.device)
inputs, targets_a, targets_b, lam = mixup_data(
data, targets, args.alpha, args.device)
inputs, targets_a, targets_b = map(Variable,
(inputs, targets_a, targets_b))
outputs = net(inputs)
loss = mixup_criterion(criterion_labeled, outputs, targets_a,
targets_b, lam)
train_loss += loss.item()
_, predicted = torch.max(outputs.data, 1)
total += targets.size(0)
correct += (
lam * predicted.eq(targets_a.data).to('cpu').sum().float() +
(1 - lam) *
predicted.eq(targets_b.data).to('cpu').sum().float())
optimizer.zero_grad()
loss.backward()
optimizer.step()
args.scheduler.step()
losses_meter.update(loss.item())
if display_iter and iter_ % display_iter == 0:
string = 'Train (epoch {}/{}) [Iter: {:4}/{:4}]\tLr: {:.6f}\tLoss: {:.4f}\tAcc: {:.4f} ({:.4f}/{:.4f})'
string = string.format(e, args.epochs, batch_idx + 1,
args.iterations,
args.scheduler.get_last_lr()[0],
train_loss / (batch_idx + 1),
100. * correct / total, correct, total)
update = None if loss_win is None else 'append'
if display is not None:
loss_win = display.line(
X=np.arange(iter_ - display_iter, iter_),
Y=mean_losses[iter_ - display_iter:iter_],
win=loss_win,
update=update,
opts={
'title': "Training loss",
'xlabel': "Iterations",
'ylabel': "Loss"
})
tqdm.write(string)
'''
for name, param in net.named_parameters():
if param.requires_grad:
print(name, torch.min(param.data), torch.max(param.data))
'''
if len(val_accuracies) > 0 and display is not None:
val_win = display.line(Y=np.array(val_accuracies),
X=np.arange(len(val_accuracies)),
win=val_win,
opts={
'title': "Validation accuracy",
'xlabel': "Epochs",
'ylabel': "Accuracy"
})
iter_ += 1
del (inputs, targets)
# Update the scheduler
avg_loss /= len(train_loader)
if val_loader is not None:
val_acc, val_loss = val(net,
val_loader,
criterion_val,
device=args.device,
supervision='full')
#val_accuracies.append(val_acc)
#metric = -val_acc
else:
metric = avg_loss
writer.add_scalar('train/1.train_loss', losses_meter.avg, e)
writer.add_scalar('train/2.mixup_accuracy', 100. * correct / total, e)
writer.add_scalar('test/1.test_acc', val_acc.avg, e)
writer.add_scalar('test/2.test_loss', val_loss.avg, e)
# Save the weights
if e % save_epoch == 0 and args.save == True:
save_model(net,
utils.camel_to_snake(str(net.__class__.__name__)),
train_loader.dataset.name,
args.save_dir,
epoch=args.epochs,
metric=abs(-val_acc.avg))
from utils import (camel_to_snake, snake_to_camel)
print(snake_to_camel('python_exercises'))
print(camel_to_snake('PythonExercises'))
def train(net,
optimizer,
criterion_labeled,
criterion_unlabeled,
criterion_val,
labeled_data_loader,
unlabeled_data_loader,
writer,
args,
display_iter=10,
display=None,
val_loader=None):
"""
Training loop to optimize a network for several epochs and a specified loss
Args:
net: a PyTorch model
optimizer: a PyTorch optimizer
labeled_data_loader: a PyTorch dataset loader for the labeled dataset
unlabeled_data_loader: a PyTorch dataset loader for the weakly and
strongly augmented, unlabeled dataset
epoch: int specifying the number of training epochs
threshold: probability threshold for pseudo labels acceptance
criterion_labeled: a PyTorch-compatible loss function, e.g. nn.CrossEntropyLoss
for the labeled Training
criterion_unlabeled: a PyTorch-compatible loss function, e.g. nn.CrossEntropyLoss
for the unlabeled training
device (optional): torch device to use (defaults to CPU)
display_iter (optional): number of iterations before refreshing the
display (False/None to switch off).
scheduler (optional): PyTorch scheduler
val_loader (optional): validation dataset
supervision (optional): 'full' or 'semi'
"""
save_dir = args.save_dir
if criterion_labeled is None or criterion_unlabeled is None:
raise Exception("Missing criterion. You must specify a loss function.")
net.to(args.device)
save_epoch = args.epochs // 20 if args.epochs > 20 else 1
losses = np.zeros(1000000)
mean_losses = np.zeros(100000000)
iter_ = 1
loss_win, val_win = None, None
val_accuracies = []
for e in tqdm(range(1, args.epochs + 1), desc="Training the network"):
# Set the network to training mode
net.train()
avg_loss = 0.
losses_meter = AverageMeter()
losses_x = AverageMeter()
losses_u = AverageMeter()
train_loader = zip(labeled_data_loader, unlabeled_data_loader)
# Run the training loop for one epoch
for batch_idx, (data_x,
data_u) in tqdm(enumerate(train_loader),
total=len(unlabeled_data_loader)):
inputs_x, targets_x = data_x
#Try to remove prediction for ignored class
targets_x = targets_x - 1
inputs_u_w, inputs_u_s = data_u
batch_size = inputs_x.shape[0]
# Load the data into the GPU if required
inputs = torch.cat(
(inputs_x, inputs_u_w, inputs_u_s)).to(args.device)
targets_x = targets_x.to(args.device)
logits = net(inputs)
logits_x = logits[:args.batch_size]
logits_u_w, logits_u_s = logits[args.batch_size:].chunk(2)
del logits
Lx = criterion_labeled(logits_x, targets_x)
pseudo_label = torch.softmax(logits_u_w.detach_(), dim=-1)
max_probs, targets_u = torch.max(pseudo_label, dim=-1)
#To see that we don't predict any ignored labels
#ignored_index = np.nonzero(targets_u==args.ignored_labels)
mask = max_probs.ge(args.threshold).float()
#mask[ignored_index] = 0
Lu = (criterion_unlabeled(logits_u_s, targets_u) * mask).mean()
if e < args.pretrain:
loss = Lx
else:
loss = Lx + 1 * Lu
optimizer.zero_grad()
loss.backward()
optimizer.step()
args.scheduler.step()
losses_meter.update(loss.item())
losses_x.update(Lx.item())
losses_u.update(Lu.item())
mask_prob = mask.mean().item()
avg_loss += loss.item()
losses[iter_] = loss.item()
mean_losses[iter_] = np.mean(losses[max(0, iter_ - 100):iter_ + 1])
if display_iter and iter_ % display_iter == 0:
string = 'Train (epoch {}/{}) [Iter: {:4}/{:4}]\tLr: {:.6f}\tLoss: {:.4f}\tLoss_labeled: {:.4f}\tLoss_unlabeled: {:.4f}\tMask: {:.4f}'
string = string.format(e, args.epochs, batch_idx + 1,
args.iterations,
args.scheduler.get_last_lr()[0],
losses_meter.avg, losses_x.avg,
losses_u.avg, mask_prob)
update = None if loss_win is None else 'append'
if display is not None:
loss_win = display.line(
X=np.arange(iter_ - display_iter, iter_),
Y=mean_losses[iter_ - display_iter:iter_],
win=loss_win,
update=update,
opts={
'title': "Training loss",
'xlabel': "Iterations",
'ylabel': "Loss"
})
tqdm.write(string)
'''
for name, param in net.named_parameters():
if param.requires_grad:
print(name, torch.min(param.data), torch.max(param.data))
'''
if len(val_accuracies) > 0 and display is not None:
val_win = display.line(Y=np.array(val_accuracies),
X=np.arange(len(val_accuracies)),
win=val_win,
opts={
'title': "Validation accuracy",
'xlabel': "Epochs",
'ylabel': "Accuracy"
})
iter_ += 1
del (data_x, data_u, loss, inputs_u_s, inputs_u_w, inputs_x,
targets_x, logits_x, logits_u_s, logits_u_w, Lx, Lu)
# Update the scheduler
avg_loss /= len(labeled_data_loader)
if val_loader is not None:
val_acc, val_loss = val(net,
val_loader,
criterion_val,
device=args.device,
supervision='full')
#val_accuracies.append(val_acc)
#metric = -val_acc
else:
metric = avg_loss
writer.add_scalar('train/1.train_loss', losses_meter.avg, e)
writer.add_scalar('train/2.train_loss_x', losses_x.avg, e)
writer.add_scalar('train/3.train_loss_u', losses_u.avg, e)
writer.add_scalar('train/4.mask', mask_prob, e)
writer.add_scalar('test/1.test_acc', val_acc.avg, e)
writer.add_scalar('test/2.test_loss', val_loss.avg, e)
# Save the weights
if e % save_epoch == 0 and args.save == True:
save_model(net,
utils.camel_to_snake(str(net.__class__.__name__)),
labeled_data_loader.dataset.name,
args.save_dir,
epoch=args.epochs,
metric=abs(-val_acc.avg))
def get_vt_field_name(field_name: str) -> str:
"""Converts Logs API parameter name to Vertica column name"""
prefixes = ['ym:s:', 'ym:pv:']
for prefix in prefixes:
field_name = field_name.replace(prefix, '')
return utils.camel_to_snake(field_name)
def test_camel_to_snake():
camel_to_snake("HelloWorld") == "hello_world"
underscore("HelloWorld") == "hello_world"
def get_name(node):
return camel_to_snake(node.__class__.__name__)
def code(self):
return camel_to_snake(self.__class__.__name__)
def __init__(cls, name, bases, attrs):
super(PluginMount, cls).__init__(name, bases, attrs)
command_name = cls.command_name if hasattr(
cls, 'command_name') else camel_to_snake(cls.__name__)
setattr(Commands, command_name, cls)
def _collection_name(cls):
"""Get name of collection by converting class name to snake case"""
return camel_to_snake(cls.__name__)
