def build(self,
task: Task,
base_dataset: Dataset,
epochs=50,
components=200,
gmm_clusters=10,
logloss_weight=2.0,
lr=5e-3,
batch_size=5000):
task.set_user_properties(
**to_clear_ml_params(locals(), ["task", "self"]))
self.item_ids = list(range(len(base_dataset.id_to_item)))
self.avg_ratings_per_user = base_dataset.n_ratings / base_dataset.n_users
if self.user_vectors is None:
self._build_mf(base_dataset, task, components)
self._build_torch_model(base_dataset, task, components)
self._train_torch_model(base_dataset,
task,
epochs,
logloss_weight=logloss_weight,
lr=lr,
batch_size=batch_size)
self._build_gmm(task, gmm_clusters)
def main(task: Task, dataset_name, generator_name, extra):
if generator_name not in generators.name_to_class:
raise Exception("Unknown generator name" + ", ".join(generators.name_to_class.keys()))
dataset_obj = datasets.get_dataset(dataset_name) # type: Dataset
generator_obj = generators.name_to_class[generator_name](verbose=True) # type: SyntheticDataGenerator
generator_obj.build(task, dataset_obj, **extra)
with open("generator.pickle", "wb") as f:
pickle.dump(generator_obj, f)
task.upload_artifact('generator', artifact_object='generator.pickle', delete_after_upload=True)
def mp_worker(arguments):
print('sub process', os.getpid())
inputs, the_time = arguments
from random import randint
additional_parameters = {
'stuff_' + str(randint(0, 100)): 'some stuff ' + str(randint(0, 100))
}
Task.current_task().connect(additional_parameters)
print(" Process %s\tWaiting %s seconds" % (inputs, the_time))
time.sleep(int(the_time))
print(" Process %s\tDONE" % inputs)
def main(epochs):
Task.init(project_name="examples", task_name="fastai v1")
path = untar_data(URLs.MNIST_SAMPLE)
data = ImageDataBunch.from_folder(path, ds_tfms=(rand_pad(2, 28), []), bs=64, num_workers=0)
data.normalize(imagenet_stats)
learn = cnn_learner(data, models.resnet18, metrics=accuracy)
accuracy(*learn.get_preds())
learn.fit_one_cycle(epochs, 0.01)
def __init__(self, **kwargs: Any):
try:
from clearml import Task
from clearml.binding.frameworks.tensorflow_bind import WeightsGradientHistHelper
except ImportError:
try:
# Backwards-compatibility for legacy Trains SDK
from trains import Task
from trains.binding.frameworks.tensorflow_bind import WeightsGradientHistHelper
except ImportError:
raise RuntimeError(
"This contrib module requires clearml to be installed. "
"You may install clearml using: \n pip install clearml \n")
experiment_kwargs = {
k: v
for k, v in kwargs.items()
if k not in ("project_name", "task_name", "task_type")
}
if self.bypass_mode():
warnings.warn("ClearMLSaver: running in bypass mode")
class _Stub(object):
def __call__(self, *_: Any, **__: Any) -> "_Stub":
return self
def __getattr__(self, attr: str) -> "_Stub":
if attr in ("name", "id"):
return "" # type: ignore[return-value]
return self
def __setattr__(self, attr: str, val: Any) -> None:
pass
self._task = _Stub()
else:
# Try to retrieve current the ClearML Task before trying to create a new one
self._task = Task.current_task()
if self._task is None:
self._task = Task.init(
project_name=kwargs.get("project_name"),
task_name=kwargs.get("task_name"),
task_type=kwargs.get("task_type", Task.TaskTypes.training),
**experiment_kwargs,
)
self.clearml_logger = self._task.get_logger()
self.grad_helper = WeightsGradientHistHelper(
logger=self.clearml_logger)
def model_prune(task_args):
# Create an output directory if it doesn't exist.
get_output("mkdir -p /home/{}/experiment_dir_pruned".format(
task_args.arch))
train_task = Task.get_task(task_id=task_args.trains_model_task)
unpruned_weights = train_task.artifacts["unpruned_weights"].get_local_copy(
)
tlt_prune(task_args, unpruned_weights)
tlt_task = Task.current_task()
tlt_task.upload_artifact(
name="pruned_weights",
artifact_object=os.path.join(
os.path.expandvars("{}".format(task_args.output_file))),
)
def main():
task = Task.init(project_name="Nvidia Clara examples with ClearML", task_name="Export models to Artifacts")
task.set_base_docker(
"nvcr.io/nvidia/clara-train-sdk:v3.1.01 --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864"
)
parser = ArgumentParser()
parser.add_argument('--model_name', required=True)
parser.add_argument('--model_file_path', required=True)
parser.add_argument('--input_node_names', required=True)
parser.add_argument('--output_node_names', required=True)
parser.add_argument('--checkpoint_ext', default='.ckpt')
parser.add_argument('--meta_file_ext', default='.meta')
parser.add_argument('--regular_frozen_file_ext', default='.fzn.pb')
parser.add_argument('--trt_file_ext', default='.trt.pb')
parser.add_argument('--trt_precision_mode', default='FP32')
parser.add_argument('--trt_dynamic_mode', action='store_true')
parser.add_argument('--max_batch_size', type=int, default=4)
parser.add_argument('--trt_min_seg_size', type=int, default=50)
parser.add_argument('--model_file_format', default='CKPT')
parser.add_argument('--trtis_export', action='store_true')
parser.add_argument('--trtis_model_name', type=str, default='tlt_model')
parser.add_argument('--trtis_input_shape',
nargs='+',
type=int,
help='Full input shape. For example, --trtis_input_shape dim1 dim2 dim3 dim4 in CDHW case')
parser.add_argument('--models_task',
type=str,
help='The training task id')
set_env_vars()
args = parser.parse_args()
if args.models_task:
m_task = Task.get_task(task_id=args.models_task)
output_models = m_task.get_models().get("output")
for mdl in output_models:
m_output = mdl.get_local_copy()
for model in os.listdir(m_output):
os.system("mv {} {}".format(os.path.join(m_output, model), args.model_file_path))
export_main()
# noinspection PyBroadException
try:
task.upload_artifact(name="fzn file", artifact_object="{}{}".format(os.path.join(args.model_file_path, args.model_name), args.regular_frozen_file_ext))
print("frozen file uploaded as artifact")
except Exception:
pass
# noinspection PyBroadException
try:
task.upload_artifact(name="trt file", artifact_object="{}{}".format(os.path.join(args.model_file_path, args.model_name), args.trt_file_ext))
print("trt file uploaded as artifact")
except Exception:
pass
def remote_run_experiment(self):
for parameter_setup in self._parameter_setups:
print(parameter_setup)
task = Task.create(
project_name=f"{self._project_name}",
task_name=self.make_task_name(parameter_setup),
repo=self._repo,
branch=self._branch,
script=self._script,
requirements_file="../requirements.txt"
)
task.set_parent(Task.current_task().id)
task.connect(parameter_setup)
Task.enqueue(task, self._queue)
def cli():
title = 'ClearML launch - launch any codebase on remote machine running clearml-agent'
print(title)
parser = ArgumentParser(description=title)
setup_parser(parser)
# get the args
args = parser.parse_args()
if args.version:
from ...version import __version__
print('Version {}'.format(__version__))
exit(0)
create_populate = CreateAndPopulate(
project_name=args.project,
task_name=args.name,
task_type=args.task_type,
repo=args.repo or args.folder,
branch=args.branch,
commit=args.commit,
script=args.script,
working_directory=args.cwd,
packages=args.packages,
requirements_file=args.requirements,
docker=args.docker,
base_task_id=args.base_task_id,
add_task_init_call=not args.skip_task_init,
raise_on_missing_entries=True,
verbose=True,
)
# verify args
create_populate.update_task_args(args.args)
print('Creating new task')
create_populate.create_task()
# update Task args
create_populate.update_task_args(args.args)
print('New task created id={}'.format(create_populate.get_id()))
if not args.queue:
print('Warning: No queue was provided, leaving task in draft-mode.')
exit(0)
Task.enqueue(create_populate.task, queue_name=args.queue)
print('Task id={} sent for execution on queue {}'.format(
create_populate.get_id(), args.queue))
print('Execution log at: {}'.format(
create_populate.task.get_output_log_web_page()))
def main(epochs):
Task.init(project_name="examples", task_name="fastai v2")
path = untar_data(URLs.PETS)
files = get_image_files(path / "images")
dls = ImageDataLoaders.from_name_func(path,
files,
label_func,
item_tfms=Resize(224),
num_workers=0)
dls.show_batch()
learn = cnn_learner(dls, resnet34, metrics=error_rate)
learn.fine_tune(epochs)
learn.show_results()
def main():
# a bit of code that creates clearml logging (formerly trains) if clearml
# is available
if trains_loaded and not ONLINE:
task = Task.init(project_name='MineRL',
task_name='kmeans pic+pic+tre 1024 + flips whatever')
logger = task.get_logger()
else:
logger = None
os.makedirs("train", exist_ok=True)
cached_kmeans("train", "MineRLObtainDiamondVectorObf-v0")
print("lets gooo", file=sys.stderr)
train_files = absolute_file_paths('data/MineRLObtainIronPickaxeVectorObf-v0')+\
absolute_file_paths('data/MineRLObtainIronPickaxeVectorObf-v0')+\
absolute_file_paths('data/MineRLTreechopVectorObf-v0')
model = Model()
shuffle(train_files)
loader = BatchSeqLoader(16, train_files, SEQ_LEN, model)
if LOAD:
model.load_state_dict(torch.load("train/model.tm"))
model.cuda()
train(model, "train", 150000000, loader, logger)
torch.save(model.state_dict(), "train/model.tm")
print("ok", file=sys.stderr)
loader.kill()
def main():
# Connecting ClearML with the current process,
# from here on everything is logged automatically
task = Task.init(project_name="examples",
task_name="HTML samples reporting")
print('reporting html files into debug samples section')
# Get the task logger,
# You can also call Task.current_task().get_logger() from anywhere in your code.
logger = task.get_logger()
# report html as debug samples
report_html_image(logger)
report_html_graph(logger)
report_html_groupby(logger)
report_html_periodic_table(logger)
report_html_url(logger)
# force flush reports
# If flush is not called, reports are flushed in the background every couple of seconds,
# and at the end of the process execution
logger.flush()
print('We are done reporting, have a great day :)')
def main():
task = Task.init(project_name="pbl_example", task_name="tf2 mnist args", output_uri='/datasets/output')
parameters = {'epochs': 5}
task.connect(parameters)
ds = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = ds.load_data()
x_test, x_train = x_test.reshape((10000, 28, 28, 1)), x_train.reshape(60000, 28, 28, 1)
x_test, x_train = x_test/255.0, x_train/255.0
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(20, (5, 5), padding="SAME", activation="relu"),
tf.keras.layers.MaxPool2D(2, 2, padding="SAME"),
tf.keras.layers.Conv2D(40, (5, 5), padding="SAME", activation="relu"),
tf.keras.layers.MaxPool2D(2, 2, padding="SAME"),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(512, activation="relu"),
tf.keras.layers.Dense(10, activation="softmax")
])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
model.fit(x_train, y_train, epochs=parameters['epochs'])
optimizer = tf.keras.optimizers.Adam()
ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, net=model)
manager = tf.train.CheckpointManager(ckpt, os.path.join(gettempdir(), 'tf_ckpts'), max_to_keep=3)
save_path = manager.save()
test_loss, test_acc = model.evaluate(x_test, y_test)
# model.save_weights('./model')
print(save_path)
def __init__(self,
task: Task = None,
projectName: str = None,
taskName: str = None,
additionalLoggingValuesDict=None):
"""
:param task: instances of trains.Task
:param projectName: only necessary if task is not provided
:param taskName: only necessary if task is not provided
:param additionalLoggingValuesDict:
"""
if task is None:
if projectName is None or taskName is None:
raise ValueError(
"Either the trains task or the project name and task name have to be provided"
)
self.task = Task.init(project_name=projectName,
task_name=taskName,
reuse_last_task_id=False)
else:
if projectName is not None:
log.warning(
f"projectName parameter with value {projectName} passed even though task has been given, "
f"will ignore this parameter")
if taskName is not None:
log.warning(
f"taskName parameter with value {taskName} passed even though task has been given, "
f"will ignore this parameter")
self.task = task
self.logger = self.task.get_logger()
super().__init__(
additionalLoggingValuesDict=additionalLoggingValuesDict)
def seed(self):
for id in self.get_ids():
task = self.call_func(
"Task.create",
id,
lambda id_: Task.create(project_name="MLFlow Migration",
task_name=id_),
self.get_run_name_by_id(id),
)
self.call_func(
"transmit_information",
id,
lambda id_: self.transmit_information(id_),
id,
)
self.call_func("transmit_metrics", id,
lambda id_: self.transmit_metrics(id_), id)
self.call_func("transmit_artifacts", id,
lambda id_: self.transmit_artifacts(id_), id)
task.mark_started()
task.completed()
output_log_web_page = task.get_output_log_web_page()
url_parts = output_log_web_page.split("projects")
project_id = url_parts[1].split("/")[1]
self.project_link = url_parts[0] + "/projects/" + project_id
self.migration_count += 1
self.pbar.update(1)
def get_pruning_results(project_name, pruning_methods, y_label):
print("Retrieving tasks...")
tasks = Task.get_tasks(project_name=project_name)
print('Done')
# logger = get_clearml_logger(project_name, 'Summary').logger
print(f"Generating graph comparison for {y_label}")
x_values, y_values, labels, num_funcs = compare_clearml_project(
pruning_methods,
project_name=project_name,
tasks=tasks,
x_name='dim_edges',
y_name=y_label)
data = seperate_labels(x_values, y_values, labels, num_funcs)
for n_funcs, n_funcs_data in data.items():
plt.figure()
for method, m_data in n_funcs_data.items():
x = m_data['x']
y = m_data['y']
order = np.argsort(x)
x = [x[i] for i in order]
y = [y[i] for i in order]
plt.plot(x, y, label=method)
plt.title(f"Prunning using {n_funcs} minhash functions")
plt.xlabel("Edge dim")
plt.ylabel("Accuracy")
plt.legend()
plt.show()
def get_output(command, return_command=False):
save_artifact = False
if command.startswith("tlt") and (
command.partition(" ")[0] != "tlt-train"
and command.partition(" ")[0] != "tlt-converter"):
command_prefix, _, command_args = command.partition(" ")
command_prefix = shutil.which(command_prefix)
command = "{} {} {}".format(sys.executable, command_prefix,
command_args)
elif command.startswith("ls -rlt"): # we will save as artifact if needed
save_artifact = True
print("=============== Running command: {}".format(command))
result = run(command,
stdout=PIPE,
stderr=STDOUT,
universal_newlines=True,
shell=True)
print(result.stdout)
if save_artifact:
name = result.stdout.split("\n")[-2].rpartition(" ")[2]
if name.endswith("tlt") or name.endswith("etlt") or name.endswith(
"hdf5"):
command_path = command.partition(" ")[2].rpartition(" ")[2]
tlt_task = Task.current_task()
tlt_task.upload_artifact(
name=name,
artifact_object=os.path.join(os.path.expandvars(command_path),
name),
)
if return_command:
return result.stdout
def download_pretrained_model(model_name, ngc_model, conf_file):
model_file = (get_field_from_config(
conf_file, "pretrained_model_file").strip().strip('"'))
if model_file:
model_dir = model_file.rpartition("/")[0].rpartition("/")[0]
os.makedirs(model_dir)
else:
model_dir = "tmp/"
os.makedirs(model_dir)
# Download the pretrained model from NGC
download_path = None
command_output = get_output(
"ngc registry model download-version {} --dest {}".format(
ngc_model, model_dir),
return_command=True,
)
for output in command_output.split("\n"):
if output.startswith("Downloaded local path"):
download_path = output.partition(":")[2].strip()
break
if download_path:
tlt_task = Task.current_task()
tlt_task.upload_artifact(
name=model_name,
artifact_object=os.path.join(
os.path.expandvars("{}".format(download_path)),
"{}.hdf5".format(model_name),
),
)
def summary_clearml_project(project_name, x_label, seperate_legends):
print("Retrieving tasks...")
tasks = Task.get_tasks(project_name=project_name)
print('Done')
y_labels = set(tasks[0].data.hyperparams['General'])
y_labels.remove(x_label)
logger = get_clearml_logger(project_name, 'Summary').logger
for y_label in y_labels:
print(f"Generating graph comparison for {y_label}")
x_values, y_values, labels = compare_clearml_project(
seperate_legends,
project_name=project_name,
tasks=tasks,
x_name=x_label,
y_name=y_label)
labeled_x, labeled_y, unique_labels = seperate_labels(
x_values, y_values, labels)
for label in unique_labels:
# plt.plot(labeled_x[label], labeled_y[label], label=label)
scatter2d = np.vstack((labeled_x[label], labeled_y[label])).T
logger.report_scatter2d(title=y_label,
series=label,
iteration=0,
scatter=scatter2d,
xaxis=x_label,
yaxis=y_label)
def get_pruning_results(project_name, pruning_methods, y_label):
print("Retrieving tasks...")
tasks = Task.get_tasks(project_name=project_name)
print('Done')
# logger = get_clearml_logger(project_name, 'Summary').logger
print(f"Generating graph comparison for {y_label}")
x_values, y_values, labels = compare_clearml_project(
pruning_methods,
project_name=project_name,
tasks=tasks,
x_name='keep edges',
y_name=y_label)
labeled_x, labeled_y, unique_labels = seperate_labels(
x_values, y_values, labels)
for label in unique_labels:
print(f'======================= {label} =======================')
x = np.array(labeled_x[label])
y = np.array(labeled_y[label])
order = np.argsort(x)
x = x[order]
y = y[order]
for x_, y_ in zip(x, y):
print(f'({x_}, {y_})')
print()
def transmit_artifacts(self, id):
artifacts = (self.info[id][self.artifacts]
if self.artifacts in self.info[id].keys() else {})
task = self.call_func(
"Task.get_task",
id,
lambda id_: Task.get_task(project_name="MLFlow Migration",
task_name=id_),
self.get_run_name_by_id(id),
)
for type_, l in artifacts.items():
if type_ == "folder":
for name, obj in l:
task.upload_artifact(name=name, artifact_object=obj)
elif type_ == "text":
for name, obj in l:
task.upload_artifact(name=name, artifact_object=obj)
elif type_ == "dataframe":
for name, obj in l:
task.upload_artifact(name=name, artifact_object=obj)
elif type_ == "image":
for name, obj in l:
task.upload_artifact(name=name, artifact_object=obj)
elif type_ == "dictionary":
for name, obj in l:
task.upload_artifact(name=name, artifact_object=obj)
elif type_ == "storage-server":
for name, obj in l:
task.upload_artifact(name=name, artifact_object=obj)
def compare_clearml_project(seperate_legends,
project_name=None,
tasks=None,
x_name=None,
y_name=None):
tasks = Task.get_tasks(
project_name=project_name) if tasks is None else tasks
x_y_values = []
for task in tasks:
try:
x_y_values.append((
float(task.data.hyperparams['Args'][x_name].value),
float(task.data.hyperparams['General'][y_name].value),
task.data.hyperparams['Args'][seperate_legends].value,
float(
task.data.hyperparams['Args']['num_minhash_funcs'].value)))
except Exception as e:
print(e)
pass
x_values = [x for x, y, sep, sep2 in x_y_values]
y_values = [y for x, y, sep, sep2 in x_y_values]
labels1 = [sep for x, y, sep, sep2 in x_y_values]
labels2 = [sep2 for x, y, sep, sep2 in x_y_values]
return x_values, y_values, labels1, labels2
def setup_experiment_tracking(config, with_clearml, task_type="training"):
from datetime import datetime
assert task_type in ("training", "testing"), task_type
output_path = ""
if idist.get_rank() == 0:
if with_clearml:
from clearml import Task
schema = TrainvalConfigSchema if task_type == "training" else InferenceConfigSchema
task = Task.init("Pascal-VOC12 Training", config.config_filepath.stem, task_type=task_type)
task.connect_configuration(config.config_filepath.as_posix())
task.upload_artifact(config.script_filepath.name, config.script_filepath.as_posix())
task.upload_artifact(config.config_filepath.name, config.config_filepath.as_posix())
task.connect(get_params(config, schema))
output_path = Path(os.environ.get("CLEARML_OUTPUT_PATH", "/tmp"))
output_path = output_path / "clearml" / datetime.now().strftime("%Y%m%d-%H%M%S")
else:
import shutil
output_path = Path(os.environ.get("OUTPUT_PATH", "/tmp/output-pascal-voc12"))
output_path = output_path / task_type / config.config_filepath.stem
output_path = output_path / datetime.now().strftime("%Y%m%d-%H%M%S")
output_path.mkdir(parents=True, exist_ok=True)
shutil.copyfile(config.script_filepath.as_posix(), output_path / config.script_filepath.name)
shutil.copyfile(config.config_filepath.as_posix(), output_path / config.config_filepath.name)
output_path = output_path.as_posix()
return Path(idist.broadcast(output_path, src=0))
def main():
# Connecting ClearML with the current process,
# from here on everything is logged automatically
task = Task.init(project_name="examples", task_name="text reporting")
print("reporting text logs")
# report regular console print
print("This is standard output test")
# report stderr
print("This is standard error test", file=sys.stderr)
# Get the task logger,
# You can also call Task.current_task().get_logger() from anywhere in your code.
logger = task.get_logger()
# report text based logs
report_logs(logger)
# report text as debug example
report_debug_text(logger)
# force flush reports
# If flush is not called, reports are flushed in the background every couple of seconds,
# and at the end of the process execution
logger.flush()
print("We are done reporting, have a great day :)")
def _setup_check_clearml(self, logger: ClearMLLogger,
output_uri: str) -> None:
try:
from clearml import Task
except ImportError:
try:
# Backwards-compatibility for legacy Trains SDK
from trains import Task
except ImportError:
raise RuntimeError(
"This contrib module requires clearml to be installed. "
"You may install clearml using: \n pip install clearml \n")
if logger and not isinstance(logger, ClearMLLogger):
raise TypeError("logger must be an instance of ClearMLLogger")
self._task = Task.current_task()
if not self._task:
raise RuntimeError(
"ClearMLSaver requires a ClearML Task to be initialized. "
"Please use the `logger` argument or call `clearml.Task.init()`."
)
if output_uri:
self._task.output_uri = output_uri
def my_app(cfg):
# type (DictConfig) -> None
task = Task.init(project_name="examples", task_name="hydra configuration")
logger = task.get_logger()
logger.report_text(
"You can view your full hydra configuration under Configuration tab in the UI"
)
print(OmegaConf.to_yaml(cfg))
def run_fixed_lambda_bbcluster(train_cluster_data, val_cluster_data, test_cluster_data, output_path, train_batch_size, eval_steps,
num_epochs, warmup_frac, lambda_val, reg, beta, loss_name, use_model_device, model_name='distilbert-base-uncased', out_features=256):
task = Task.init(project_name='BB Clustering', task_name='bbclustering_fixed_lambda')
config_dict = {'lambda_val': lambda_val, 'reg': reg}
config_dict = task.connect(config_dict)
if torch.cuda.is_available():
device = torch.device('cuda')
print('CUDA is available and using device: '+str(device))
else:
device = torch.device('cpu')
print('CUDA not available, using device: '+str(device))
### Configure sentence transformers for training and train on the provided dataset
# Use Huggingface/transformers model (like BERT, RoBERTa, XLNet, XLM-R) for mapping tokens to embeddings
word_embedding_model = models.Transformer(model_name)
# Apply mean pooling to get one fixed sized sentence vector
pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),
pooling_mode_mean_tokens=True,
pooling_mode_cls_token=False,
pooling_mode_max_tokens=False)
doc_dense_model = models.Dense(in_features=pooling_model.get_sentence_embedding_dimension(), out_features=out_features,
activation_function=nn.Tanh())
model = CustomSentenceTransformer(modules=[word_embedding_model, pooling_model, doc_dense_model])
# model = SentenceTransformer(modules=[word_embedding_model, pooling_model])
GPUtil.showUtilization()
if loss_name == 'bbspec':
loss_model = BBSpectralClusterLossModel(model=model, device=device,
lambda_val=config_dict.get('lambda_val', lambda_val),
reg_const=config_dict.get('reg', reg), beta=beta)
else:
loss_model = BBClusterLossModel(model=model, device=device,
lambda_val=config_dict.get('lambda_val', lambda_val),
reg_const=config_dict.get('reg', reg))
# reg_loss_model = ClusterDistLossModel(model=model)
train_dataloader = DataLoader(train_cluster_data, shuffle=True, batch_size=train_batch_size)
GPUtil.showUtilization()
# train_dataloader2 = DataLoader(train_cluster_data, shuffle=True, batch_size=train_batch_size)
evaluator = ClusterEvaluator.from_input_examples(val_cluster_data, use_model_device)
test_evaluator = ClusterEvaluator.from_input_examples(test_cluster_data, use_model_device)
GPUtil.showUtilization()
warmup_steps = int(len(train_dataloader) * num_epochs * warmup_frac) # 10% of train data
print("Raw BERT embedding performance")
model.to(device)
evaluator(model, output_path)
GPUtil.showUtilization()
# Train the model
model.fit(train_objectives=[(train_dataloader, loss_model)],
evaluator=evaluator,
test_evaluator=test_evaluator,
epochs=num_epochs,
evaluation_steps=eval_steps,
warmup_steps=warmup_steps,
output_path=output_path)
def transmit_information(self, id):
parameters = self.get_params(id)
general_information = self.get_general_information(id)
artifact = self.get_artifact(id)
tags = self.get_tags(id)
task = self.call_func(
"Task.get_task",
id,
lambda id_: Task.get_task(project_name="MLFlow Migration",
task_name=id_),
self.get_run_name_by_id(id),
)
task_values = self.call_func(
"task.export_task",
id,
lambda _: task.export_task(),
self.get_run_name_by_id(id),
)
task_values["comment"] = (tags["note.content"]
if "note.content" in tags.keys() else "")
task_values["hyperparams"]["Args"] = parameters
task_values["started"] = general_information["started"]
task_values["completed"] = general_information["completed"]
task_values["script"]["branch"] = (tags["source.git.branch"]
if "source.git.branch"
in tags.keys() else self.branch)
task_values["script"]["repository"] = (tags["source.git.repoURL"]
if "source.git.repoURL"
in tags.keys() else "")
task_values["script"]["version_num"] = (tags["source.git.commit"]
if "source.git.commit"
in tags.keys() else "")
task_values["script"]["entry_point"] = tags["entry_point"]
task_values["script"]["working_dir"] = tags["working_dir"]
if "project.env" in tags.keys():
task_values["script"]["requirements"][tags["project.env"]] = (
artifact["requirements"]
if "requirements" in artifact.keys() else "")
task_values["user"] = tags["user"]
self.call_func(
"task.update_task",
id,
lambda _task_values: task.update_task(_task_values),
task_values,
)
if len(tags["VALUETAG"].keys()) > 0:
self.call_func(
"task.connect_configuration",
id,
lambda _dict: task.connect_configuration(_dict,
name="MLflow Tags"),
tags["VALUETAG"],
)
def _clearml_log_params(params_dict):
try:
from clearml import Task
except ImportError:
# Backwards-compatibility for legacy Trains SDK
from trains import Task
task = Task.current_task()
task.connect(params_dict)
def _clearml_log_artifact(fp):
try:
from clearml import Task
except ImportError:
# Backwards-compatibility for legacy Trains SDK
from trains import Task
task = Task.current_task()
task.upload_artifact(Path(fp).name, fp)
