def build(self, task: Task, base_dataset: Dataset, epochs=50, components=200, gmm_clusters=10, logloss_weight=2.0, lr=5e-3, batch_size=5000): task.set_user_properties( **to_clear_ml_params(locals(), ["task", "self"])) self.item_ids = list(range(len(base_dataset.id_to_item))) self.avg_ratings_per_user = base_dataset.n_ratings / base_dataset.n_users if self.user_vectors is None: self._build_mf(base_dataset, task, components) self._build_torch_model(base_dataset, task, components) self._train_torch_model(base_dataset, task, epochs, logloss_weight=logloss_weight, lr=lr, batch_size=batch_size) self._build_gmm(task, gmm_clusters)
def main(task: Task, dataset_name, generator_name, extra): if generator_name not in generators.name_to_class: raise Exception("Unknown generator name" + ", ".join(generators.name_to_class.keys())) dataset_obj = datasets.get_dataset(dataset_name) # type: Dataset generator_obj = generators.name_to_class[generator_name](verbose=True) # type: SyntheticDataGenerator generator_obj.build(task, dataset_obj, **extra) with open("generator.pickle", "wb") as f: pickle.dump(generator_obj, f) task.upload_artifact('generator', artifact_object='generator.pickle', delete_after_upload=True)
def mp_worker(arguments): print('sub process', os.getpid()) inputs, the_time = arguments from random import randint additional_parameters = { 'stuff_' + str(randint(0, 100)): 'some stuff ' + str(randint(0, 100)) } Task.current_task().connect(additional_parameters) print(" Process %s\tWaiting %s seconds" % (inputs, the_time)) time.sleep(int(the_time)) print(" Process %s\tDONE" % inputs)
def main(epochs): Task.init(project_name="examples", task_name="fastai v1") path = untar_data(URLs.MNIST_SAMPLE) data = ImageDataBunch.from_folder(path, ds_tfms=(rand_pad(2, 28), []), bs=64, num_workers=0) data.normalize(imagenet_stats) learn = cnn_learner(data, models.resnet18, metrics=accuracy) accuracy(*learn.get_preds()) learn.fit_one_cycle(epochs, 0.01)
def __init__(self, **kwargs: Any): try: from clearml import Task from clearml.binding.frameworks.tensorflow_bind import WeightsGradientHistHelper except ImportError: try: # Backwards-compatibility for legacy Trains SDK from trains import Task from trains.binding.frameworks.tensorflow_bind import WeightsGradientHistHelper except ImportError: raise RuntimeError( "This contrib module requires clearml to be installed. " "You may install clearml using: \n pip install clearml \n") experiment_kwargs = { k: v for k, v in kwargs.items() if k not in ("project_name", "task_name", "task_type") } if self.bypass_mode(): warnings.warn("ClearMLSaver: running in bypass mode") class _Stub(object): def __call__(self, *_: Any, **__: Any) -> "_Stub": return self def __getattr__(self, attr: str) -> "_Stub": if attr in ("name", "id"): return "" # type: ignore[return-value] return self def __setattr__(self, attr: str, val: Any) -> None: pass self._task = _Stub() else: # Try to retrieve current the ClearML Task before trying to create a new one self._task = Task.current_task() if self._task is None: self._task = Task.init( project_name=kwargs.get("project_name"), task_name=kwargs.get("task_name"), task_type=kwargs.get("task_type", Task.TaskTypes.training), **experiment_kwargs, ) self.clearml_logger = self._task.get_logger() self.grad_helper = WeightsGradientHistHelper( logger=self.clearml_logger)
def model_prune(task_args): # Create an output directory if it doesn't exist. get_output("mkdir -p /home/{}/experiment_dir_pruned".format( task_args.arch)) train_task = Task.get_task(task_id=task_args.trains_model_task) unpruned_weights = train_task.artifacts["unpruned_weights"].get_local_copy( ) tlt_prune(task_args, unpruned_weights) tlt_task = Task.current_task() tlt_task.upload_artifact( name="pruned_weights", artifact_object=os.path.join( os.path.expandvars("{}".format(task_args.output_file))), )
def main(): task = Task.init(project_name="Nvidia Clara examples with ClearML", task_name="Export models to Artifacts") task.set_base_docker( "nvcr.io/nvidia/clara-train-sdk:v3.1.01 --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864" ) parser = ArgumentParser() parser.add_argument('--model_name', required=True) parser.add_argument('--model_file_path', required=True) parser.add_argument('--input_node_names', required=True) parser.add_argument('--output_node_names', required=True) parser.add_argument('--checkpoint_ext', default='.ckpt') parser.add_argument('--meta_file_ext', default='.meta') parser.add_argument('--regular_frozen_file_ext', default='.fzn.pb') parser.add_argument('--trt_file_ext', default='.trt.pb') parser.add_argument('--trt_precision_mode', default='FP32') parser.add_argument('--trt_dynamic_mode', action='store_true') parser.add_argument('--max_batch_size', type=int, default=4) parser.add_argument('--trt_min_seg_size', type=int, default=50) parser.add_argument('--model_file_format', default='CKPT') parser.add_argument('--trtis_export', action='store_true') parser.add_argument('--trtis_model_name', type=str, default='tlt_model') parser.add_argument('--trtis_input_shape', nargs='+', type=int, help='Full input shape. For example, --trtis_input_shape dim1 dim2 dim3 dim4 in CDHW case') parser.add_argument('--models_task', type=str, help='The training task id') set_env_vars() args = parser.parse_args() if args.models_task: m_task = Task.get_task(task_id=args.models_task) output_models = m_task.get_models().get("output") for mdl in output_models: m_output = mdl.get_local_copy() for model in os.listdir(m_output): os.system("mv {} {}".format(os.path.join(m_output, model), args.model_file_path)) export_main() # noinspection PyBroadException try: task.upload_artifact(name="fzn file", artifact_object="{}{}".format(os.path.join(args.model_file_path, args.model_name), args.regular_frozen_file_ext)) print("frozen file uploaded as artifact") except Exception: pass # noinspection PyBroadException try: task.upload_artifact(name="trt file", artifact_object="{}{}".format(os.path.join(args.model_file_path, args.model_name), args.trt_file_ext)) print("trt file uploaded as artifact") except Exception: pass
def remote_run_experiment(self): for parameter_setup in self._parameter_setups: print(parameter_setup) task = Task.create( project_name=f"{self._project_name}", task_name=self.make_task_name(parameter_setup), repo=self._repo, branch=self._branch, script=self._script, requirements_file="../requirements.txt" ) task.set_parent(Task.current_task().id) task.connect(parameter_setup) Task.enqueue(task, self._queue)
def cli(): title = 'ClearML launch - launch any codebase on remote machine running clearml-agent' print(title) parser = ArgumentParser(description=title) setup_parser(parser) # get the args args = parser.parse_args() if args.version: from ...version import __version__ print('Version {}'.format(__version__)) exit(0) create_populate = CreateAndPopulate( project_name=args.project, task_name=args.name, task_type=args.task_type, repo=args.repo or args.folder, branch=args.branch, commit=args.commit, script=args.script, working_directory=args.cwd, packages=args.packages, requirements_file=args.requirements, docker=args.docker, base_task_id=args.base_task_id, add_task_init_call=not args.skip_task_init, raise_on_missing_entries=True, verbose=True, ) # verify args create_populate.update_task_args(args.args) print('Creating new task') create_populate.create_task() # update Task args create_populate.update_task_args(args.args) print('New task created id={}'.format(create_populate.get_id())) if not args.queue: print('Warning: No queue was provided, leaving task in draft-mode.') exit(0) Task.enqueue(create_populate.task, queue_name=args.queue) print('Task id={} sent for execution on queue {}'.format( create_populate.get_id(), args.queue)) print('Execution log at: {}'.format( create_populate.task.get_output_log_web_page()))
def main(epochs): Task.init(project_name="examples", task_name="fastai v2") path = untar_data(URLs.PETS) files = get_image_files(path / "images") dls = ImageDataLoaders.from_name_func(path, files, label_func, item_tfms=Resize(224), num_workers=0) dls.show_batch() learn = cnn_learner(dls, resnet34, metrics=error_rate) learn.fine_tune(epochs) learn.show_results()
def main(): # a bit of code that creates clearml logging (formerly trains) if clearml # is available if trains_loaded and not ONLINE: task = Task.init(project_name='MineRL', task_name='kmeans pic+pic+tre 1024 + flips whatever') logger = task.get_logger() else: logger = None os.makedirs("train", exist_ok=True) cached_kmeans("train", "MineRLObtainDiamondVectorObf-v0") print("lets gooo", file=sys.stderr) train_files = absolute_file_paths('data/MineRLObtainIronPickaxeVectorObf-v0')+\ absolute_file_paths('data/MineRLObtainIronPickaxeVectorObf-v0')+\ absolute_file_paths('data/MineRLTreechopVectorObf-v0') model = Model() shuffle(train_files) loader = BatchSeqLoader(16, train_files, SEQ_LEN, model) if LOAD: model.load_state_dict(torch.load("train/model.tm")) model.cuda() train(model, "train", 150000000, loader, logger) torch.save(model.state_dict(), "train/model.tm") print("ok", file=sys.stderr) loader.kill()
def main(): # Connecting ClearML with the current process, # from here on everything is logged automatically task = Task.init(project_name="examples", task_name="HTML samples reporting") print('reporting html files into debug samples section') # Get the task logger, # You can also call Task.current_task().get_logger() from anywhere in your code. logger = task.get_logger() # report html as debug samples report_html_image(logger) report_html_graph(logger) report_html_groupby(logger) report_html_periodic_table(logger) report_html_url(logger) # force flush reports # If flush is not called, reports are flushed in the background every couple of seconds, # and at the end of the process execution logger.flush() print('We are done reporting, have a great day :)')
def main(): task = Task.init(project_name="pbl_example", task_name="tf2 mnist args", output_uri='/datasets/output') parameters = {'epochs': 5} task.connect(parameters) ds = tf.keras.datasets.mnist (x_train, y_train), (x_test, y_test) = ds.load_data() x_test, x_train = x_test.reshape((10000, 28, 28, 1)), x_train.reshape(60000, 28, 28, 1) x_test, x_train = x_test/255.0, x_train/255.0 model = tf.keras.models.Sequential([ tf.keras.layers.Conv2D(20, (5, 5), padding="SAME", activation="relu"), tf.keras.layers.MaxPool2D(2, 2, padding="SAME"), tf.keras.layers.Conv2D(40, (5, 5), padding="SAME", activation="relu"), tf.keras.layers.MaxPool2D(2, 2, padding="SAME"), tf.keras.layers.Flatten(), tf.keras.layers.Dense(512, activation="relu"), tf.keras.layers.Dense(10, activation="softmax") ]) model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) model.fit(x_train, y_train, epochs=parameters['epochs']) optimizer = tf.keras.optimizers.Adam() ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, net=model) manager = tf.train.CheckpointManager(ckpt, os.path.join(gettempdir(), 'tf_ckpts'), max_to_keep=3) save_path = manager.save() test_loss, test_acc = model.evaluate(x_test, y_test) # model.save_weights('./model') print(save_path)
def __init__(self, task: Task = None, projectName: str = None, taskName: str = None, additionalLoggingValuesDict=None): """ :param task: instances of trains.Task :param projectName: only necessary if task is not provided :param taskName: only necessary if task is not provided :param additionalLoggingValuesDict: """ if task is None: if projectName is None or taskName is None: raise ValueError( "Either the trains task or the project name and task name have to be provided" ) self.task = Task.init(project_name=projectName, task_name=taskName, reuse_last_task_id=False) else: if projectName is not None: log.warning( f"projectName parameter with value {projectName} passed even though task has been given, " f"will ignore this parameter") if taskName is not None: log.warning( f"taskName parameter with value {taskName} passed even though task has been given, " f"will ignore this parameter") self.task = task self.logger = self.task.get_logger() super().__init__( additionalLoggingValuesDict=additionalLoggingValuesDict)
def seed(self): for id in self.get_ids(): task = self.call_func( "Task.create", id, lambda id_: Task.create(project_name="MLFlow Migration", task_name=id_), self.get_run_name_by_id(id), ) self.call_func( "transmit_information", id, lambda id_: self.transmit_information(id_), id, ) self.call_func("transmit_metrics", id, lambda id_: self.transmit_metrics(id_), id) self.call_func("transmit_artifacts", id, lambda id_: self.transmit_artifacts(id_), id) task.mark_started() task.completed() output_log_web_page = task.get_output_log_web_page() url_parts = output_log_web_page.split("projects") project_id = url_parts[1].split("/")[1] self.project_link = url_parts[0] + "/projects/" + project_id self.migration_count += 1 self.pbar.update(1)
def get_pruning_results(project_name, pruning_methods, y_label): print("Retrieving tasks...") tasks = Task.get_tasks(project_name=project_name) print('Done') # logger = get_clearml_logger(project_name, 'Summary').logger print(f"Generating graph comparison for {y_label}") x_values, y_values, labels, num_funcs = compare_clearml_project( pruning_methods, project_name=project_name, tasks=tasks, x_name='dim_edges', y_name=y_label) data = seperate_labels(x_values, y_values, labels, num_funcs) for n_funcs, n_funcs_data in data.items(): plt.figure() for method, m_data in n_funcs_data.items(): x = m_data['x'] y = m_data['y'] order = np.argsort(x) x = [x[i] for i in order] y = [y[i] for i in order] plt.plot(x, y, label=method) plt.title(f"Prunning using {n_funcs} minhash functions") plt.xlabel("Edge dim") plt.ylabel("Accuracy") plt.legend() plt.show()
def get_output(command, return_command=False): save_artifact = False if command.startswith("tlt") and ( command.partition(" ")[0] != "tlt-train" and command.partition(" ")[0] != "tlt-converter"): command_prefix, _, command_args = command.partition(" ") command_prefix = shutil.which(command_prefix) command = "{} {} {}".format(sys.executable, command_prefix, command_args) elif command.startswith("ls -rlt"): # we will save as artifact if needed save_artifact = True print("=============== Running command: {}".format(command)) result = run(command, stdout=PIPE, stderr=STDOUT, universal_newlines=True, shell=True) print(result.stdout) if save_artifact: name = result.stdout.split("\n")[-2].rpartition(" ")[2] if name.endswith("tlt") or name.endswith("etlt") or name.endswith( "hdf5"): command_path = command.partition(" ")[2].rpartition(" ")[2] tlt_task = Task.current_task() tlt_task.upload_artifact( name=name, artifact_object=os.path.join(os.path.expandvars(command_path), name), ) if return_command: return result.stdout
def download_pretrained_model(model_name, ngc_model, conf_file): model_file = (get_field_from_config( conf_file, "pretrained_model_file").strip().strip('"')) if model_file: model_dir = model_file.rpartition("/")[0].rpartition("/")[0] os.makedirs(model_dir) else: model_dir = "tmp/" os.makedirs(model_dir) # Download the pretrained model from NGC download_path = None command_output = get_output( "ngc registry model download-version {} --dest {}".format( ngc_model, model_dir), return_command=True, ) for output in command_output.split("\n"): if output.startswith("Downloaded local path"): download_path = output.partition(":")[2].strip() break if download_path: tlt_task = Task.current_task() tlt_task.upload_artifact( name=model_name, artifact_object=os.path.join( os.path.expandvars("{}".format(download_path)), "{}.hdf5".format(model_name), ), )
def summary_clearml_project(project_name, x_label, seperate_legends): print("Retrieving tasks...") tasks = Task.get_tasks(project_name=project_name) print('Done') y_labels = set(tasks[0].data.hyperparams['General']) y_labels.remove(x_label) logger = get_clearml_logger(project_name, 'Summary').logger for y_label in y_labels: print(f"Generating graph comparison for {y_label}") x_values, y_values, labels = compare_clearml_project( seperate_legends, project_name=project_name, tasks=tasks, x_name=x_label, y_name=y_label) labeled_x, labeled_y, unique_labels = seperate_labels( x_values, y_values, labels) for label in unique_labels: # plt.plot(labeled_x[label], labeled_y[label], label=label) scatter2d = np.vstack((labeled_x[label], labeled_y[label])).T logger.report_scatter2d(title=y_label, series=label, iteration=0, scatter=scatter2d, xaxis=x_label, yaxis=y_label)
def get_pruning_results(project_name, pruning_methods, y_label): print("Retrieving tasks...") tasks = Task.get_tasks(project_name=project_name) print('Done') # logger = get_clearml_logger(project_name, 'Summary').logger print(f"Generating graph comparison for {y_label}") x_values, y_values, labels = compare_clearml_project( pruning_methods, project_name=project_name, tasks=tasks, x_name='keep edges', y_name=y_label) labeled_x, labeled_y, unique_labels = seperate_labels( x_values, y_values, labels) for label in unique_labels: print(f'======================= {label} =======================') x = np.array(labeled_x[label]) y = np.array(labeled_y[label]) order = np.argsort(x) x = x[order] y = y[order] for x_, y_ in zip(x, y): print(f'({x_}, {y_})') print()
def transmit_artifacts(self, id): artifacts = (self.info[id][self.artifacts] if self.artifacts in self.info[id].keys() else {}) task = self.call_func( "Task.get_task", id, lambda id_: Task.get_task(project_name="MLFlow Migration", task_name=id_), self.get_run_name_by_id(id), ) for type_, l in artifacts.items(): if type_ == "folder": for name, obj in l: task.upload_artifact(name=name, artifact_object=obj) elif type_ == "text": for name, obj in l: task.upload_artifact(name=name, artifact_object=obj) elif type_ == "dataframe": for name, obj in l: task.upload_artifact(name=name, artifact_object=obj) elif type_ == "image": for name, obj in l: task.upload_artifact(name=name, artifact_object=obj) elif type_ == "dictionary": for name, obj in l: task.upload_artifact(name=name, artifact_object=obj) elif type_ == "storage-server": for name, obj in l: task.upload_artifact(name=name, artifact_object=obj)
def compare_clearml_project(seperate_legends, project_name=None, tasks=None, x_name=None, y_name=None): tasks = Task.get_tasks( project_name=project_name) if tasks is None else tasks x_y_values = [] for task in tasks: try: x_y_values.append(( float(task.data.hyperparams['Args'][x_name].value), float(task.data.hyperparams['General'][y_name].value), task.data.hyperparams['Args'][seperate_legends].value, float( task.data.hyperparams['Args']['num_minhash_funcs'].value))) except Exception as e: print(e) pass x_values = [x for x, y, sep, sep2 in x_y_values] y_values = [y for x, y, sep, sep2 in x_y_values] labels1 = [sep for x, y, sep, sep2 in x_y_values] labels2 = [sep2 for x, y, sep, sep2 in x_y_values] return x_values, y_values, labels1, labels2
def setup_experiment_tracking(config, with_clearml, task_type="training"): from datetime import datetime assert task_type in ("training", "testing"), task_type output_path = "" if idist.get_rank() == 0: if with_clearml: from clearml import Task schema = TrainvalConfigSchema if task_type == "training" else InferenceConfigSchema task = Task.init("Pascal-VOC12 Training", config.config_filepath.stem, task_type=task_type) task.connect_configuration(config.config_filepath.as_posix()) task.upload_artifact(config.script_filepath.name, config.script_filepath.as_posix()) task.upload_artifact(config.config_filepath.name, config.config_filepath.as_posix()) task.connect(get_params(config, schema)) output_path = Path(os.environ.get("CLEARML_OUTPUT_PATH", "/tmp")) output_path = output_path / "clearml" / datetime.now().strftime("%Y%m%d-%H%M%S") else: import shutil output_path = Path(os.environ.get("OUTPUT_PATH", "/tmp/output-pascal-voc12")) output_path = output_path / task_type / config.config_filepath.stem output_path = output_path / datetime.now().strftime("%Y%m%d-%H%M%S") output_path.mkdir(parents=True, exist_ok=True) shutil.copyfile(config.script_filepath.as_posix(), output_path / config.script_filepath.name) shutil.copyfile(config.config_filepath.as_posix(), output_path / config.config_filepath.name) output_path = output_path.as_posix() return Path(idist.broadcast(output_path, src=0))
def main(): # Connecting ClearML with the current process, # from here on everything is logged automatically task = Task.init(project_name="examples", task_name="text reporting") print("reporting text logs") # report regular console print print("This is standard output test") # report stderr print("This is standard error test", file=sys.stderr) # Get the task logger, # You can also call Task.current_task().get_logger() from anywhere in your code. logger = task.get_logger() # report text based logs report_logs(logger) # report text as debug example report_debug_text(logger) # force flush reports # If flush is not called, reports are flushed in the background every couple of seconds, # and at the end of the process execution logger.flush() print("We are done reporting, have a great day :)")
def _setup_check_clearml(self, logger: ClearMLLogger, output_uri: str) -> None: try: from clearml import Task except ImportError: try: # Backwards-compatibility for legacy Trains SDK from trains import Task except ImportError: raise RuntimeError( "This contrib module requires clearml to be installed. " "You may install clearml using: \n pip install clearml \n") if logger and not isinstance(logger, ClearMLLogger): raise TypeError("logger must be an instance of ClearMLLogger") self._task = Task.current_task() if not self._task: raise RuntimeError( "ClearMLSaver requires a ClearML Task to be initialized. " "Please use the `logger` argument or call `clearml.Task.init()`." ) if output_uri: self._task.output_uri = output_uri
def my_app(cfg): # type (DictConfig) -> None task = Task.init(project_name="examples", task_name="hydra configuration") logger = task.get_logger() logger.report_text( "You can view your full hydra configuration under Configuration tab in the UI" ) print(OmegaConf.to_yaml(cfg))
def run_fixed_lambda_bbcluster(train_cluster_data, val_cluster_data, test_cluster_data, output_path, train_batch_size, eval_steps, num_epochs, warmup_frac, lambda_val, reg, beta, loss_name, use_model_device, model_name='distilbert-base-uncased', out_features=256): task = Task.init(project_name='BB Clustering', task_name='bbclustering_fixed_lambda') config_dict = {'lambda_val': lambda_val, 'reg': reg} config_dict = task.connect(config_dict) if torch.cuda.is_available(): device = torch.device('cuda') print('CUDA is available and using device: '+str(device)) else: device = torch.device('cpu') print('CUDA not available, using device: '+str(device)) ### Configure sentence transformers for training and train on the provided dataset # Use Huggingface/transformers model (like BERT, RoBERTa, XLNet, XLM-R) for mapping tokens to embeddings word_embedding_model = models.Transformer(model_name) # Apply mean pooling to get one fixed sized sentence vector pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(), pooling_mode_mean_tokens=True, pooling_mode_cls_token=False, pooling_mode_max_tokens=False) doc_dense_model = models.Dense(in_features=pooling_model.get_sentence_embedding_dimension(), out_features=out_features, activation_function=nn.Tanh()) model = CustomSentenceTransformer(modules=[word_embedding_model, pooling_model, doc_dense_model]) # model = SentenceTransformer(modules=[word_embedding_model, pooling_model]) GPUtil.showUtilization() if loss_name == 'bbspec': loss_model = BBSpectralClusterLossModel(model=model, device=device, lambda_val=config_dict.get('lambda_val', lambda_val), reg_const=config_dict.get('reg', reg), beta=beta) else: loss_model = BBClusterLossModel(model=model, device=device, lambda_val=config_dict.get('lambda_val', lambda_val), reg_const=config_dict.get('reg', reg)) # reg_loss_model = ClusterDistLossModel(model=model) train_dataloader = DataLoader(train_cluster_data, shuffle=True, batch_size=train_batch_size) GPUtil.showUtilization() # train_dataloader2 = DataLoader(train_cluster_data, shuffle=True, batch_size=train_batch_size) evaluator = ClusterEvaluator.from_input_examples(val_cluster_data, use_model_device) test_evaluator = ClusterEvaluator.from_input_examples(test_cluster_data, use_model_device) GPUtil.showUtilization() warmup_steps = int(len(train_dataloader) * num_epochs * warmup_frac) # 10% of train data print("Raw BERT embedding performance") model.to(device) evaluator(model, output_path) GPUtil.showUtilization() # Train the model model.fit(train_objectives=[(train_dataloader, loss_model)], evaluator=evaluator, test_evaluator=test_evaluator, epochs=num_epochs, evaluation_steps=eval_steps, warmup_steps=warmup_steps, output_path=output_path)
def transmit_information(self, id): parameters = self.get_params(id) general_information = self.get_general_information(id) artifact = self.get_artifact(id) tags = self.get_tags(id) task = self.call_func( "Task.get_task", id, lambda id_: Task.get_task(project_name="MLFlow Migration", task_name=id_), self.get_run_name_by_id(id), ) task_values = self.call_func( "task.export_task", id, lambda _: task.export_task(), self.get_run_name_by_id(id), ) task_values["comment"] = (tags["note.content"] if "note.content" in tags.keys() else "") task_values["hyperparams"]["Args"] = parameters task_values["started"] = general_information["started"] task_values["completed"] = general_information["completed"] task_values["script"]["branch"] = (tags["source.git.branch"] if "source.git.branch" in tags.keys() else self.branch) task_values["script"]["repository"] = (tags["source.git.repoURL"] if "source.git.repoURL" in tags.keys() else "") task_values["script"]["version_num"] = (tags["source.git.commit"] if "source.git.commit" in tags.keys() else "") task_values["script"]["entry_point"] = tags["entry_point"] task_values["script"]["working_dir"] = tags["working_dir"] if "project.env" in tags.keys(): task_values["script"]["requirements"][tags["project.env"]] = ( artifact["requirements"] if "requirements" in artifact.keys() else "") task_values["user"] = tags["user"] self.call_func( "task.update_task", id, lambda _task_values: task.update_task(_task_values), task_values, ) if len(tags["VALUETAG"].keys()) > 0: self.call_func( "task.connect_configuration", id, lambda _dict: task.connect_configuration(_dict, name="MLflow Tags"), tags["VALUETAG"], )
def _clearml_log_params(params_dict): try: from clearml import Task except ImportError: # Backwards-compatibility for legacy Trains SDK from trains import Task task = Task.current_task() task.connect(params_dict)
def _clearml_log_artifact(fp): try: from clearml import Task except ImportError: # Backwards-compatibility for legacy Trains SDK from trains import Task task = Task.current_task() task.upload_artifact(Path(fp).name, fp)