def get_dataset_obj(): import dtlpy as dl dl.login_token('token here') dl.setenv('dev') project = dl.projects.get(project_name='buffs_project') dataset_obj = project.datasets.get('my_data') return dataset_obj
def create_trainer(): dl.setenv('dev') plugin_name = 'trainer' project_name = 'ZazuProject' project = dl.projects.get(project_name=project_name) dl.projects.checkout(project_name) dl.plugins.checkout(plugin_name) ############### # push plugin # ############### plugin = project.plugins.push(plugin_name=plugin_name, src_path=os.getcwd()) plugin = project.plugins.get(plugin_name=plugin_name) ##################### # create deployment # ##################### deployment = plugin.deployments.deploy(deployment_name=plugin.name, plugin=plugin, runtime={'gpu': False, 'numReplicas': 1, 'concurrency': 2, }) deployment = plugin.deployments.get(deployment_name=plugin.name) ############## # for update # ############## deployment.pluginRevision = plugin.version deployment.update()
def create_tuner(): dl.setenv('prod') plugin_name = 'tuner' project_name = 'ZazuProject' project = dl.projects.get(project_name=project_name) ############### # push plugin # ############### plugin = project.plugins.push(plugin_name=plugin_name, src_path=os.getcwd(), inputs=[{"type": "Json", "name": "configs"}]) plugin = project.plugins.get(plugin_name=plugin_name) ##################### # create deployment # ##################### deployment = plugin.deployments.deploy(deployment_name=plugin.name, plugin=plugin, runtime={'gpu': False, 'numReplicas': 1, 'concurrency': 2, 'image': 'gcr.io/viewo-g/piper/agent/runner/gpu/main/zazu:latest' }, bot=None) deployment = plugin.deployments.get(deployment_name=plugin.name) ############## # for update # ############## deployment.pluginRevision = plugin.version deployment.update() #################### # invoke a session # #################### inputs = [dl.PluginInput(type='Json', value={'annotation_type': 'binary', 'confidence_th': 0.50, 'output_action': 'annotations'}, name='config')] session = deployment.sessions.create(deployment_id=deployment.id, session_input=inputs) # check updates for i in range(5): _session = deployment.sessions.get(session_id=session.id) pprint.pprint(_session.status[-1]) time.sleep(2)
def _push_and_deploy_plugin(self): import dtlpy as dl dl.setenv('dev') project = dl.projects.get( project_id="fcdd792b-5146-4c62-8b27-029564f1b74e") plugin = project.plugins.push(src_path='/Users/noam/zazuML') self.deployment = plugin.deployments.deploy( deployment_name='thisdeployment', plugin=plugin, config={ 'project_id': project.id, 'plugin_name': plugin.name }, runtime={ 'gpu': True, 'numReplicas': 1, 'concurrency': 1, 'image': 'gcr.io/viewo-g/piper/custom/zazuim:1.0' })
def add_dataloop_remote_annotations(self, project_name, dataset_name, filter_value, model_name): dlp.setenv('prod') project = dlp.projects.get(project_name) dataset = project.datasets.get(dataset_name) filters = dlp.Filters() filters.add(field='filename', values=filter_value) pages = dataset.items.list(filters=filters) self.models_alowed_ls = [model_name, 'gt'] i_item = 0 pool = ThreadPool(processes=32) for page in pages: for item in page: if item.filename.startswith('/.dataloop'): continue pool.apply_async(self._collect_annotations, kwds={'w_item': item}) i_item += 1 pool.close() pool.join()
# for local import from tests.env_from_git_branch import get_env_from_git_branch except ImportError: # for remote import from env_from_git_branch import get_env_from_git_branch try: # for local import from tests.env_from_git_branch import get_env_from_git_branch except ImportError: # for remote import from env_from_git_branch import get_env_from_git_branch logging.basicConfig(level='DEBUG') dl.setenv(get_env_from_git_branch()) # check token payload = jwt.decode(dl.token(), algorithms=['HS256'], verify=False) if payload['email'] not in [ '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**' ]: assert False, 'Cannot run test on user: "******". only test users'.format( payload['email']) TEST_DIR = os.path.dirname(os.path.realpath(__file__)) ASSETS_PATH = os.path.join(TEST_DIR, 'assets') image_path = os.path.join(ASSETS_PATH, '0000000162.jpg') annotations_path = os.path.join(ASSETS_PATH, 'annotations_new.json') project = dl.projects.create(
def maybe_login(env): try: dl.setenv(env) except: dl.login() dl.setenv(env)
import dtlpy as dl import os path = '/home/noam/data' dl.login_token('insert_token_here') dl.setenv('dev') project = dl.projects.get(project_id='buffs_project') dataset_obj = project.datasets.get('my_data') # dataset_obj.items.upload(local_path='/Users/noam/tiny_coco', remote_path='') dataset_obj.items.download(local_path=path) os.rename(os.path.join(path, 'items', 'tiny_coco'), os.path.join(path, 'tiny_coco'))
def maybe_login(): try: dl.setenv('dev') except: dl.login() dl.setenv('dev')
def __init__(self, configs, time, test_dataset_id, query): logger.info('dtlpy version: ' + str(dl.__version__)) logger.info('dtlpy info: ' + str(dl.info())) time = int(time) dl.setenv('prod') configs = json.loads(configs) query = json.loads(query) self.configs_input = dl.FunctionIO(type='Json', name='configs', value=configs) self.service = dl.services.get('zazu') project_name = configs['dataloop']['project'] self.project = dl.projects.get(project_name) test_dataset = self.project.datasets.get(dataset_id=test_dataset_id) maybe_download_pred_data(dataset_obj=test_dataset, val_query=query) # add gt annotations filters = dl.Filters() filters.custom_filter = query dataset_name = test_dataset.name path_to_dataset = os.path.join(os.getcwd(), dataset_name) # only download if doesnt exist if not os.path.exists(path_to_dataset): download_and_organize(path_to_dataset=path_to_dataset, dataset_obj=test_dataset, filters=filters) json_file_path = os.path.join(path_to_dataset, 'json') self.model_obj = self.project.models.get(model_name='retinanet') self.adapter = self.model_obj.build(local_path=os.getcwd()) logger.info('model built') while 1: self.compute = precision_recall_compute() self.compute.add_dataloop_local_annotations(json_file_path) logger.info("running new execution") execution_obj = self.service.execute(function_name='search', execution_input=[self.configs_input], project_id='72bb623f-517f-472b-ad69-104fed8ee94a') while execution_obj.latest_status['status'] != 'success': sleep(5) execution_obj = dl.executions.get(execution_id=execution_obj.id) if execution_obj.latest_status['status'] == 'failed': raise Exception("plugin execution failed") logger.info("execution object status is successful") self.project.artifacts.download(package_name='zazuml', execution_id=execution_obj.id, local_path=os.getcwd()) logs_file_name = 'timer_logs_' + str(execution_obj.id) + '.conf' graph_file_name = 'precision_recall_' + str(execution_obj.id) + '.png' self.cycle_logger = init_logging(__name__, filename=logs_file_name) logger.info('artifact download finished') logger.info(str(os.listdir('.'))) # load new checkpoint and change to unique name new_checkpoint_name = 'checkpoint_' + str(execution_obj.id) + '.pt' logger.info(str(os.listdir('.'))) os.rename('checkpoint0.pt', new_checkpoint_name) new_model_name = new_checkpoint_name[:-3] logger.info(str(os.listdir('.'))) new_checkpoint = torch.load(new_checkpoint_name, map_location=torch.device('cpu')) # self.model_obj = self.project.models.get(model_name=new_checkpoint['model_specs']['name']) # self.adapter = self.model_obj.build(local_path=os.getcwd()) # logger.info('model built') self.new_home_path = new_checkpoint['model_specs']['data']['home_path'] self._compute_predictions(checkpoint_path=new_checkpoint_name, model_name=new_model_name) if len(self.compute.by_model_name.keys()) < 2: # if the model cant predict anything then just skip it logger.info('''model couldn't make any predictions, trying to train again''') continue # if previous best checkpoint doesnt exist there must not be a service, launch prediction service with new # new_checkpoint and create trigger if 'check0' not in [checkp.name for checkp in self.model_obj.checkpoints.list()]: logger.info('there is no check0, will add upload new checkpoint as check0 and ' 'deploy prediction service') new_checkpoint_obj = self.model_obj.checkpoints.upload(checkpoint_name='check0', local_path=new_checkpoint_name) logger.info('uploaded this checkpoint as the new check0 : ' + new_checkpoint_name[:-3]) self._maybe_launch_predict(new_checkpoint_obj) continue logger.info('i guess check0 does exist') best_checkpoint = self.model_obj.checkpoints.get('check0') check0_path = best_checkpoint.download(local_path=os.getcwd()) logger.info('downloading best checkpoint') logger.info(str(os.listdir('.'))) logger.info('check0 path is: ' + str(check0_path)) self._compute_predictions(checkpoint_path=check0_path, model_name=best_checkpoint.name) # compute metrics new_checkpoint_mAP = self.compute.get_metric(model_name=new_model_name, precision_to_recall_ratio=1.) best_checkpoint_mAP = self.compute.get_metric(model_name=best_checkpoint.name, precision_to_recall_ratio=1.) logger.info('best checkpoint: ' + str(best_checkpoint_mAP)) logger.info('new checkpoint: ' + str(new_checkpoint_mAP)) # if new checkpoint performs better switch out prediction if new_checkpoint_mAP > best_checkpoint_mAP: logger.info('new checkpoint is better') logger.info('uploading old best checkpoint under new name') self.model_obj.checkpoints.upload(checkpoint_name='checkpoint_' + check0_path.split('_')[-1][:-3], local_path=check0_path) logger.info('deleting old best checkpoint') best_checkpoint.delete() logger.info('uploading new best checkpoint as check0') new_best_checkpoint_obj = self.model_obj.checkpoints.upload(checkpoint_name='check0', local_path=new_checkpoint_name) if 'predict' not in [s.name for s in dl.services.list()]: self._maybe_launch_predict(new_best_checkpoint_obj) else: self._update_predict_service(new_best_checkpoint_obj) logger.info('switched with new checkpoint') self.compute.save_plot_metrics(save_path=graph_file_name) self.project.artifacts.upload(filepath=logs_file_name, package_name='zazuml', execution_id=execution_obj.id) self.project.artifacts.upload(filepath=graph_file_name, package_name='zazuml', execution_id=execution_obj.id) logger.info('waiting ' + str(time) + ' seconds for next execution . . . .') sleep(time)