def get_dataset_obj():
import dtlpy as dl
dl.login_token('token here')
dl.setenv('dev')
project = dl.projects.get(project_name='buffs_project')
dataset_obj = project.datasets.get('my_data')
return dataset_obj
def create_trainer():
dl.setenv('dev')
plugin_name = 'trainer'
project_name = 'ZazuProject'
project = dl.projects.get(project_name=project_name)
dl.projects.checkout(project_name)
dl.plugins.checkout(plugin_name)
###############
# push plugin #
###############
plugin = project.plugins.push(plugin_name=plugin_name,
src_path=os.getcwd())
plugin = project.plugins.get(plugin_name=plugin_name)
#####################
# create deployment #
#####################
deployment = plugin.deployments.deploy(deployment_name=plugin.name,
plugin=plugin,
runtime={'gpu': False,
'numReplicas': 1,
'concurrency': 2,
})
deployment = plugin.deployments.get(deployment_name=plugin.name)
##############
# for update #
##############
deployment.pluginRevision = plugin.version
deployment.update()
def create_tuner():
dl.setenv('prod')
plugin_name = 'tuner'
project_name = 'ZazuProject'
project = dl.projects.get(project_name=project_name)
###############
# push plugin #
###############
plugin = project.plugins.push(plugin_name=plugin_name,
src_path=os.getcwd(),
inputs=[{"type": "Json",
"name": "configs"}])
plugin = project.plugins.get(plugin_name=plugin_name)
#####################
# create deployment #
#####################
deployment = plugin.deployments.deploy(deployment_name=plugin.name,
plugin=plugin,
runtime={'gpu': False,
'numReplicas': 1,
'concurrency': 2,
'image': 'gcr.io/viewo-g/piper/agent/runner/gpu/main/zazu:latest'
},
bot=None)
deployment = plugin.deployments.get(deployment_name=plugin.name)
##############
# for update #
##############
deployment.pluginRevision = plugin.version
deployment.update()
####################
# invoke a session #
####################
inputs = [dl.PluginInput(type='Json',
value={'annotation_type': 'binary',
'confidence_th': 0.50,
'output_action': 'annotations'},
name='config')]
session = deployment.sessions.create(deployment_id=deployment.id,
session_input=inputs)
# check updates
for i in range(5):
_session = deployment.sessions.get(session_id=session.id)
pprint.pprint(_session.status[-1])
time.sleep(2)
def _push_and_deploy_plugin(self):
import dtlpy as dl
dl.setenv('dev')
project = dl.projects.get(
project_id="fcdd792b-5146-4c62-8b27-029564f1b74e")
plugin = project.plugins.push(src_path='/Users/noam/zazuML')
self.deployment = plugin.deployments.deploy(
deployment_name='thisdeployment',
plugin=plugin,
config={
'project_id': project.id,
'plugin_name': plugin.name
},
runtime={
'gpu': True,
'numReplicas': 1,
'concurrency': 1,
'image': 'gcr.io/viewo-g/piper/custom/zazuim:1.0'
})
def add_dataloop_remote_annotations(self, project_name, dataset_name,
filter_value, model_name):
dlp.setenv('prod')
project = dlp.projects.get(project_name)
dataset = project.datasets.get(dataset_name)
filters = dlp.Filters()
filters.add(field='filename', values=filter_value)
pages = dataset.items.list(filters=filters)
self.models_alowed_ls = [model_name, 'gt']
i_item = 0
pool = ThreadPool(processes=32)
for page in pages:
for item in page:
if item.filename.startswith('/.dataloop'):
continue
pool.apply_async(self._collect_annotations,
kwds={'w_item': item})
i_item += 1
pool.close()
pool.join()
# for local import
from tests.env_from_git_branch import get_env_from_git_branch
except ImportError:
# for remote import
from env_from_git_branch import get_env_from_git_branch
try:
# for local import
from tests.env_from_git_branch import get_env_from_git_branch
except ImportError:
# for remote import
from env_from_git_branch import get_env_from_git_branch
logging.basicConfig(level='DEBUG')
dl.setenv(get_env_from_git_branch())
# check token
payload = jwt.decode(dl.token(), algorithms=['HS256'], verify=False)
if payload['email'] not in [
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**'
]:
assert False, 'Cannot run test on user: "******". only test users'.format(
payload['email'])
TEST_DIR = os.path.dirname(os.path.realpath(__file__))
ASSETS_PATH = os.path.join(TEST_DIR, 'assets')
image_path = os.path.join(ASSETS_PATH, '0000000162.jpg')
annotations_path = os.path.join(ASSETS_PATH, 'annotations_new.json')
project = dl.projects.create(
def maybe_login(env):
try:
dl.setenv(env)
except:
dl.login()
dl.setenv(env)
import dtlpy as dl
import os
path = '/home/noam/data'
dl.login_token('insert_token_here')
dl.setenv('dev')
project = dl.projects.get(project_id='buffs_project')
dataset_obj = project.datasets.get('my_data')
# dataset_obj.items.upload(local_path='/Users/noam/tiny_coco', remote_path='')
dataset_obj.items.download(local_path=path)
os.rename(os.path.join(path, 'items', 'tiny_coco'),
os.path.join(path, 'tiny_coco'))
def maybe_login():
try:
dl.setenv('dev')
except:
dl.login()
dl.setenv('dev')
def __init__(self, configs, time, test_dataset_id, query):
logger.info('dtlpy version: ' + str(dl.__version__))
logger.info('dtlpy info: ' + str(dl.info()))
time = int(time)
dl.setenv('prod')
configs = json.loads(configs)
query = json.loads(query)
self.configs_input = dl.FunctionIO(type='Json', name='configs', value=configs)
self.service = dl.services.get('zazu')
project_name = configs['dataloop']['project']
self.project = dl.projects.get(project_name)
test_dataset = self.project.datasets.get(dataset_id=test_dataset_id)
maybe_download_pred_data(dataset_obj=test_dataset, val_query=query)
# add gt annotations
filters = dl.Filters()
filters.custom_filter = query
dataset_name = test_dataset.name
path_to_dataset = os.path.join(os.getcwd(), dataset_name)
# only download if doesnt exist
if not os.path.exists(path_to_dataset):
download_and_organize(path_to_dataset=path_to_dataset, dataset_obj=test_dataset, filters=filters)
json_file_path = os.path.join(path_to_dataset, 'json')
self.model_obj = self.project.models.get(model_name='retinanet')
self.adapter = self.model_obj.build(local_path=os.getcwd())
logger.info('model built')
while 1:
self.compute = precision_recall_compute()
self.compute.add_dataloop_local_annotations(json_file_path)
logger.info("running new execution")
execution_obj = self.service.execute(function_name='search', execution_input=[self.configs_input],
project_id='72bb623f-517f-472b-ad69-104fed8ee94a')
while execution_obj.latest_status['status'] != 'success':
sleep(5)
execution_obj = dl.executions.get(execution_id=execution_obj.id)
if execution_obj.latest_status['status'] == 'failed':
raise Exception("plugin execution failed")
logger.info("execution object status is successful")
self.project.artifacts.download(package_name='zazuml',
execution_id=execution_obj.id,
local_path=os.getcwd())
logs_file_name = 'timer_logs_' + str(execution_obj.id) + '.conf'
graph_file_name = 'precision_recall_' + str(execution_obj.id) + '.png'
self.cycle_logger = init_logging(__name__, filename=logs_file_name)
logger.info('artifact download finished')
logger.info(str(os.listdir('.')))
# load new checkpoint and change to unique name
new_checkpoint_name = 'checkpoint_' + str(execution_obj.id) + '.pt'
logger.info(str(os.listdir('.')))
os.rename('checkpoint0.pt', new_checkpoint_name)
new_model_name = new_checkpoint_name[:-3]
logger.info(str(os.listdir('.')))
new_checkpoint = torch.load(new_checkpoint_name, map_location=torch.device('cpu'))
# self.model_obj = self.project.models.get(model_name=new_checkpoint['model_specs']['name'])
# self.adapter = self.model_obj.build(local_path=os.getcwd())
# logger.info('model built')
self.new_home_path = new_checkpoint['model_specs']['data']['home_path']
self._compute_predictions(checkpoint_path=new_checkpoint_name,
model_name=new_model_name)
if len(self.compute.by_model_name.keys()) < 2:
# if the model cant predict anything then just skip it
logger.info('''model couldn't make any predictions, trying to train again''')
continue
# if previous best checkpoint doesnt exist there must not be a service, launch prediction service with new
# new_checkpoint and create trigger
if 'check0' not in [checkp.name for checkp in self.model_obj.checkpoints.list()]:
logger.info('there is no check0, will add upload new checkpoint as check0 and '
'deploy prediction service')
new_checkpoint_obj = self.model_obj.checkpoints.upload(checkpoint_name='check0',
local_path=new_checkpoint_name)
logger.info('uploaded this checkpoint as the new check0 : ' + new_checkpoint_name[:-3])
self._maybe_launch_predict(new_checkpoint_obj)
continue
logger.info('i guess check0 does exist')
best_checkpoint = self.model_obj.checkpoints.get('check0')
check0_path = best_checkpoint.download(local_path=os.getcwd())
logger.info('downloading best checkpoint')
logger.info(str(os.listdir('.')))
logger.info('check0 path is: ' + str(check0_path))
self._compute_predictions(checkpoint_path=check0_path, model_name=best_checkpoint.name)
# compute metrics
new_checkpoint_mAP = self.compute.get_metric(model_name=new_model_name, precision_to_recall_ratio=1.)
best_checkpoint_mAP = self.compute.get_metric(model_name=best_checkpoint.name, precision_to_recall_ratio=1.)
logger.info('best checkpoint: ' + str(best_checkpoint_mAP))
logger.info('new checkpoint: ' + str(new_checkpoint_mAP))
# if new checkpoint performs better switch out prediction
if new_checkpoint_mAP > best_checkpoint_mAP:
logger.info('new checkpoint is better')
logger.info('uploading old best checkpoint under new name')
self.model_obj.checkpoints.upload(checkpoint_name='checkpoint_' + check0_path.split('_')[-1][:-3],
local_path=check0_path)
logger.info('deleting old best checkpoint')
best_checkpoint.delete()
logger.info('uploading new best checkpoint as check0')
new_best_checkpoint_obj = self.model_obj.checkpoints.upload(checkpoint_name='check0',
local_path=new_checkpoint_name)
if 'predict' not in [s.name for s in dl.services.list()]:
self._maybe_launch_predict(new_best_checkpoint_obj)
else:
self._update_predict_service(new_best_checkpoint_obj)
logger.info('switched with new checkpoint')
self.compute.save_plot_metrics(save_path=graph_file_name)
self.project.artifacts.upload(filepath=logs_file_name,
package_name='zazuml',
execution_id=execution_obj.id)
self.project.artifacts.upload(filepath=graph_file_name,
package_name='zazuml',
execution_id=execution_obj.id)
logger.info('waiting ' + str(time) + ' seconds for next execution . . . .')
sleep(time)
