def __init__(
self,
upload_experiment,
api_key,
output_dir,
force_reupload,
mlflow_store_uri,
answer,
email,
):
# type: (bool, str, str, bool, str, Optional[bool], str) -> None
self.answer = answer
self.email = email
self.config = get_config()
# Display the start banner
LOGGER.info(BANNER)
# get_api_key
self.api_key, self.token = self.get_api_key_or_login(api_key)
# May not need this always?
self.api_client = API(self.api_key, cache=False)
self.workspace = self.config["comet.workspace"]
# Get a workspace
if not self.workspace:
details = self.api_client.get_account_details()
self.workspace = details["defaultWorkspaceName"]
if output_dir is None:
output_dir = tempfile.mkdtemp()
# MLFlow conversion
self.store = _get_store(mlflow_store_uri)
try:
self.model_registry_store = get_model_registry_store(
mlflow_store_uri)
except UnsupportedModelRegistryStoreURIException:
self.model_registry_store = None
# Most of list_experiments returns a list anyway
self.mlflow_experiments = list(self.store.list_experiments())
self.len_experiments = len(
self.mlflow_experiments) # We start counting at 0
self.summary = {
"experiments": 0,
"runs": 0,
"tags": 0,
"params": 0,
"metrics": 0,
"artifacts": 0,
}
self.upload_experiment = upload_experiment
self.output_dir = output_dir
self.force_reupload = force_reupload
self.mlflow_store_uri = mlflow_store_uri
def list_models(args):
api = API()
model_names = api.get_registry_model_names(args.workspace)
if len(model_names) == 0:
LOGGER.info("This workspace has no registered models")
else:
LOGGER.info("This workspace has the following registered models:")
for i, name in enumerate(model_names):
LOGGER.info(" %d. %s" % (i + 1, name))
def download_model(args):
api = API()
api.download_registry_model(
args.workspace,
args.model_name,
expand=True,
output_path=args.output,
stage=args.model_stage,
version=args.model_version,
)
def _download_data(comet_api: comet_ml.API, experiments: APIExperiments, p1_name: str, p2_name: str, metrics: List[str],
parameters: List[str]) \
-> Dict[str, List[Tuple[float, float, float]]]:
targets_data = defaultdict(list)
list2float = lambda l: float(l[0])
for experiment in experiments:
p1_value = list2float(comet_api.get_experiment_parameters(experiment.key, p1_name))
p2_value = list2float(comet_api.get_experiment_parameters(experiment.key, p2_name))
for parameter in parameters:
target_data = list2float(comet_api.get_experiment_parameters(experiment.key, parameter))
targets_data[parameter].append((p1_value, p2_value, target_data))
for metric in metrics:
target_data = list2float(comet_api.get_experiment_metrics(experiment.key, metric))
targets_data[metric].append((p1_value, p2_value, target_data))
return targets_data
def connect_to_comet(comet_apikey,comet_restapikey,comet_username,comet_project):
if os.path.isfile("settings.json"):
with open("settings.json") as f:
keys = json.load(f)
comet_apikey = keys.get("apikey")
comet_username = keys.get("username")
comet_restapikey = keys.get("restapikey")
print("COMET_REST_API_KEY=%s" %(comet_restapikey))
with open('.env', 'w') as writer:
writer.write("COMET_API_KEY=%s\n" %(comet_apikey))
writer.write("COMET_REST_API_KEY=%s\n" %(comet_restapikey))
comet_api = API()
return comet_api, comet_username, comet_project
parser.add_argument('-patience', default=300, type=int)
parser.add_argument('-restartperiod', default=2000, type=int)
parser.add_argument('-reducemethod', default='average', type=str) # softmax, average
parser.add_argument('-objective', default='xent', type=str) # cw, xent
parser.add_argument('-trajectory', default='clean', type=str) # clean or poison
args = parser.parse_args()
# Node info, especially important to check if the right ressources were allocated by SLURM
# args.gpu = set_available_gpus(args)
args.gpu = [1]
ncpu = multiprocessing.cpu_count()
print('==> Rank {}/{}, localrank {}, host {}, GPU {}/{}, nCPUs {}'.format(rank, nproc,
localrank, socket.gethostname(), localrank % len(args.gpu), len(args.gpu), ncpu))
# comet initialization
api = API(api_key="your_api_key")
# api = comet_ml.api.API()
weightapi = API(api_key="your_api_key")
experiment = Dummy()
if rank == 0 and not args.nocomet:
experiment = Experiment(project_name='metapoison2', auto_param_logging=False, auto_metric_logging=False)
experiment.add_tag(args.tag)
experiment.log_parameters(vars(args))
experiment.log_parameter('nproc', nproc)
experiment.log_parameter('nmeta', args.nreplay * nproc)
print(experiment.get_key())
def craft():
def increment_or_reset_epoch():
# restore weights to initial (with coldstart) after trainstep reaches k
from comet_ml import Experiment
from comet_ml import API, Experiment
from cometml_api import api as cometapi
import numpy as np
from scipy.io import savemat
from scipy import stats
import os
from os.path import join, basename
# initialize comet and comet api callers
experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k",
parse_args=False,
project_name='images',
workspace="wronnyhuang")
cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl')
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
experiments = api.get('wronnyhuang/swissroll-rollout-3')
# desired pretrain_dir
# pretrain_dir = 'poison_repro2_697197'
# pretrain_dir = 'poison_repro2_8463'
# pretrain_dir = 'repeat-60152396'
pretrain_dir = 'repeat-75870743'
home = os.environ['HOME']
logdir = join(home, 'ckpt/swissroll/analyze_rollout')
os.makedirs(logdir, exist_ok=True)
xents, accs = [], []
## loop through all experiments in this comet project
for i, expt in enumerate(experiments):
where:
* level: detail, maximum, or minimum
* section: metric, param, log_other, etc.
* name: name of metric, param, etc.
$ comet offline --csv *.zip
Use --level, --section, or --name to filter the rows.
"""
import sys
import argparse
from comet_ml import API
parser = argparse.ArgumentParser(description="Set seldon-core version")
parser.add_argument("--workspace", required=True)
parser.add_argument("--registry_name", required=True)
parser.add_argument("--model_version", required=True)
opts = parser.parse_args()
api = API()
api.download_registry_model(opts.workspace,
opts.registry_name,
opts.model_version,
output_path="model",
expand=True)
from _warnings import warn
from comet_ml import API
import numpy as np
import matplotlib.pyplot as plt
api = API()
exp_ids = {
"SEVN-Train-AllObs-Shaped-v1-s0-10p":
"mweiss17/navi-corl-2019/008004e9c9a940e088437e4ddeab9eb4",
"SEVN-Train-AllObs-Shaped-v1-s1-10p":
"mweiss17/navi-corl-2019/dbd36b752d6a4703904161d95ee09868",
"SEVN-Train-AllObs-Shaped-v1-s2-10p":
"mweiss17/navi-corl-2019/84dbd53a36db4b39a7afc9acc66609a0",
"SEVN-Train-AllObs-Shaped-v1-s3-10p":
"mweiss17/navi-corl-2019/12f4aec90e284d1188bbe6307bdc33bd",
"SEVN-Train-AllObs-Shaped-v1-s4-10p":
"mweiss17/navi-corl-2019/bb6af29d7336411b92e31f750b5087bb",
}
# oracle_random_ids = {
# "Hyrule-Mini-Random-v1": "mweiss17/navi-corl-2019/80b8b611c84242ffa61d08cc3364ba4b",
# "Hyrule-Mini-Oracle-v1": "mweiss17/navi-corl-2019/c212813764de4a66994912dae21a8628",
# }
plot_info = {
"SEVN-Train-AllObs-Shaped-v1": {
'color': '#22a784',
'plot_name': 'AllObs'
},
from meta import Meta
from data import *
from utils import *
import pickle
import json
from time import time, sleep
from socket import gethostname
from collections import defaultdict
from mpi4py import MPI
# initialize mpi
mpi = MPI.COMM_WORLD
nproc, rank = mpi.Get_size(), mpi.Get_rank()
localrank = int(os.environ.get('OMPI_COMM_WORLD_LOCAL_RANK', rank))
api = API()
weightapi = API()
cometconfig = read_comet_config()
parser, attrs = get_parser(True)
for exclude in ['gpu']: attrs.remove(exclude)
args = parser.parse_args()
args.craftkey = args.uid if len(args.uid) == 32 else uid2craftkey(args.uid, args.craftproj)
craftexpt = api.get_experiment(cometconfig["workspace"], args.craftproj, args.craftkey)
copy_to_args_from_experiment(args, craftexpt, attrs)
# if args.Xnvictimepoch > 0: args.nvictimepoch = args.Xnvictimepoch
# if args.Xntrial > 0: args.ntrial = args.Xntrial
if args.Xvictimproj is not None: args.victimproj = args.Xvictimproj
if args.Xtag is not None: args.tag = args.Xtag
if args.Xweightdecay: args.weightdecay = True
if args.Xaugment: args.augment = True
if args.Xbatchsize:
dw1 = evaluator.get_random_dir()
dw2 = evaluator.get_random_dir()
# span
clin = args.span / 2 * np.linspace(-1, 1, args.res)
cc1, cc2 = np.meshgrid(clin, clin)
cfeed = list(zip(cc1.ravel(), cc2.ravel()))
# loop over all points along surface direction
weights = evaluator.get_weights()
# get the already-logged data
print('Gathering already-evaluated indices')
xent = {}
acc = {}
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
allexperiments = api.get('wronnyhuang/landscape2d')
for expt in allexperiments:
if exptname != api.get_experiment_other(expt, 'Name')[0]: continue
raw = api.get_experiment_metrics_raw(expt)
for r in raw:
if r['metricName'] == 'xent':
xent[r['step']] = r['metricValue']
elif r['metricName'] == 'acc':
acc[r['step']] = r['metricValue']
for idx, (c1, c2) in enumerate(cfeed):
if np.mod(idx, args.npart) != args.part: continue
if idx in xent and idx in acc:
print('skipping idx ' + str(idx))
parser.add_argument('-reducemethod', default='average',
type=str) # softmax, average
parser.add_argument('-objective', default='xent', type=str) # cw, xent
parser.add_argument('-trajectory', default='clean',
type=str) # clean or poison
args = parser.parse_args()
# Node info, especially important to check if the right ressources were allocated by SLURM
args.gpu = set_available_gpus(args)
ncpu = multiprocessing.cpu_count()
print('==> Rank {}/{}, localrank {}, host {}, GPU {}/{}, nCPUs {}'.format(
rank, nproc, localrank, socket.gethostname(), localrank % len(args.gpu),
len(args.gpu), ncpu))
# comet initialization
api = API()
weightapi = API()
experiment = Dummy()
if rank == 0 and not args.nocomet:
experiment = Experiment(project_name='metapoison2',
auto_param_logging=False,
auto_metric_logging=False)
experiment.add_tag(args.tag)
experiment.log_parameters(vars(args))
experiment.log_parameter('nproc', nproc)
experiment.log_parameter('nmeta', args.nreplay * nproc)
print(experiment.get_key())
def craft():
def increment_or_reset_epoch():
# overwrite
parser.add_argument('-Xweightdecay', action='store_true')
parser.add_argument('-Xaugment', action='store_true')
parser.add_argument('-Xoptimizer', action='store_true')
parser.add_argument('-Xbatchsize', action='store_true')
parser.add_argument('-Xlrnrate', action='store_true')
parser.add_argument('-Xnet', action='store_true')
parser.add_argument('-neval', default=None, type=int)
# unused, but needed for proper meta construction
parser.add_argument('-eps', default=16, type=float)
parser.add_argument('-nadapt', default=1, type=int)
parser.add_argument('-reducemethod', default='average',
type=str) # softmax, average
parser.add_argument('-objective', default='xent', type=str) # cw, xent
args = parser.parse_args()
api = API()
attrs = [
'lrnrate', 'batchsize', 'nbatch', 'npoison', 'ntarget', 'pretrain',
'warmupperiod', 'optimizer', 'weightdecay', 'augment', 'net',
'multiclasspoison', 'targetclass', 'poisonclass', 'ytargetadv'
]
copy_to_args_from_experiment(args, args.key, api, attrs)
if args.tag == 'multi': args.multiclasspoison = True
if args.Xweightdecay: args.weightdecay = True
if args.Xaugment: args.augment = True
if args.Xbatchsize: args.batchsize = 250
if args.Xlrnrate: args.lrnrate = .2
if args.Xoptimizer: args.optimizer = 'sgd'
if args.Xnet:
args.net = 'ConvNet'
args.lrnrate = .05
def api(self):
# only use API if requested
if self._api is None:
self._api = API()
return self._api
import numpy as np
from comet_ml import API
from matplotlib.pyplot import plot, imshow, colorbar, show, axis, hist, subplot, xlabel, ylabel, title, legend, savefig, figure, close, suptitle, tight_layout, xlim, ylim
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
experiments = api.get('wronnyhuang/ranksweep-1')
ranks = []
errors = []
for experiment in experiments:
# get the rank
rank = int(api.get_experiment_parameters(experiment, 'rank')[0])
ranks.append(rank)
# get the minimum test error
metrics = {
m.pop('name'): m
for m in api.get_experiment_metrics(experiment)
}
error = float(metrics['test_tranfrob']['valueMin'])
errors.append(error)
plot(ranks, errors, '.', markersize=8)
title('rank vs error')
xlabel('rank')
ylabel('transition error (frob norm)')
show()
def craft():
# comet initialization
weightapi = API()
experiment = Dummy()
if rank == 0:
experiment = Experiment(project_name=args.craftproj,
auto_param_logging=False,
auto_metric_logging=False,
parse_args=False)
comet_ml.config.experiment = None
experiment.add_tag(args.tag)
experiment.log_parameters(vars(args))
experiment.log_other('command', print_command_and_args(args))
print(
f'Logged uid: {experiment.get_parameter("uid")}, expt key: {experiment.get_key()}'
)
def comet_log_poison():
poisoninputs = sess.run(meta.poisoninputs,
{meta.poisonmask: [True] * args.npoison})
# log poison assets for victim eval
fname = str(time()).replace('.', '')
with open(fname, 'wb') as f:
pickle.dump(poisoninputs, f)
experiment.log_asset(fname,
file_name='poisoninputs-{}'.format(craftstep),
step=craftstep)
os.remove(fname)
# log poison figures
npoison_to_display = 10
for i in np.linspace(0,
args.npoison - 1,
npoison_to_display,
dtype=int):
imagesc(poisoninputs[i],
title='poison-{}'.format(i),
experiment=experiment,
step=craftstep,
scale=[0, 255])
imagesc(poisoninputs[i] - xbase[i],
title='perturb-{}'.format(i),
experiment=experiment,
step=craftstep,
scale=127.5)
for i in range(len(xtarget)):
imagesc(xtarget[i],
title='target-{}'.format(i),
experiment=experiment,
step=craftstep,
scale=[0, 255])
def restart_poison():
perts = np.random.uniform(-args.eps, args.eps, xbase.shape)
perts = np.clip(xbase + perts, 0, 255) - xbase
mpi.Bcast(perts, root=0)
meta.restart_poison(perts, sess)
def log_epoch_results(resMs, resLs, craftstep):
resMgather = mpi.gather(resMs, root=0)
resLgather = mpi.gather(resLs, root=0)
if rank == 0:
resMgather = sum(resMgather, []) # flattens the list of lists
resLgather = sum(resLgather, [])
resM, resL = avg_n_dicts(resMgather), avg_n_dicts(resLgather)
experiment.log_metrics(resM, step=craftstep)
experiment.log_metrics(resL, step=craftstep)
[
experiment.log_metric(f'xent{i}', r['xent'], step=craftstep)
for i, r in enumerate(resLgather)
]
# [experiment.log_metric(f'epoch{i}', r['epoch'], step=craftstep) for i, r in enumerate(resLgather)]
print(' | '.join(['craftstep {}'.format(craftstep)] +
['elapsed {}'.format(round(time() - tic, 3))] + [
'{} {}'.format(key, round(val, 2))
for key, val in resM.items()
]))
def dock_weights_and_buffers(epoch, craftstep):
if epoch == 0: # randomly initialize
meta.init_weights(sess, pretrain_weights)
elif craftstep == 0: # train or load from weightset to correct epoch
if args.weightsettrain or args.weightset == '': train(epoch)
else:
meta.load_weights(
sess, comet_pull_weight(epoch, weightapi, args, rank))
else: # restore weights from previous replay
meta.cache_weights(sess, cache=f'replay-{replay}', restore=True)
print('==> begin crafting poisons on rank {}'.format(rank))
for craftstep in range(args.ncraftstep):
# auxiliary tasks
tic = time()
if not craftstep % args.restartperiod: restart_poison()
if not craftstep % args.logperiod and rank == 0: comet_log_poison()
craftrate = cr_schedule(
args.craftrate,
craftstep,
schedule=[i * args.crdropperiod for i in [1, 2]])
resMs, resLs = [], []
for replay in range(args.nreplay):
epoch = ((rank + replay * args.nproc) * args.stagger +
craftstep) % (args.nmeta * args.stagger)
lrnrate = lr_schedule(args.lrnrate, epoch, args.warmupperiod)
dock_weights_and_buffers(epoch, craftstep)
# iterate through all batches in epoch
for craftfeed, trainfeed, hasPoison in feeddict_generator(
xtrain, ytrain, lrnrate, meta, args):
if args.trajectory == 'clean':
if hasPoison:
_, resM, = sess.run([
meta.accumop,
meta.resultM,
], craftfeed)
_, resL, = sess.run([
meta.trainop,
meta.resultL,
], trainfeed)
elif args.trajectory == 'poison':
_, _, resM, resL, = sess.run([
meta.accumop,
meta.trainop,
meta.resultM,
meta.resultL,
], craftfeed)
meta.cache_weights(sess, cache=f'replay-{replay}')
resL.update(dict(epoch=epoch, craftrate=craftrate))
resMs.append(resM)
resLs.append(resL)
avg_metagrads = []
for metagrad_accum in sess.run(meta.metagrads_accum):
avg_metagrad = np.zeros_like(metagrad_accum)
mpi.Allreduce(metagrad_accum, avg_metagrad, op=MPI.SUM)
avg_metagrads.append(avg_metagrad / args.nmeta)
sess.run([
meta.craftop,
], {
meta.avg_metagrads: tuple(avg_metagrads),
meta.craftrate: craftrate
})
log_epoch_results(resMs, resLs, craftstep)
experiment.send_notification(f'{args.tag} finished', 'finished')
experiment.end()
if not args.skipvictim:
print('==> crafting finished. begin victim.')
meta.init_weights(sess, pretrain_weights)
from victim import victim
argsmod = dict(craftsteps=[craftstep], ntrial=1, Xtag=None)
kwargs = dict(argsmod=argsmod,
sess=sess,
meta=meta,
xtrain=xtrain,
ytrain=ytrain,
xvalid=xvalid,
yvalid=yvalid,
xbase=xbase,
ybase=ybase,
xtarget=xtarget,
ytarget=ytarget,
ytargetadv=ytargetadv)
victim(kwargs)
def make_values(mu, sigma):
return [random.gauss(mu, sigma) for r in range(SIZE)]
print("Logging...")
for epoch in range(EPOCHS):
for step in range(STEPS):
e.log_histogram_3d(make_values(epoch * 10, 1 + step),
"%s/v1" % epoch,
step=step)
print("Uploading...")
e.end()
print("Testing...")
api = API(cache=False)
apie = APIExperiment(previous_experiment=e.id, api=api)
histograms_json = apie.get_asset_list("histogram_combined_3d")
count = 0
while len(histograms_json) != EPOCHS and count < MAX_TRIES:
print("Retry get assets...")
histograms_json = apie.get_asset_list("histogram_combined_3d")
count += 1
if count == MAX_TRIES:
print("ERROR: missing histogram at %s :" %
(apie.url + "?experiment-tab=histograms", ))
for histogram_json in histograms_json:
asset_id = histogram_json["assetId"]
from scipy.io import savemat
import os
from os.path import join, basename
import pickle
import utils_rollout
home = os.environ['HOME']
logdir = join(home, 'ckpt/swissroll/analyze_poisonfrac')
os.makedirs(logdir, exist_ok=True)
experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k",
parse_args=False,
project_name='images',
workspace="wronnyhuang")
cometapi.set_api_key('W2gBYYtc8ZbGyyNct5qYGR2Gl')
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
experiments = api.get('wronnyhuang/swissroll-poisonfrac-11')
## grab data
sweep = {}
for i, expt in enumerate(experiments):
# get the distrfrac value and initialize if needed
parameters = {d.pop('name'): d for d in expt.parameters}
distrfrac = float(parameters['distrfrac']['valueMin'])
if distrfrac in sweep: continue
if distrfrac not in sweep:
sweep[distrfrac] = {}
sweep[distrfrac]['trainlast'] = []
class Translator(object):
def __init__(
self,
upload_experiment,
api_key,
output_dir,
force_reupload,
mlflow_store_uri,
answer,
email,
):
# type: (bool, str, str, bool, str, Optional[bool], str) -> None
self.answer = answer
self.email = email
self.config = get_config()
# Display the start banner
LOGGER.info(BANNER)
# get_api_key
self.api_key, self.token = self.get_api_key_or_login(api_key)
# May not need this always?
self.api_client = API(self.api_key, cache=False)
self.workspace = self.config["comet.workspace"]
# Get a workspace
if not self.workspace:
details = self.api_client.get_account_details()
self.workspace = details["defaultWorkspaceName"]
if output_dir is None:
output_dir = tempfile.mkdtemp()
# MLFlow conversion
self.store = _get_store(mlflow_store_uri)
try:
self.model_registry_store = get_model_registry_store(
mlflow_store_uri)
except UnsupportedModelRegistryStoreURIException:
self.model_registry_store = None
# Most of list_experiments returns a list anyway
self.mlflow_experiments = list(self.store.list_experiments())
self.len_experiments = len(
self.mlflow_experiments) # We start counting at 0
self.summary = {
"experiments": 0,
"runs": 0,
"tags": 0,
"params": 0,
"metrics": 0,
"artifacts": 0,
}
self.upload_experiment = upload_experiment
self.output_dir = output_dir
self.force_reupload = force_reupload
self.mlflow_store_uri = mlflow_store_uri
def prepare(self):
LOGGER.info("Starting Comet Extension for MLFlow")
LOGGER.info("")
LOGGER.info("Preparing data locally from: %r",
get_store_id(self.store))
LOGGER.info("You will have an opportunity to review.")
LOGGER.info("")
prepared_data = []
# First prepare all the data except the metadata as we need a project name
for experiment_number, experiment in enumerate(
self.mlflow_experiments):
experiment_name = experiment.experiment_id
if experiment.name:
experiment_name = experiment.name
LOGGER.info(
"# Preparing experiment %d/%d: %s",
experiment_number + 1,
self.len_experiments,
experiment_name,
)
LOGGER.debug(
"# Preparing experiment %d/%d: %r",
experiment_number + 1,
self.len_experiments,
experiment,
)
self.summary["experiments"] += 1
try:
prepared_runs = list(self.prepare_mlflow_exp(experiment))
prepared_data.append({
"experiment": experiment,
"runs": prepared_runs
})
LOGGER.info("")
except Exception:
LOGGER.exception(
"# Error preparing experiment %d/%d: %r",
experiment_number + 1,
self.len_experiments,
experiment,
)
LOGGER.error("")
Reporting.report("mlflow_error",
api_key=self.api_key,
err_msg=traceback.format_exc())
table = [
("Experiments", "Projects", self.summary["experiments"]),
("Runs", "Experiments", self.summary["runs"]),
("Tags", "Others", self.summary["tags"]),
("Parameters", "Parameters", self.summary["params"]),
("Metrics", "Metrics", self.summary["metrics"]),
("Artifacts", "Assets", self.summary["artifacts"]),
]
LOGGER.info(
tabulate(
table,
headers=["MLFlow name:", "Comet.ml name:", "Prepared count:"],
tablefmt="presto",
))
LOGGER.info("")
LOGGER.info("All prepared data has been saved to: %s",
abspath(self.output_dir))
# Upload or not?
print("")
if self.answer is None:
upload = input("Upload prepared data to Comet.ml? [y/N] ") in ("Y",
"y")
else:
upload = self.answer
print("")
should_upload = self.upload_experiment
should_upload = should_upload and upload
if should_upload:
self.upload(prepared_data)
else:
self.save_locally(prepared_data)
LOGGER.info("")
LOGGER.info(
"If you need support, you can contact us at http://chat.comet.ml/ or https://comet.ml/docs/quick-start/#getting-support"
)
LOGGER.info("")
def prepare_mlflow_exp(
self,
exp,
):
runs_info = self.store.list_run_infos(exp.experiment_id, ViewType.ALL)
len_runs = len(runs_info)
for run_number, run_info in enumerate(runs_info):
try:
run_id = run_info.run_id
run = self.store.get_run(run_id)
LOGGER.info(
"## Preparing run %d/%d [%s]",
run_number + 1,
len_runs,
run_id,
)
LOGGER.debug("## Preparing run %d/%d: %r", run_number + 1,
len_runs, run)
offline_archive = self.prepare_single_mlflow_run(run, exp.name)
if offline_archive:
self.summary["runs"] += 1
yield (run, offline_archive)
except Exception:
LOGGER.exception(
"## Error preparing run %d/%d [%s]",
run_number + 1,
len_runs,
run_id,
)
LOGGER.error("")
Reporting.report("mlflow_error",
api_key=self.api_key,
err_msg=traceback.format_exc())
def prepare_single_mlflow_run(self, run, original_experiment_name):
self.tmpdir = tempfile.mkdtemp()
if not run.info.end_time:
# Seems to be the case when using the optimizer, some runs doesn't have an end_time
LOGGER.warning("### Skipping run, no end time")
return False
run_start_time = run.info.start_time
messages_file_path = os.path.join(self.tmpdir, "messages.json")
with JsonLinesFile(messages_file_path, self.tmpdir) as json_writer:
# Get mlflow tags
tags = run.data.tags
if not tags:
tags = {}
LOGGER.debug("### Preparing env details")
json_writer.write_filename_msg(tags["mlflow.source.name"],
run_start_time)
json_writer.write_user_msg(tags["mlflow.user"], run_start_time)
LOGGER.debug("### Preparing git details")
json_writer.write_git_meta_msg(
tags.get("mlflow.source.git.commit"),
tags.get("mlflow.source.git.repoURL"),
run_start_time,
)
# Import any custom name
if tags.get("mlflow.runName"):
tags["Name"] = tags["mlflow.runName"]
# Save the run id as tag too as Experiment id can be different in case
# of multiple uploads
tags["mlflow.runId"] = run.info.run_id
if tags.get("mlflow.parentRunId"):
base_url = url_join(self.api_client.server_url,
"/api/experiment/redirect")
tags["mlflow.parentRunUrl"] = format_url(
base_url, experimentKey=tags["mlflow.parentRunId"])
# Save the original MLFlow experiment name too as Comet.ml project might
# get renamed
tags["mlflow.experimentName"] = original_experiment_name
LOGGER.debug("### Importing tags")
for tag_name, tag_value in tags.items():
LOGGER.debug("#### Tag %r: %r", tag_name, tag_value)
json_writer.write_log_other_msg(tag_name, tag_value,
run_start_time)
self.summary["tags"] += 1
# Mark the experiments has being uploaded from MLFlow
json_writer.write_log_other_msg("Uploaded from", "MLFlow",
run_start_time)
LOGGER.debug("### Importing params")
for param_key, param_value in run.data.params.items():
LOGGER.debug("#### Param %r: %r", param_key, param_value)
json_writer.write_param_msg(param_key, param_value,
run_start_time)
self.summary["params"] += 1
LOGGER.debug("### Importing metrics")
for metric in run.data._metric_objs:
metric_history = self.store.get_metric_history(
run.info.run_id, metric.key)
# Check if all steps are uniques, if not we don't pass any so the backend
# fallback to the unique timestamp
steps = [mh.step for mh in metric_history]
use_steps = True
if len(set(steps)) != len(metric_history):
LOGGER.warning(
"Non-unique steps detected, importing metrics with wall time instead"
)
use_steps = False
for mh in metric_history:
if use_steps:
step = mh.step
else:
step = None
json_writer.write_metric_msg(mh.key, step, mh.timestamp,
mh.value)
self.summary["metrics"] += 1
LOGGER.debug("#### Metric %r: %r", metric.key, metric_history)
LOGGER.debug("### Importing artifacts")
artifact_store = get_artifact_repository(run.info.artifact_uri)
# List all the registered models if possible
models_prefixes = {}
if self.model_registry_store:
query = "run_id='%s'" % run.info.run_id
registered_models = self.model_registry_store.search_model_versions(
query)
for model in registered_models:
model_relpath = os.path.relpath(model.source,
run.info.artifact_uri)
models_prefixes[model_relpath] = model
for artifact in walk_run_artifacts(artifact_store):
artifact_path = artifact.path
LOGGER.debug("### Artifact %r: %r", artifact, artifact_path)
# Check if the file is an visualization or not
_, extension = os.path.splitext(artifact_path)
local_artifact_path = artifact_store.download_artifacts(
artifact_path)
self.summary["artifacts"] += 1
# Check if it's belonging to one of the registered model
matching_model = None
for model_prefix, model in models_prefixes.items():
if artifact_path.startswith(model_prefix):
matching_model = model
# We should match at most one model
break
if matching_model:
json_writer.log_artifact_as_model(
local_artifact_path,
artifact_path,
run_start_time,
matching_model.registered_model.name,
)
else:
json_writer.log_artifact_as_asset(
local_artifact_path,
artifact_path,
run_start_time,
)
return self.compress_archive(run.info.run_id)
def upload(self, prepared_data):
LOGGER.info("# Start uploading data to Comet.ml")
all_project_names = []
with tqdm(total=self.summary["runs"]) as pbar:
for experiment_data in prepared_data:
experiment = experiment_data["experiment"]
project_name = self.get_or_create_comet_project(experiment)
# Sync the experiment note
project_note = experiment.tags.get("mlflow.note.content", None)
if project_note:
note_template = (
u"/!\\ This project notes has been copied from MLFlow. It might be overwritten if you run comet_for_mlflow again/!\\ \n%s"
% project_note)
# We don't support Unicode project notes yet
self.api_client.set_project_notes(
self.workspace,
project_name,
note_template,
)
all_project_names.append(project_name)
runs = experiment_data["runs"]
for mlflow_run, archive_path in runs:
write_comet_experiment_metadata_file(
mlflow_run, project_name, archive_path, self.workspace)
upload_single_offline_experiment(
archive_path,
self.api_key,
force_reupload=self.force_reupload,
display_level="debug",
)
pbar.update(1)
LOGGER.info("")
LOGGER.info(
"Explore your experiment data on Comet.ml with the following links:",
)
if len(all_project_names) < 6:
for project_name in all_project_names:
project_url = url_join(
self.api_client._get_url_server(),
self.workspace + "/",
project_name,
loginToken=self.token,
)
LOGGER.info("\t- %s", project_url)
else:
url = url_join(
self.api_client._get_url_server(),
self.workspace,
query="mlflow",
loginToken=self.token,
)
LOGGER.info("\t- %s", url)
LOGGER.info(
"Get deeper instrumentation by adding Comet SDK to your project: https://comet.ml/docs/python-sdk/mlflow/"
)
LOGGER.info("")
def save_locally(self, prepared_data):
for experiment_data in prepared_data:
experiment = experiment_data["experiment"]
project_name = get_comet_project_name(self.store, experiment.name)
runs = experiment_data["runs"]
for mlflow_run, archive_path in runs:
write_comet_experiment_metadata_file(mlflow_run, project_name,
archive_path,
self.workspace)
LOGGER.info("Data not uploaded. To upload later run:")
LOGGER.info(" comet upload %s/*.zip", abspath(self.output_dir))
LOGGER.info("")
LOGGER.info("To get a preview of what was prepared, run:")
LOGGER.info(" comet offline %s/*.zip", abspath(self.output_dir))
def compress_archive(self, run_id):
filepath = os.path.join(self.output_dir, "%s.zip" % run_id)
zipfile = ZipFile(filepath, "w")
for file in os.listdir(self.tmpdir):
zipfile.write(os.path.join(self.tmpdir, file), file)
zipfile.close()
shutil.rmtree(self.tmpdir)
return filepath
def create_and_save_comet_project(self, exp, tag_name):
# Create a Comet project with the name and description
project_name = get_comet_project_name(self.store, exp.name)
# Check if the project exists already
project = self.api_client.get_project(self.workspace, project_name)
if not project:
project = self.api_client.create_project(self.workspace,
project_name,
public=False)
project_id = project["projectId"]
else:
project_id = project["projectId"]
# Save the project id to the experiment tags
self.store.set_experiment_tag(exp.experiment_id,
RunTag(tag_name, project_id))
return project_name
def get_or_create_comet_project(self, exp):
# Check if the mlflow experiment has already a project ID for this workspace
tag_name = "comet-project-{}".format(self.workspace)
project_id = None
if tag_name in exp.tags:
project_id = exp.tags[tag_name]
# Check if the project exists
project = self.api_client.get_project_by_id(project_id)
if not project:
# A previous project ID has been saved but don't exists anymore (at
# least in this environment), recreate it
return self.create_and_save_comet_project(exp, tag_name)
return project["projectName"]
else:
return self.create_and_save_comet_project(exp, tag_name)
def create_or_login(self):
auth_api_client = get_comet_api_client(None)
LOGGER.info("Please create a free Comet account with your email.")
if self.email is None:
email = input("Email: ")
print("")
else:
email = self.email
# Check if the email exists in the system
try:
status = auth_api_client.check_email(email, "comet-for-mlflow")
except CometRestApiException as e:
status = e.response.status_code
# We hit rate-limitting
if status == 429:
# We hit rate-limitting
LOGGER.error(
"Too many user login requests, please try again in one minute."
)
sys.exit(1)
new_account = status != 200
if new_account:
LOGGER.info("Please enter a username for your new account.")
username = input("Username: "******"")
new_account = auth_api_client.create_user(email, username,
"comet-for-mlflow")
Reporting.report("mlflow_new_user", api_key=new_account["apiKey"])
LOGGER.info(
"A Comet.ml account has been created for you and an email was sent to you to setup your password later."
)
save_api_key(new_account["apiKey"])
LOGGER.info(
"Your Comet API Key has been saved to ~/.comet.ini, it is also available on your Comet.ml dashboard."
)
return (
new_account["apiKey"],
new_account["token"],
)
else:
LOGGER.info(
"An account already exists for this account, please input your API Key below (you can find it in your Settings page, https://comet.ml/docs/quick-start/#getting-your-comet-api-key):"
)
api_key = input("API Key: ")
Reporting.report("mlflow_existing_user", api_key=api_key)
return (
api_key,
None,
)
def get_api_key_or_login(self, api_key):
# ok
api_key = get_api_key(api_key, self.config)
if api_key is None:
return self.create_or_login()
Reporting.report("mlflow_existing_user", api_key=api_key)
return (api_key, None)
parser.add_argument('-Xweightdecay', action='store_true')
parser.add_argument('-Xaugment', action='store_true')
parser.add_argument('-Xoptimizer', action='store_true')
parser.add_argument('-Xbatchsize', action='store_true')
parser.add_argument('-Xlrnrate', action='store_true')
parser.add_argument('-Xnet', action='store_true')
parser.add_argument('-neval', default=None, type=int)
# unused, but needed for proper meta construction
parser.add_argument('-eps', default=16, type=float)
parser.add_argument('-nadapt', default=1, type=int)
parser.add_argument('-reducemethod', default='average',
type=str) # softmax, average
parser.add_argument('-objective', default='xent', type=str) # cw, xent
args = parser.parse_args()
# api = API()
api = API(api_key="your_api_key")
attrs = [
'lrnrate', 'batchsize', 'nbatch', 'npoison', 'ntarget', 'pretrain',
'warmupperiod', 'optimizer', 'weightdecay', 'augment', 'net',
'multiclasspoison', 'targetclass', 'poisonclass', 'ytargetadv'
]
copy_to_args_from_experiment(args, args.key, api, attrs)
if args.tag == 'multi': args.multiclasspoison = True
if args.Xweightdecay: args.weightdecay = True
if args.Xaugment: args.augment = True
if args.Xbatchsize: args.batchsize = 250
if args.Xlrnrate: args.lrnrate = .2
if args.Xoptimizer: args.optimizer = 'sgd'
if args.Xnet:
args.net = 'ConvNet'
args.lrnrate = .05
from matplotlib.pyplot import plot, imshow, colorbar, show, axis, hist, subplot, xlabel, ylabel, title, legend, savefig, figure, close, suptitle, tight_layout, xlim, ylim, contourf
from comet_ml import API, Experiment
import numpy as np
from scipy.io import savemat
experiment = Experiment(api_key="vPCPPZrcrUBitgoQkvzxdsh9k",
parse_args=False,
project_name='images',
workspace="wronnyhuang")
api = API(rest_api_key='W2gBYYtc8ZbGyyNct5qYGR2Gl')
allexperiments = api.get('wronnyhuang/surface-mean')
class args:
res = 30
span = 2
# span
clin = args.span / 2 * np.linspace(-1, 1, args.res)
data = {}
for i, expt in enumerate(allexperiments):
# filter experiments by name
name = expt.name
namename = 'poison' if 'poison' in name else 'clean'
# merge data into cfeed
raw = expt.metrics_raw