def main(args):
comet_api = API()
comet_files = Path(args.root_dir).glob("**/comet.url")
exp_dict = {}
df_list = []
df_keys = []
for path in comet_files:
print(f"Retrieving data from {path}")
with open(path, "r") as f:
url = f.readline()
key = url.split("/")[-1].strip()
exp_dict.update({key: {"path": path}})
exp = comet_api.get_experiment(workspace=args.workspace,
project_name=args.project,
experiment=key)
metrics = exp.get_metrics()
df = metrics2df(metrics)
df["path"] = path
df_list.append(df)
df_keys.append(path.parent)
if args.save_each_csv:
df.to_csv(path.parent / "comet.csv")
df = pd.concat(df_list, keys=df_keys, names=["path", "step"])
if args.output_csv:
df.to_csv(args.output_csv)
def __init__(
self,
api_key: Optional[str] = None,
save_dir: Optional[str] = None,
project_name: Optional[str] = None,
rest_api_key: Optional[str] = None,
experiment_name: Optional[str] = None,
experiment_key: Optional[str] = None,
offline: bool = False,
prefix: str = "",
**kwargs,
):
if comet_ml is None:
raise ImportError(
"You want to use `comet_ml` logger which is not installed yet, install it with `pip install comet-ml`."
)
super().__init__()
self._experiment = None
# Determine online or offline mode based on which arguments were passed to CometLogger
api_key = api_key or comet_ml.config.get_api_key(
None, comet_ml.config.get_config())
if api_key is not None and save_dir is not None:
self.mode = "offline" if offline else "online"
self.api_key = api_key
self._save_dir = save_dir
elif api_key is not None:
self.mode = "online"
self.api_key = api_key
self._save_dir = None
elif save_dir is not None:
self.mode = "offline"
self._save_dir = save_dir
else:
# If neither api_key nor save_dir are passed as arguments, raise an exception
raise MisconfigurationException(
"CometLogger requires either api_key or save_dir during initialization."
)
log.info(f"CometLogger will be initialized in {self.mode} mode")
self._project_name = project_name
self._experiment_key = experiment_key
self._experiment_name = experiment_name
self._prefix = prefix
self._kwargs = kwargs
self._future_experiment_key = None
if rest_api_key is not None:
# Comet.ml rest API, used to determine version number
self.rest_api_key = rest_api_key
self.comet_api = API(self.rest_api_key)
else:
self.rest_api_key = None
self.comet_api = None
self._kwargs = kwargs
def delete_on_exit(exp):
"""
Registers a callback to delete the comet exp at program exit
Args:
exp (comet_ml.Experiment): The exp to delete
"""
rest_api_key = get_comet_rest_api_key()
api = API(api_key=rest_api_key)
atexit.register(comet_handler(exp, api))
def __init__(self, workspace="dna-i", project=None):
'''
Initializes API key and optionally which workspace and projects to use
Parameters
----------
workspace : STR, optional
Comet workspace. The default is "dna-i".
project : list(STR), optional
List of Projects to use. The default is None.
Returns
-------
None.
'''
self.workspace = workspace
self.project = project
self.api = API(api_key="V0OXnWOi4KVNS4OkwLjdnxSgK")
def __init__(self,
api_key: Optional[str] = None,
save_dir: Optional[str] = None,
workspace: Optional[str] = None,
project_name: Optional[str] = None,
rest_api_key: Optional[str] = None,
experiment_name: Optional[str] = None,
experiment_key: Optional[str] = None,
offline: bool = False,
**kwargs):
if not _COMET_AVAILABLE:
raise ImportError(
'You want to use `comet_ml` logger which is not installed yet,'
' install it with `pip install comet-ml`.')
super().__init__()
self._experiment = None
# Determine online or offline mode based on which arguments were passed to CometLogger
api_key = api_key or get_api_key(None, get_config())
if api_key is not None and save_dir is not None:
self.mode = "offline" if offline else "online"
self.api_key = api_key
self._save_dir = save_dir
elif api_key is not None:
self.mode = "online"
self.api_key = api_key
elif save_dir is not None:
self.mode = "offline"
self._save_dir = save_dir
else:
# If neither api_key nor save_dir are passed as arguments, raise an exception
raise MisconfigurationException(
"CometLogger requires either api_key or save_dir during initialization."
)
log.info(f"CometLogger will be initialized in {self.mode} mode")
self.workspace = workspace
self.project_name = project_name
self.experiment_key = experiment_key
self._kwargs = kwargs
if rest_api_key is not None:
# Comet.ml rest API, used to determine version number
self.rest_api_key = rest_api_key
self.comet_api = API(self.rest_api_key)
else:
self.rest_api_key = None
self.comet_api = None
if experiment_name:
self.experiment.set_name(experiment_name)
self._kwargs = kwargs
def __init__(self,
api_key: Optional[str] = None,
save_dir: Optional[str] = None,
workspace: Optional[str] = None,
project_name: Optional[str] = None,
rest_api_key: Optional[str] = None,
experiment_name: Optional[str] = None,
experiment_key: Optional[str] = None,
**kwargs):
super().__init__()
self._experiment = None
# Determine online or offline mode based on which arguments were passed to CometLogger
if api_key is not None:
self.mode = "online"
self.api_key = api_key
elif save_dir is not None:
self.mode = "offline"
self.save_dir = save_dir
else:
# If neither api_key nor save_dir are passed as arguments, raise an exception
raise MisconfigurationException(
"CometLogger requires either api_key or save_dir during initialization."
)
log.info(f"CometLogger will be initialized in {self.mode} mode")
self.workspace = workspace
self.project_name = project_name
self.experiment_key = experiment_key
self._kwargs = kwargs
if rest_api_key is not None:
# Comet.ml rest API, used to determine version number
self.rest_api_key = rest_api_key
self.comet_api = API(self.rest_api_key)
else:
self.rest_api_key = None
self.comet_api = None
if experiment_name:
try:
self.name = experiment_name
except TypeError as e:
log.exception(
"Failed to set experiment name for comet.ml logger")
self._kwargs = kwargs
def __init__(self,
api_key: Optional[str] = None,
save_dir: Optional[str] = None,
workspace: Optional[str] = None,
project_name: Optional[str] = None,
rest_api_key: Optional[str] = None,
experiment_name: Optional[str] = None,
experiment_key: Optional[str] = None,
**kwargs):
r"""
Requires either an API Key (online mode) or a local directory path (offline mode)
.. code-block:: python
# ONLINE MODE
from pytorch_lightning.loggers import CometLogger
# arguments made to CometLogger are passed on to the comet_ml.Experiment class
comet_logger = CometLogger(
api_key=os.environ["COMET_API_KEY"],
workspace=os.environ["COMET_WORKSPACE"], # Optional
project_name="default_project", # Optional
rest_api_key=os.environ["COMET_REST_API_KEY"], # Optional
experiment_name="default" # Optional
)
trainer = Trainer(logger=comet_logger)
.. code-block:: python
# OFFLINE MODE
from pytorch_lightning.loggers import CometLogger
# arguments made to CometLogger are passed on to the comet_ml.Experiment class
comet_logger = CometLogger(
save_dir=".",
workspace=os.environ["COMET_WORKSPACE"], # Optional
project_name="default_project", # Optional
rest_api_key=os.environ["COMET_REST_API_KEY"], # Optional
experiment_name="default" # Optional
)
trainer = Trainer(logger=comet_logger)
Args:
api_key (str): Required in online mode. API key, found on Comet.ml
save_dir (str): Required in offline mode. The path for the directory to save local comet logs
workspace (str): Optional. Name of workspace for this user
project_name (str): Optional. Send your experiment to a specific project.
Otherwise will be sent to Uncategorized Experiments.
If project name does not already exists Comet.ml will create a new project.
rest_api_key (str): Optional. Rest API key found in Comet.ml settings.
This is used to determine version number
experiment_name (str): Optional. String representing the name for this particular experiment on Comet.ml.
experiment_key (str): Optional. If set, restores from existing experiment.
"""
super().__init__()
self._experiment = None
# Determine online or offline mode based on which arguments were passed to CometLogger
if api_key is not None:
self.mode = "online"
self.api_key = api_key
elif save_dir is not None:
self.mode = "offline"
self.save_dir = save_dir
else:
# If neither api_key nor save_dir are passed as arguments, raise an exception
raise MisconfigurationException("CometLogger requires either api_key or save_dir during initialization.")
log.info(f"CometLogger will be initialized in {self.mode} mode")
self.workspace = workspace
self.project_name = project_name
self.experiment_key = experiment_key
self._kwargs = kwargs
if rest_api_key is not None:
# Comet.ml rest API, used to determine version number
self.rest_api_key = rest_api_key
self.comet_api = API(self.rest_api_key)
else:
self.rest_api_key = None
self.comet_api = None
if experiment_name:
try:
self.name = experiment_name
except TypeError as e:
log.exception("Failed to set experiment name for comet.ml logger")
self._kwargs = kwargs
"--get_optimizer_weights",
action="store_true",
help=
"Also grab the state dictionary of the optimizer at the end of the specified epoch"
)
#XMUs9uI19KQNdYrQhuXPnAfpB
args = parser.parse_args()
experiment_key = args.experiment_key
epoch_number = args.epoch_number
apikey = args.api_key
get_optimizer_weights = args.get_optimizer_weights
output_directory = os.path.join(args.output_directory, experiment_key)
if not os.path.isdir(output_directory):
os.makedirs(output_directory)
api = API(api_key=apikey)
experiment: APIExperiment = api.get_experiment("electric-turtle",
"probabilisticbeziercurves",
experiment_key)
assetlist = experiment.get_asset_list()
assetdict = {d['fileName']: d['assetId'] for d in assetlist}
#lookahead_indices = int(experiment.get_parameters_summary(parameter="lookahead_indices")["valueCurrent"])
print("Getting hyperparameters from comet")
datasets_file_name = "datasets.yaml"
experiment_file_name = "experiment_config.yaml"
config_file_name = "model_config.yaml"
config_yaml = experiment.get_asset(assetdict[config_file_name],
return_type="text")
config = yaml.load(io.StringIO(config_yaml), Loader=yaml.SafeLoader)
projectdict = {
"cnnlstm": "deepracingcnnlstm",
"bezier": "deepracingbezierpredictor",
"pilot": "deepracingpilotnet"
}
#XMUs9uI19KQNdYrQhuXPnAfpB
args = parser.parse_args()
experiment_key = args.experiment_key
epoch_number = args.epoch_number
restkey = args.restkey
output_directory = os.path.join(args.output_directory,
experiment_key + "_from_comet")
if not os.path.isdir(output_directory):
os.makedirs(output_directory)
if restkey is None:
api = API()
else:
api = API(rest_api_key=restkey)
experiment: APIExperiment = api.get_experiment("electric-turtle",
"deepracingposepredictor",
experiment_key)
assetlist = experiment.get_asset_list()
assetdict = {d['fileName']: d['assetId'] for d in assetlist}
#get network weight file
weightfilename = "epoch_%d_params.pt" % (epoch_number, )
optimizerfilename = "epoch_%d_optimizer.pt" % (epoch_number, )
print("Getting network weights from comet")
params_binary = experiment.get_asset(assetdict[weightfilename])
def __init__(self,
api_key: Optional[str] = None,
save_dir: Optional[str] = None,
workspace: Optional[str] = None,
project_name: Optional[str] = None,
rest_api_key: Optional[str] = None,
experiment_name: Optional[str] = None,
experiment_key: Optional[str] = None,
force_offline: bool = False,
**kwargs):
if not _COMET_AVAILABLE:
raise ImportError(
'You want to use `comet_ml` logger which is not installed yet,'
' install it with `pip install comet-ml`.')
super().__init__()
self._experiment = None
self.save_dir = save_dir
self.api_key = api_key
# Determine online or offline mode based on which arguments were passed to CometLogger
if api_key is None and save_dir is None:
# If neither api_key nor save_dir are passed as arguments, raise an exception
raise MisconfigurationException(
"CometLogger requires either api_key or save_dir during initialization."
)
elif api_key is None:
self.mode = "offline"
elif save_dir is None:
self.mode = "online"
else: # both given so need explicit argument
self.mode = "offline" if force_offline else "online"
log.info(f"CometLogger will be initialized in {self.mode} mode")
self.workspace = workspace
self.project_name = project_name
self.experiment_name = experiment_name
self.experiment_key = experiment_key
self._kwargs = kwargs
if rest_api_key is not None:
# Comet.ml rest API, used to determine version number
self.rest_api_key = rest_api_key
self.comet_api = API(self.rest_api_key)
else:
self.rest_api_key = None
self.comet_api = None
if self.experiment_name is None:
self._version = self.experiment.id
else:
# ensure that the directory name is unique by appending a number to the end
root_save_dir = Path(self.save_dir) / self.name
root_save_dir.mkdir(exist_ok=True, parents=True)
max_i = -1
for path in root_save_dir.glob("*"):
name = path.name
if name.startswith(self.experiment_name):
try:
i = int(name.split("_")[-1])
max_i = max(max_i, i)
except (IndexError, ValueError): # no _ or no int at end
continue
self._version = f"{self.experiment_name}_{max_i + 1}"
self._kwargs = kwargs
class fetcher:
'''
An Interface for fetching experiment results through comet API.
Example
-------
>>> y = fetcher(project=["crossa-wheat-env-0", "crossa-wheat-env-1"])
>>> results = y.get_results(parameters = ["run_arg_9", "model", "coding"])
'''
def __init__(self, workspace="dna-i", project=None):
'''
Initializes API key and optionally which workspace and projects to use
Parameters
----------
workspace : STR, optional
Comet workspace. The default is "dna-i".
project : list(STR), optional
List of Projects to use. The default is None.
Returns
-------
None.
'''
self.workspace = workspace
self.project = project
self.api = API(api_key="V0OXnWOi4KVNS4OkwLjdnxSgK")
def get_all_projects(self, workspace="dna-i"):
'''
Gets and prints all available projects.
Parameters
----------
workspace : STR, optional
The default is "dna-i".
Returns
-------
LIST(STR)
available projects
'''
self.available_projects = self.api.get(workspace)
print(self.available_projects)
return self.available_projects
def get_all_experiments(self, project=None):
'''
Fetches all experiments for a given project
Parameters
----------
project : STR, optional
The default is None.
Returns
-------
COMET EXPERIMENT OBJ
'''
if project is not None:
self.experiments = self.api.get(f"{self.workspace}/{project}")
print("fetching experiments")
else:
experiment = []
for proj in self.available_projects:
elem = {'project': proj,
'experiments':self.api.get(f"{self.workspace}/{proj}")}
experiment.append(elem)
self.experiments = experiment
return self.experiments
def get_exp_keys(self):
'''
Converts list of comet experiment objects to experiment keys.
Returns
-------
list of str
'''
self.keys = []
print("fetching experiment keys")
for exp in self.experiments:
self.keys.append(exp.get_metadata()['experimentKey'])
return self.keys
def result_to_df(self):
'''
Converts list of dictionaries result created with get_results to
human friendly to a pandas dataframe.
Returns
-------
DataFrame
'''
r = []
for p in self.results:
result = p["metrics"]
project = p["project"]
for k,v in result.items():
d = {}
d["database"] = project
d['experimentKey']=v["experimentKey"]
d = {**d, **v['params']}
for metric in v["metrics"]:
d['metricName']= metric["metricName"]
for value, step in zip(metric["values"], metric["epochs"]):
aux= { **d, 'value': value,'step': step,}
r.append(aux)
self.results = pd.DataFrame(r)
def get_results(self, metrics=["r_test (r)", "mse_test", "r_train (r)",
"mse_train", "r_test (r)_mean", "mse_test_mean"],
parameters=["model", "coding"], project=None):
'''
Fetches experiment metrics and parameters
Parameters
----------
metrics : LIST(STR), optional
Which metrics to retrieve. The default is ["r_test (r)", "mse_test", "r_train (r)", "mse_train", "r_test (r)_mean", "mse_test_mean"].
parameters : LIST(STR), optional
Which param to retrieve. The default is ["model", "coding"].
project : LIST(STR), optional
Which projects to retriever data from. The default is None.
Returns
-------
Pandas dataframe with results
'''
print(f"Fetching {metrics} and {parameters}")
if project is None:
project = self.project
self.results = []
for proj in project:
print(f"Project {proj}")
self.get_all_experiments(project=proj)
self.get_exp_keys()
value = self.api.get_metrics_for_chart(self.keys, full=True, metrics = metrics, parameters=parameters)
self.results.append({"project": proj, "metrics":value })
self.result_to_df()
return self.results
parser = argparse.ArgumentParser(description="Download experiment from Comet.ml")
parser.add_argument("experiment_key", type=str, help="Experiment key to grab from comet.")
parser.add_argument("--restkey", type=str, required=False, default=None, help="Experiment key to grab from comet.")
parser.add_argument("--epoch_number", type=int, required=False, default=100, help="Experiment key to grab from comet.")
parser.add_argument("--output_directory", type=str, required=False, default=".", help="Where to put the config and data files.")
#XMUs9uI19KQNdYrQhuXPnAfpB
args = parser.parse_args()
experiment_key = args.experiment_key
epoch_number = args.epoch_number
restkey = args.restkey
output_directory = os.path.join(args.output_directory, experiment_key + "_from_comet")
if not os.path.isdir(output_directory):
os.makedirs(output_directory)
if restkey is None:
api = API()
else:
api = API(rest_api_key=restkey)
experiment : APIExperiment = api.get_experiment("electric-turtle", "deepracingpilotnet", experiment_key)
assetlist = experiment.get_asset_list()
assetdict = {d['fileName']: d['assetId'] for d in assetlist}
#get network weight file
weightfilename = "pilotnet_epoch_%d_params.pt" %(epoch_number,)
print("Getting network weights from comet")
params_binary = experiment.get_asset(assetdict[weightfilename])
outputweightfile = os.path.join(output_directory,weightfilename)
with open(outputweightfile, 'wb') as f:
import comet_ml
comet_ml.config.save(api_key="WRmA8ms9A78K85fLxcv8Nsld9")
from comet_ml.api import API
import numpy as np
#import pandas as pd
import matplotlib.pyplot as plt
api = API()
exp_urls_lr1 = {
"run1": "yasmeenvh/growspaceenv-ppo/91df8e59877e484f96022b6b3548fcf8",
"run2": "yasmeenvh/growspaceenv-ppo/a633ea5fc5064259a013889cf46dfe64",
"run3": "yasmeenvh/growspaceenv-ppo/b9d798bcf5d545c884162ac3373c8d8e"
}
# exp_urls_lr2 = {"run1":"https://www.comet.ml/yasmeenvh/growspaceenv-ppo/91df8e59877e484f96022b6b3548fcf8",
# "run2":"https://www.comet.ml/yasmeenvh/growspaceenv-ppo/a633ea5fc5064259a013889cf46dfe64",
# "run3":"https://www.comet.ml/yasmeenvh/growspaceenv-ppo/b9d798bcf5d545c884162ac3373c8d8e"}
comet_metrics = [
"Reward Mean", "Reward Max", "Reward Min", "Reward Max",
"Number of Mean Branches"
]
def get_data(url_link, comet_metrics):
min_numframes = 0000000
max_numframes = 100000000
reward_means = []
for run, link in url_link.items():
experiment = api.get(link)
from comet_ml.api import API, APIExperiment
from config import comet_ml_key
comet_api = API(api_key=comet_ml_key)
for key in ["405", "301", "1000"]:
exps = comet_api.get_experiments(
"ericzhao28",
project_name="active-label-shift-adaptation",
pattern=".*" + key + ".*")
for exp in exps:
if exp.get_metadata()["archived"]:
continue
if key in exp.get_name():
print(exp.get_name())
x = comet_api.get("ericzhao28/active-label-shift-adaptation/" +
exp.get_metadata()["experimentKey"])
for asset in x.get_asset_list():
assetid = asset["assetId"]
print(asset["fileName"])
if "csv" not in asset["fileName"]:
continue
with open("../data/" + exp.get_name() + ".csv", "w") as f:
f.write(exp.get_asset(asset_id=assetid,
return_type="text"))
def __init__(self,
api_key=None,
save_dir=None,
workspace=None,
rest_api_key=None,
project_name=None,
experiment_name=None,
**kwargs):
"""Initialize a Comet.ml logger.
Requires either an API Key (online mode) or a local directory path (offline mode)
:param str api_key: Required in online mode. API key, found on Comet.ml
:param str save_dir: Required in offline mode. The path for the directory to save local comet logs
:param str workspace: Optional. Name of workspace for this user
:param str project_name: Optional. Send your experiment to a specific project.
Otherwise will be sent to Uncategorized Experiments.
If project name does not already exists Comet.ml will create a new project.
:param str rest_api_key: Optional. Rest API key found in Comet.ml settings.
This is used to determine version number
:param str experiment_name: Optional. String representing the name for this particular experiment on Comet.ml
"""
super().__init__()
self._experiment = None
# Determine online or offline mode based on which arguments were passed to CometLogger
if save_dir is not None and api_key is not None:
# If arguments are passed for both save_dir and api_key, preference is given to online mode
self.mode = "online"
self.api_key = api_key
elif api_key is not None:
self.mode = "online"
self.api_key = api_key
elif save_dir is not None:
self.mode = "offline"
self.save_dir = save_dir
else:
# If neither api_key nor save_dir are passed as arguments, raise an exception
raise MisconfigurationException(
"CometLogger requires either api_key or save_dir during initialization."
)
logger.info(f"CometLogger will be initialized in {self.mode} mode")
self.workspace = workspace
self.project_name = project_name
self._kwargs = kwargs
if rest_api_key is not None:
# Comet.ml rest API, used to determine version number
self.rest_api_key = rest_api_key
self.comet_api = API(self.rest_api_key)
else:
self.rest_api_key = None
self.comet_api = None
if experiment_name:
try:
self.name = experiment_name
except TypeError as e:
logger.exception(
"Failed to set experiment name for comet.ml logger")