def main(args):
    comet_api = API()
    comet_files = Path(args.root_dir).glob("**/comet.url")
    exp_dict = {}
    df_list = []
    df_keys = []
    for path in comet_files:
        print(f"Retrieving data from {path}")
        with open(path, "r") as f:
            url = f.readline()
        key = url.split("/")[-1].strip()
        exp_dict.update({key: {"path": path}})
        exp = comet_api.get_experiment(workspace=args.workspace,
                                       project_name=args.project,
                                       experiment=key)
        metrics = exp.get_metrics()
        df = metrics2df(metrics)
        df["path"] = path
        df_list.append(df)
        df_keys.append(path.parent)
        if args.save_each_csv:
            df.to_csv(path.parent / "comet.csv")
    df = pd.concat(df_list, keys=df_keys, names=["path", "step"])
    if args.output_csv:
        df.to_csv(args.output_csv)
Ejemplo n.º 2
0
    def __init__(
        self,
        api_key: Optional[str] = None,
        save_dir: Optional[str] = None,
        project_name: Optional[str] = None,
        rest_api_key: Optional[str] = None,
        experiment_name: Optional[str] = None,
        experiment_key: Optional[str] = None,
        offline: bool = False,
        prefix: str = "",
        **kwargs,
    ):
        if comet_ml is None:
            raise ImportError(
                "You want to use `comet_ml` logger which is not installed yet, install it with `pip install comet-ml`."
            )
        super().__init__()
        self._experiment = None

        # Determine online or offline mode based on which arguments were passed to CometLogger
        api_key = api_key or comet_ml.config.get_api_key(
            None, comet_ml.config.get_config())

        if api_key is not None and save_dir is not None:
            self.mode = "offline" if offline else "online"
            self.api_key = api_key
            self._save_dir = save_dir
        elif api_key is not None:
            self.mode = "online"
            self.api_key = api_key
            self._save_dir = None
        elif save_dir is not None:
            self.mode = "offline"
            self._save_dir = save_dir
        else:
            # If neither api_key nor save_dir are passed as arguments, raise an exception
            raise MisconfigurationException(
                "CometLogger requires either api_key or save_dir during initialization."
            )

        log.info(f"CometLogger will be initialized in {self.mode} mode")

        self._project_name = project_name
        self._experiment_key = experiment_key
        self._experiment_name = experiment_name
        self._prefix = prefix
        self._kwargs = kwargs
        self._future_experiment_key = None

        if rest_api_key is not None:
            # Comet.ml rest API, used to determine version number
            self.rest_api_key = rest_api_key
            self.comet_api = API(self.rest_api_key)
        else:
            self.rest_api_key = None
            self.comet_api = None

        self._kwargs = kwargs
Ejemplo n.º 3
0
def delete_on_exit(exp):
    """
    Registers a callback to delete the comet exp at program exit

    Args:
        exp (comet_ml.Experiment): The exp to delete
    """
    rest_api_key = get_comet_rest_api_key()
    api = API(api_key=rest_api_key)
    atexit.register(comet_handler(exp, api))
Ejemplo n.º 4
0
    def __init__(self, workspace="dna-i", project=None):
        '''
        Initializes API key and optionally which workspace and projects to use

        Parameters
        ----------
        workspace : STR, optional
            Comet workspace. The default is "dna-i".
        project : list(STR), optional
            List of Projects to use. The default is None.

        Returns
        -------
        None.

        '''
        self.workspace = workspace
        self.project = project
        self.api = API(api_key="V0OXnWOi4KVNS4OkwLjdnxSgK")
Ejemplo n.º 5
0
    def __init__(self,
                 api_key: Optional[str] = None,
                 save_dir: Optional[str] = None,
                 workspace: Optional[str] = None,
                 project_name: Optional[str] = None,
                 rest_api_key: Optional[str] = None,
                 experiment_name: Optional[str] = None,
                 experiment_key: Optional[str] = None,
                 offline: bool = False,
                 **kwargs):

        if not _COMET_AVAILABLE:
            raise ImportError(
                'You want to use `comet_ml` logger which is not installed yet,'
                ' install it with `pip install comet-ml`.')
        super().__init__()
        self._experiment = None

        # Determine online or offline mode based on which arguments were passed to CometLogger
        api_key = api_key or get_api_key(None, get_config())

        if api_key is not None and save_dir is not None:
            self.mode = "offline" if offline else "online"
            self.api_key = api_key
            self._save_dir = save_dir
        elif api_key is not None:
            self.mode = "online"
            self.api_key = api_key
        elif save_dir is not None:
            self.mode = "offline"
            self._save_dir = save_dir
        else:
            # If neither api_key nor save_dir are passed as arguments, raise an exception
            raise MisconfigurationException(
                "CometLogger requires either api_key or save_dir during initialization."
            )

        log.info(f"CometLogger will be initialized in {self.mode} mode")

        self.workspace = workspace
        self.project_name = project_name
        self.experiment_key = experiment_key
        self._kwargs = kwargs

        if rest_api_key is not None:
            # Comet.ml rest API, used to determine version number
            self.rest_api_key = rest_api_key
            self.comet_api = API(self.rest_api_key)
        else:
            self.rest_api_key = None
            self.comet_api = None

        if experiment_name:
            self.experiment.set_name(experiment_name)
        self._kwargs = kwargs
Ejemplo n.º 6
0
    def __init__(self,
                 api_key: Optional[str] = None,
                 save_dir: Optional[str] = None,
                 workspace: Optional[str] = None,
                 project_name: Optional[str] = None,
                 rest_api_key: Optional[str] = None,
                 experiment_name: Optional[str] = None,
                 experiment_key: Optional[str] = None,
                 **kwargs):

        super().__init__()
        self._experiment = None

        # Determine online or offline mode based on which arguments were passed to CometLogger
        if api_key is not None:
            self.mode = "online"
            self.api_key = api_key
        elif save_dir is not None:
            self.mode = "offline"
            self.save_dir = save_dir
        else:
            # If neither api_key nor save_dir are passed as arguments, raise an exception
            raise MisconfigurationException(
                "CometLogger requires either api_key or save_dir during initialization."
            )

        log.info(f"CometLogger will be initialized in {self.mode} mode")

        self.workspace = workspace
        self.project_name = project_name
        self.experiment_key = experiment_key
        self._kwargs = kwargs

        if rest_api_key is not None:
            # Comet.ml rest API, used to determine version number
            self.rest_api_key = rest_api_key
            self.comet_api = API(self.rest_api_key)
        else:
            self.rest_api_key = None
            self.comet_api = None

        if experiment_name:
            try:
                self.name = experiment_name
            except TypeError as e:
                log.exception(
                    "Failed to set experiment name for comet.ml logger")
        self._kwargs = kwargs
Ejemplo n.º 7
0
    def __init__(self,
                 api_key: Optional[str] = None,
                 save_dir: Optional[str] = None,
                 workspace: Optional[str] = None,
                 project_name: Optional[str] = None,
                 rest_api_key: Optional[str] = None,
                 experiment_name: Optional[str] = None,
                 experiment_key: Optional[str] = None,
                 **kwargs):
        r"""

        Requires either an API Key (online mode) or a local directory path (offline mode)

        .. code-block:: python

            # ONLINE MODE
            from pytorch_lightning.loggers import CometLogger
            # arguments made to CometLogger are passed on to the comet_ml.Experiment class
            comet_logger = CometLogger(
                api_key=os.environ["COMET_API_KEY"],
                workspace=os.environ["COMET_WORKSPACE"], # Optional
                project_name="default_project", # Optional
                rest_api_key=os.environ["COMET_REST_API_KEY"], # Optional
                experiment_name="default" # Optional
            )
            trainer = Trainer(logger=comet_logger)

        .. code-block:: python

            # OFFLINE MODE
            from pytorch_lightning.loggers import CometLogger
            # arguments made to CometLogger are passed on to the comet_ml.Experiment class
            comet_logger = CometLogger(
                save_dir=".",
                workspace=os.environ["COMET_WORKSPACE"], # Optional
                project_name="default_project", # Optional
                rest_api_key=os.environ["COMET_REST_API_KEY"], # Optional
                experiment_name="default" # Optional
            )
            trainer = Trainer(logger=comet_logger)

        Args:
            api_key (str): Required in online mode. API key, found on Comet.ml
            save_dir (str): Required in offline mode. The path for the directory to save local comet logs
            workspace (str): Optional. Name of workspace for this user
            project_name (str): Optional. Send your experiment to a specific project.
            Otherwise will be sent to Uncategorized Experiments.
            If project name does not already exists Comet.ml will create a new project.
            rest_api_key (str): Optional. Rest API key found in Comet.ml settings.
                This is used to determine version number
            experiment_name (str): Optional. String representing the name for this particular experiment on Comet.ml.
            experiment_key (str): Optional. If set, restores from existing experiment.
        """
        super().__init__()
        self._experiment = None

        # Determine online or offline mode based on which arguments were passed to CometLogger
        if api_key is not None:
            self.mode = "online"
            self.api_key = api_key
        elif save_dir is not None:
            self.mode = "offline"
            self.save_dir = save_dir
        else:
            # If neither api_key nor save_dir are passed as arguments, raise an exception
            raise MisconfigurationException("CometLogger requires either api_key or save_dir during initialization.")

        log.info(f"CometLogger will be initialized in {self.mode} mode")

        self.workspace = workspace
        self.project_name = project_name
        self.experiment_key = experiment_key
        self._kwargs = kwargs

        if rest_api_key is not None:
            # Comet.ml rest API, used to determine version number
            self.rest_api_key = rest_api_key
            self.comet_api = API(self.rest_api_key)
        else:
            self.rest_api_key = None
            self.comet_api = None

        if experiment_name:
            try:
                self.name = experiment_name
            except TypeError as e:
                log.exception("Failed to set experiment name for comet.ml logger")
        self._kwargs = kwargs
Ejemplo n.º 8
0
    "--get_optimizer_weights",
    action="store_true",
    help=
    "Also grab the state dictionary of the optimizer at the end of the specified epoch"
)

#XMUs9uI19KQNdYrQhuXPnAfpB
args = parser.parse_args()
experiment_key = args.experiment_key
epoch_number = args.epoch_number
apikey = args.api_key
get_optimizer_weights = args.get_optimizer_weights
output_directory = os.path.join(args.output_directory, experiment_key)
if not os.path.isdir(output_directory):
    os.makedirs(output_directory)
api = API(api_key=apikey)

experiment: APIExperiment = api.get_experiment("electric-turtle",
                                               "probabilisticbeziercurves",
                                               experiment_key)
assetlist = experiment.get_asset_list()
assetdict = {d['fileName']: d['assetId'] for d in assetlist}
#lookahead_indices = int(experiment.get_parameters_summary(parameter="lookahead_indices")["valueCurrent"])

print("Getting hyperparameters from comet")
datasets_file_name = "datasets.yaml"
experiment_file_name = "experiment_config.yaml"
config_file_name = "model_config.yaml"
config_yaml = experiment.get_asset(assetdict[config_file_name],
                                   return_type="text")
config = yaml.load(io.StringIO(config_yaml), Loader=yaml.SafeLoader)
projectdict = {
    "cnnlstm": "deepracingcnnlstm",
    "bezier": "deepracingbezierpredictor",
    "pilot": "deepracingpilotnet"
}
#XMUs9uI19KQNdYrQhuXPnAfpB
args = parser.parse_args()
experiment_key = args.experiment_key
epoch_number = args.epoch_number
restkey = args.restkey
output_directory = os.path.join(args.output_directory,
                                experiment_key + "_from_comet")
if not os.path.isdir(output_directory):
    os.makedirs(output_directory)
if restkey is None:
    api = API()
else:
    api = API(rest_api_key=restkey)
experiment: APIExperiment = api.get_experiment("electric-turtle",
                                               "deepracingposepredictor",
                                               experiment_key)
assetlist = experiment.get_asset_list()
assetdict = {d['fileName']: d['assetId'] for d in assetlist}

#get network weight file
weightfilename = "epoch_%d_params.pt" % (epoch_number, )
optimizerfilename = "epoch_%d_optimizer.pt" % (epoch_number, )

print("Getting network weights from comet")
params_binary = experiment.get_asset(assetdict[weightfilename])
Ejemplo n.º 10
0
    def __init__(self,
                 api_key: Optional[str] = None,
                 save_dir: Optional[str] = None,
                 workspace: Optional[str] = None,
                 project_name: Optional[str] = None,
                 rest_api_key: Optional[str] = None,
                 experiment_name: Optional[str] = None,
                 experiment_key: Optional[str] = None,
                 force_offline: bool = False,
                 **kwargs):

        if not _COMET_AVAILABLE:
            raise ImportError(
                'You want to use `comet_ml` logger which is not installed yet,'
                ' install it with `pip install comet-ml`.')
        super().__init__()
        self._experiment = None
        self.save_dir = save_dir
        self.api_key = api_key

        # Determine online or offline mode based on which arguments were passed to CometLogger
        if api_key is None and save_dir is None:
            # If neither api_key nor save_dir are passed as arguments, raise an exception
            raise MisconfigurationException(
                "CometLogger requires either api_key or save_dir during initialization."
            )
        elif api_key is None:
            self.mode = "offline"
        elif save_dir is None:
            self.mode = "online"
        else:  # both given so need explicit argument
            self.mode = "offline" if force_offline else "online"

        log.info(f"CometLogger will be initialized in {self.mode} mode")

        self.workspace = workspace
        self.project_name = project_name
        self.experiment_name = experiment_name
        self.experiment_key = experiment_key
        self._kwargs = kwargs

        if rest_api_key is not None:
            # Comet.ml rest API, used to determine version number
            self.rest_api_key = rest_api_key
            self.comet_api = API(self.rest_api_key)
        else:
            self.rest_api_key = None
            self.comet_api = None

        if self.experiment_name is None:
            self._version = self.experiment.id
        else:
            # ensure that the directory name is unique by appending a number to the end
            root_save_dir = Path(self.save_dir) / self.name
            root_save_dir.mkdir(exist_ok=True, parents=True)
            max_i = -1
            for path in root_save_dir.glob("*"):
                name = path.name
                if name.startswith(self.experiment_name):
                    try:
                        i = int(name.split("_")[-1])
                        max_i = max(max_i, i)
                    except (IndexError, ValueError):  # no _ or no int at end
                        continue
            self._version = f"{self.experiment_name}_{max_i + 1}"

        self._kwargs = kwargs
Ejemplo n.º 11
0
class fetcher:
    '''
    An Interface for fetching experiment results through comet API.
    
    Example
    -------
    >>>  y = fetcher(project=["crossa-wheat-env-0", "crossa-wheat-env-1"])
    >>>  results = y.get_results(parameters = ["run_arg_9", "model", "coding"])
    '''
    
    def __init__(self, workspace="dna-i", project=None):
        '''
        Initializes API key and optionally which workspace and projects to use

        Parameters
        ----------
        workspace : STR, optional
            Comet workspace. The default is "dna-i".
        project : list(STR), optional
            List of Projects to use. The default is None.

        Returns
        -------
        None.

        '''
        self.workspace = workspace
        self.project = project
        self.api = API(api_key="V0OXnWOi4KVNS4OkwLjdnxSgK")
        
    def get_all_projects(self, workspace="dna-i"):
        '''
        Gets and prints all available projects.

        Parameters
        ----------
        workspace : STR, optional
            The default is "dna-i".

        Returns
        -------
        LIST(STR)
            available projects

        '''
        self.available_projects = self.api.get(workspace)
        print(self.available_projects)
        return self.available_projects
    
    def get_all_experiments(self, project=None):
        '''
        Fetches all experiments for a given project

        Parameters
        ----------
        project : STR, optional
             The default is None.

        Returns
        -------
        COMET EXPERIMENT OBJ

        '''
        if project is not None:
            self.experiments = self.api.get(f"{self.workspace}/{project}")
            print("fetching experiments")
        else:
            experiment = []            
            for proj in self.available_projects:
                elem = {'project': proj,
                        'experiments':self.api.get(f"{self.workspace}/{proj}")}
                experiment.append(elem)
            self.experiments = experiment
        return self.experiments
    
    def get_exp_keys(self):
        '''
        Converts list of comet experiment objects to experiment keys.
        
        Returns
        -------
        list of str

        '''
        self.keys = []
        print("fetching experiment keys")
        for exp in self.experiments:
            self.keys.append(exp.get_metadata()['experimentKey'])                
        return self.keys
    
    def result_to_df(self):
        '''
        Converts list of dictionaries result created with get_results to
        human friendly to a pandas dataframe.

        Returns
        -------
        DataFrame

        '''
        r = []
        for p in self.results:  
            result = p["metrics"]
            project = p["project"]
            for k,v in result.items():
                d = {}
                d["database"] = project
                d['experimentKey']=v["experimentKey"]
                d = {**d, **v['params']}
                for metric in v["metrics"]:
                    d['metricName']= metric["metricName"]
                    for value, step  in zip(metric["values"], metric["epochs"]):
                        aux= { **d, 'value': value,'step': step,}
                        r.append(aux)
            self.results = pd.DataFrame(r)
    
    def get_results(self, metrics=["r_test (r)", "mse_test", "r_train (r)",
                                   "mse_train", "r_test (r)_mean", "mse_test_mean"],
                    parameters=["model", "coding"], project=None):
        '''
        Fetches experiment metrics and parameters

        Parameters
        ----------
        metrics : LIST(STR), optional
            Which metrics to retrieve. The default is ["r_test (r)", "mse_test", "r_train (r)",                                   "mse_train", "r_test (r)_mean", "mse_test_mean"].
        parameters : LIST(STR), optional
            Which param to retrieve. The default is ["model", "coding"].
        project : LIST(STR), optional
            Which projects to retriever data from. The default is None.

        Returns
        -------
        Pandas dataframe with results
        '''
        print(f"Fetching {metrics} and {parameters}")
        if project is None:
            project = self.project
        self.results = []
        for proj in project:
            print(f"Project {proj}")
            self.get_all_experiments(project=proj)
            self.get_exp_keys()
            value = self.api.get_metrics_for_chart(self.keys, full=True, metrics = metrics, parameters=parameters)
            self.results.append({"project": proj, "metrics":value })
        self.result_to_df()
        return self.results
Ejemplo n.º 12
0
parser = argparse.ArgumentParser(description="Download experiment from Comet.ml")
parser.add_argument("experiment_key", type=str, help="Experiment key to grab from comet.")
parser.add_argument("--restkey", type=str, required=False, default=None, help="Experiment key to grab from comet.")
parser.add_argument("--epoch_number", type=int, required=False, default=100, help="Experiment key to grab from comet.")
parser.add_argument("--output_directory", type=str, required=False, default=".", help="Where to put the config and data files.")

#XMUs9uI19KQNdYrQhuXPnAfpB
args = parser.parse_args()
experiment_key = args.experiment_key
epoch_number = args.epoch_number
restkey = args.restkey
output_directory = os.path.join(args.output_directory, experiment_key + "_from_comet")
if not os.path.isdir(output_directory):
    os.makedirs(output_directory)
if restkey is None:
    api = API()
else:
    api = API(rest_api_key=restkey)
experiment : APIExperiment = api.get_experiment("electric-turtle", "deepracingpilotnet", experiment_key)
assetlist = experiment.get_asset_list()
assetdict = {d['fileName']: d['assetId'] for d in assetlist}

#get network weight file
weightfilename = "pilotnet_epoch_%d_params.pt" %(epoch_number,)

print("Getting network weights from comet")
params_binary = experiment.get_asset(assetdict[weightfilename])


outputweightfile = os.path.join(output_directory,weightfilename)
with open(outputweightfile, 'wb') as f:
Ejemplo n.º 13
0
import comet_ml
comet_ml.config.save(api_key="WRmA8ms9A78K85fLxcv8Nsld9")
from comet_ml.api import API
import numpy as np
#import pandas as pd
import matplotlib.pyplot as plt

api = API()
exp_urls_lr1 = {
    "run1": "yasmeenvh/growspaceenv-ppo/91df8e59877e484f96022b6b3548fcf8",
    "run2": "yasmeenvh/growspaceenv-ppo/a633ea5fc5064259a013889cf46dfe64",
    "run3": "yasmeenvh/growspaceenv-ppo/b9d798bcf5d545c884162ac3373c8d8e"
}

# exp_urls_lr2 = {"run1":"https://www.comet.ml/yasmeenvh/growspaceenv-ppo/91df8e59877e484f96022b6b3548fcf8",
#                 "run2":"https://www.comet.ml/yasmeenvh/growspaceenv-ppo/a633ea5fc5064259a013889cf46dfe64",
#                 "run3":"https://www.comet.ml/yasmeenvh/growspaceenv-ppo/b9d798bcf5d545c884162ac3373c8d8e"}

comet_metrics = [
    "Reward Mean", "Reward Max", "Reward Min", "Reward Max",
    "Number of Mean Branches"
]


def get_data(url_link, comet_metrics):
    min_numframes = 0000000
    max_numframes = 100000000
    reward_means = []
    for run, link in url_link.items():
        experiment = api.get(link)
Ejemplo n.º 14
0
from comet_ml.api import API, APIExperiment
from config import comet_ml_key

comet_api = API(api_key=comet_ml_key)
for key in ["405", "301", "1000"]:
    exps = comet_api.get_experiments(
        "ericzhao28",
        project_name="active-label-shift-adaptation",
        pattern=".*" + key + ".*")
    for exp in exps:
        if exp.get_metadata()["archived"]:
            continue
        if key in exp.get_name():
            print(exp.get_name())
            x = comet_api.get("ericzhao28/active-label-shift-adaptation/" +
                              exp.get_metadata()["experimentKey"])
            for asset in x.get_asset_list():
                assetid = asset["assetId"]
                print(asset["fileName"])
                if "csv" not in asset["fileName"]:
                    continue
                with open("../data/" + exp.get_name() + ".csv", "w") as f:
                    f.write(exp.get_asset(asset_id=assetid,
                                          return_type="text"))
Ejemplo n.º 15
0
    def __init__(self,
                 api_key=None,
                 save_dir=None,
                 workspace=None,
                 rest_api_key=None,
                 project_name=None,
                 experiment_name=None,
                 **kwargs):
        """Initialize a Comet.ml logger.
        Requires either an API Key (online mode) or a local directory path (offline mode)

        :param str api_key: Required in online mode. API key, found on Comet.ml
        :param str save_dir: Required in offline mode. The path for the directory to save local comet logs
        :param str workspace: Optional. Name of workspace for this user
        :param str project_name: Optional. Send your experiment to a specific project.
        Otherwise will be sent to Uncategorized Experiments.
        If project name does not already exists Comet.ml will create a new project.
        :param str rest_api_key: Optional. Rest API key found in Comet.ml settings.
        This is used to determine version number
        :param str experiment_name: Optional. String representing the name for this particular experiment on Comet.ml
        """
        super().__init__()
        self._experiment = None

        # Determine online or offline mode based on which arguments were passed to CometLogger
        if save_dir is not None and api_key is not None:
            # If arguments are passed for both save_dir and api_key, preference is given to online mode
            self.mode = "online"
            self.api_key = api_key
        elif api_key is not None:
            self.mode = "online"
            self.api_key = api_key
        elif save_dir is not None:
            self.mode = "offline"
            self.save_dir = save_dir
        else:
            # If neither api_key nor save_dir are passed as arguments, raise an exception
            raise MisconfigurationException(
                "CometLogger requires either api_key or save_dir during initialization."
            )

        logger.info(f"CometLogger will be initialized in {self.mode} mode")

        self.workspace = workspace
        self.project_name = project_name
        self._kwargs = kwargs

        if rest_api_key is not None:
            # Comet.ml rest API, used to determine version number
            self.rest_api_key = rest_api_key
            self.comet_api = API(self.rest_api_key)
        else:
            self.rest_api_key = None
            self.comet_api = None

        if experiment_name:
            try:
                self.name = experiment_name
            except TypeError as e:
                logger.exception(
                    "Failed to set experiment name for comet.ml logger")