コード例 #1
0
def retrieve_experiment(exp_id, sync_if_missing=True, verbose=False, force_update=False):
    """
    Retrieve an experiment in the local database based on its id.
    Can call a sync if missing if the flag is true.
    :param exp_id: Id of the experiment to retrieve
    :param sync_if_missing: Should we try to sync if not present?
    :return: The experiment found
    """
    if not exp_id: raise Exception("Trying to retrieve an experiment without providing an experiment ID")
    from uuid import UUID
    if isinstance(exp_id,UUID): exp_id = str(exp_id)

    # If we dont force the update -> look first in the DB
    exp = DataStore.get_experiment(exp_id) or DataStore.get_most_recent_experiment(exp_id)
    if exp:
        # If we have an experiment and we want to force the update -> delete it
        if not force_update:
            return exp
        else:
            DataStore.delete_experiment(exp)

    if not sync_if_missing:
        raise Exception('Experiment %s not found in the local database and sync disabled.' % exp_id)

    logger.info('Experiment with id %s not found in local database, trying sync.' % exp_id)
    with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
        endpoint = sp.get('server_endpoint')

    exp = COMPS_experiment_to_local_db(exp_id, endpoint, verbose)

    if exp: return exp
    raise Exception("Experiment '%s' could not be retrieved." % exp_id)
コード例 #2
0
 def hard_delete(self):
     """
     Delete data for experiment and marks the server entity for deletion.
     """
     # Mark experiment for deletion in COMPS.
     COMPS_login(self.endpoint)
     self.comps_experiment.delete()
     # Delete in the DB
     from simtools.DataAccess.DataStore import DataStore
     DataStore.delete_experiment(self.experiment)
コード例 #3
0
    def hard_delete(self):
        """
        Delete experiment and output data.
        """
        # Delete local simulation data.
        exp_path = self.experiment.get_path()
        if os.path.exists(exp_path):
            try:
                shutil.rmtree(exp_path)
            except Exception as e:
                print("Could not delete path: {}\nReason: {}".format(exp_path,e))

        # Delete in the DB
        from simtools.DataAccess.DataStore import DataStore
        DataStore.delete_experiment(self.experiment)
コード例 #4
0
def retrieve_object(obj_id, obj_type, sync_if_missing=True, verbose=False, force_update=False):
    """
        Retrieve an object (Experiment, Simulation) in the local database based on its id.
        Can call a sync if missing if the flag is true
        :param obj_id: Id of the object to retrieve
        :param obj_type: Type of the object to retrieve
        :param sync_if_missing: Should we try to sync if not present?
        :return: The experiment found

        #TODO: SHould also support suites
        """
    typename = obj_type.__name__
    if not obj_id: raise ValueError("Trying to retrieve an object (%s) without providing an ID" % typename)

    # Try a hit in the DB
    if obj_type == Experiment:
        obj = DataStore.get_experiment(obj_id)
    elif obj_type == Simulation:
        obj = DataStore.get_simulation(obj_id)

    if obj:
        # If we have an experiment and we want to force the update -> delete it
        if not force_update:
            return obj
        else:
            if obj_type == Experiment:
                DataStore.delete_experiment(obj_id)
            elif obj_type == Simulation:
                DataStore.delete_simulation(obj_id)

    if not sync_if_missing:
        raise Exception('%s %s not found in the local database and sync disabled.' % (typename, obj_id))

    logger.info('%s with id %s not found in local database, trying sync.' % (typename, obj_id))
    with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
        endpoint = sp.get('server_endpoint')

    if obj_type == Experiment:
        obj = COMPS_experiment_to_local_db(obj_id, endpoint, verbose)
    elif obj_type == Simulation:
        obj =None

    if obj: return obj
    raise Exception("%s '%s' could not be retrieved." % (typename, obj_id))
コード例 #5
0
ファイル: commands.py プロジェクト: m-v-nikolov/dtk-tools-py3
def sync(args, unknownArgs):
    """
    Sync COMPS db with local db
    """
    # Create a default HPC setup parser
    with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
        endpoint = sp.get('server_endpoint')
        user = sp.get('user')
        COMPS_login(endpoint)

    exp_to_save = list()
    exp_deleted = 0

    # Retrieve all the experiment id from COMPS for the current user
    exp_ids = get_experiment_ids_for_user(user)

    # Test the experiments present in the local DB to make sure they still exist in COMPS
    for exp in DataStore.get_experiments():
        if exp.location == "HPC":
            if exp.exp_id not in exp_ids:
                # The experiment doesnt exist on COMPS anymore -> delete from local
                DataStore.delete_experiment(exp)
                exp_deleted += 1

    # Consider experiment id option
    exp_id = args.exp_id if args.exp_id else None
    exp_name = args.exp_name if args.exp_name else None
    user = args.user if args.user else user

    if exp_name:
        experiments = get_experiments_by_name(exp_name, user)
        for experiment_data in experiments:
            experiment = COMPS_experiment_to_local_db(
                exp_id=str(experiment_data.id),
                endpoint=endpoint,
                verbose=True,
                save_new_experiment=False)
            if experiment:
                exp_to_save.append(experiment)

    elif exp_id:
        # Create a new experiment
        experiment = COMPS_experiment_to_local_db(exp_id=exp_id,
                                                  endpoint=endpoint,
                                                  verbose=True,
                                                  save_new_experiment=False)
        # The experiment needs to be saved
        if experiment:
            exp_to_save.append(experiment)
    else:
        # By default only get experiments created in the last month
        # day_limit = args.days if args.days else day_limit_default
        day_limit = 30
        today = datetime.date.today()
        limit_date = today - datetime.timedelta(days=int(day_limit))

        # For each of them, check if they are in the db
        for exp in get_experiments_per_user_and_date(user, limit_date):
            # Create a new experiment
            experiment = COMPS_experiment_to_local_db(
                exp_id=str(exp.id),
                endpoint=endpoint,
                save_new_experiment=False)

            # The experiment needs to be saved
            if experiment:
                exp_to_save.append(experiment)

    # Save the experiments if any
    if len(exp_to_save) > 0 and exp_deleted == 0:
        DataStore.batch_save_experiments(exp_to_save)
        logger.info(
            "The following experiments have been added to the database:")
        logger.info("\n".join(["- " + str(exp) for exp in exp_to_save]))
        logger.info("%s experiments have been updated in the DB." %
                    len(exp_to_save))
        logger.info("%s experiments have been deleted from the DB." %
                    exp_deleted)
    else:
        print("The database was already up to date.")

    # Start overseer
    BaseExperimentManager.check_overseer()