예제 #1
0
    def get(self, exp_id):
        """ Get a summary, consisting of:
            - The number of get_action calls
            - The date of the last get_action call
            - The number of set_reward calls
            - The date of the last set_reward call

        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        | http://example.com/stats/EXP_ID/summary                            |
        +--------------------------------------------------------------------+

        :requires: A secure cookie, obtained by logging in.
        :param int exp_id: The experiment ID for the summary that are to be retrieved.
        :returns: A JSON object consisting of the summary.
        :raises 401: If the experiment does not belong to this user or the exp_id is wrong.
        :raises 401: If user is not logged in or if there is no secure cookie available.
        """
        if self.get_current_user():
            if self.validate_user_experiment(exp_id):
                exp = Experiment(exp_id)
                response = exp.get_summary()
                self.write(json.dumps(response))
            else:
                raise ExceptionHandler(
                    reason="Experiment could not be validated.",
                    status_code=401)
        else:
            raise ExceptionHandler(reason="Could not validate user.",
                                   status_code=401)
예제 #2
0
    def get(self, exp_id):
        """ Get the current theta.
        
        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        | http://example.com/stats/EXP_ID/currenttheta                       |
        +--------------------------------------------------------------------+

        :requires: A secure cookie, obtained by logging in.
        :param int exp_id: The experiment ID for the theta that is to be retrieved.
        :returns: A JSON of the current theta.
        :raises 401: If the experiment does not belong to this user or the exp_id is wrong.
        :raises 401: If user is not logged in or if there is no secure cookie available.
        """
        if self.get_current_user():
            if self.validate_user_experiment(exp_id):
                exp = Experiment(exp_id)
                response = exp.get_theta()
                self.write(json.dumps(response))
            else:
                raise ExceptionHandler(
                    reason="Experiment could not be validated.",
                    status_code=401)
        else:
            raise ExceptionHandler(reason="Could not validate user.",
                                   status_code=401)
예제 #3
0
 def get(self, exp_id):
     """ Get an action given a context for a specific exp_id
     
     Input arguments:
     exp_id: Experiment ID as specified in the url
     context: string of JSON object which obtains the context. This is assumed to be passed in the query string 
     key: part of the JSON object        
     
     Returns:
     A JSON object containing "action": XX
     Or an object containing "error": ...
     """
     key = self.get_argument("key", default = False)
     
     if not key:
         self.set_status(401)
         self.write("invalid key")
         return
     
     __EXP__ = Experiment(exp_id, key)
     
     if __EXP__.is_valid():
         context = json.loads(self.get_argument("context", default="{}"))    
         response = __EXP__.run_action_code(context)
         
         if self.settings['debug']:
             self.write(json.dumps({'action':response, 'context':context}))
         else:
             self.write(json.dumps({'action':response}))
             
     else:
         self.set_status(401)       # Needs proper error handling
         self.write("invalid key")
         return
예제 #4
0
    def get(self, exp_id):
        """ Update the parameters for a given experiment

        +----------------------------------------------------------------+
        | Example                                                        |
        +================================================================+
        |http://example.com/1/setReward.json?key=XXXX&context={'age': 25}|
        |&action={'action':'A'}&reward={'click':1}                       |
        +----------------------------------------------------------------+

        :param int exp_id: Experiment ID as specified in the url
        :param JSON context: The context to train on.
        :param JSON action: The action to train on.
        :param JSON reward: The reward for the experiment.
        :param string key: The key corresponding to the experiment.

        :returns: A JSON of the form: {"status":true}
        :raises KeyError: 400 Error if Key is not valid.
        """
        key = self.get_argument("key", default = False)
        __EXP__ = Experiment(exp_id, key)

        if __EXP__.is_valid():
            context = json.loads(self.get_argument("context", default="{}"))
            action = json.loads(self.get_argument("action", default="{}"))
            reward = json.loads(self.get_argument("reward", default="{}"))
            
            __EXP__.run_reward_code(context, action, reward)
            
            if self.settings['debug']:
                self.write(json.dumps({'status':'success', 'action':action,'context':context, 'reward':reward}))
            else: 
                self.write(json.dumps({'status':'success'}))
        else:
            self.write_error(400) # Needs proper error handling
예제 #5
0
    def get(self, exp_id):
        """ Get all the (manually) logged data.

        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        | http://example.com/stats/EXP_ID/log?limit=10                       |
        +--------------------------------------------------------------------+

        :requires: A secure cookie, obtained by logging in.
        :param int exp_id: The experiment ID for the logs that are to be retrieved.
        :param int limit (optional): Set an optional limit to the amount of logs returned.
        :returns: A list of JSONs of the logs.
        :raises 401: If the experiment does not belong to this user or the exp_id is wrong.
        :raises 401: If user is not logged in or if there is no secure cookie available.
        """
        if self.get_current_user():
            if self.validate_user_experiment(exp_id):
                limit = int(self.get_argument("limit", default=0))
                exp = Experiment(exp_id)
                response = exp.get_log_data(limit=limit)
                self.write(json.dumps(response))
            else:
                raise ExceptionHandler(
                    reason="Experiment could not be validated.",
                    status_code=401)
        else:
            raise ExceptionHandler(reason="Could not validate user.",
                                   status_code=401)
예제 #6
0
    def get(self, exp_id):
        """ Update the parameters for a given experiment

        Input arguments:
        exp_id: Experiment ID as specified in the url
        context: in JSON get
        action: in JSON get
        reward: in JSON get
        key: in JSON get

        Returns:
        A JSON object containing "status":true
        Or an object containing "error": ...
        """
        key = self.get_argument("key", default = False)
        __EXP__ = Experiment(exp_id, key)

        if __EXP__.is_valid():
            context = json.loads(self.get_argument("context", default="{}"))
            action = json.loads(self.get_argument("action", default="{}"))
            reward = float(self.get_argument("reward", default=0))
            
            __EXP__.run_reward_code(context, action, reward)
            
            if self.settings['debug']:
                self.write(json.dumps({'status':'success', 'action':action,'context':context, 'reward':reward}))
            else: 
                self.write(json.dumps({'status':'success'}))
                
        else:
            self.write_error(400)       # Needs proper error handling
예제 #7
0
    def get(self, exp_id):

        key = self.get_argument("key", default = False)
        theta_key = self.get_argument("theta_key", default = False)
        theta_value = self.get_argument("theta_value", default = False)
        __EXP__ = Experiment(exp_id, key)

        if __EXP__.is_valid():
            __EXP__.delete_theta(key = theta_key, value = theta_value)

            self.write(json.dumps({'status':'success'}))
        else:
            self.write_error(400)
예제 #8
0
def log_theta():
    """ For every experiment, if Theta logging flag is set. Log theta from
    redis to mongodb.

    """
    redis_db = Database()
    mongo_db = MongoLog()
    experiment_ids = redis_db.get_experiment_ids()
    for experiment_id in experiment_ids:
        exp = Experiment(experiment_id)
        if exp.properties["hourly_theta"] == "True":
            theta = exp.get_theta()
            theta["exp_id"] = experiment_id
            mongo_db.log_hourly_theta(theta)
예제 #9
0
def log_theta():
    """ For every experiment, if Theta logging flag is set. Log theta from
    redis to mongodb.

    """
    redis_db = Database()
    mongo_db = MongoLog()
    experiment_ids = redis_db.get_experiment_ids()
    for experiment_id in experiment_ids:
        exp = Experiment(experiment_id)
        properties = redis_db.get_one_experiment(experiment_id)
        if properties['hourlyTheta'] == "True":
            theta = exp.get_theta()
            theta['exp_id'] = experiment_id
            mongo_db.log_hourly_theta(theta)
            print("We did it, we stored some stuff!")
예제 #10
0
def log_theta():
    """ For every experiment, if Theta logging flag is set. Log theta from
    redis to mongodb.

    """
    redis_db = Database()
    mongo_db = MongoLog()
    experiment_ids = redis_db.get_experiment_ids()
    for experiment_id in experiment_ids:
        exp = Experiment(experiment_id)
        properties = redis_db.get_one_experiment(experiment_id)
        if properties['hourlyTheta'] == "True":
            theta = exp.get_theta()
            theta['exp_id'] = experiment_id
            mongo_db.log_hourly_theta(theta)
            print("We did it, we stored some stuff!")
예제 #11
0
    def get(self, exp_id):
        """ Reset the theta of an experiment.

        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        | http://example.com/exp/EXP_ID/resetexperiment?key=KEY              |
        | &theta_key=THETA_KEY&theta_value=THETA_VALUE                       |
        +--------------------------------------------------------------------+

        :requires: A secure cookie obtained by logging in.
        :param int exp_id: The experiment ID.
        :param string key: The key of the experiment.
        :param string theta_key (optional): The key for theta used when setting \
        theta in the setReward and getAction code.
        :param string theta_value (optional): The value for theta used when \
        setting theta in the setReward and getAction code.
        :raises 401: If the theta_key or theta_value does not exist or is not valid.
        :raises 401: If the experiment does not belong to this user or the exp_id is wrong.
        :raises 401: If user is not logged in or if there is no secure cookie available.
        """
        if self.get_secure_cookie("user"):
            if self.validate_user_experiment(exp_id):
                key = self.get_argument("key", default=False)
                theta_key = self.get_argument("theta_key", default=None)
                theta_value = self.get_argument("theta_value", default=None)
                __EXP__ = Experiment(exp_id, key)

                status = __EXP__.delete_theta(key=theta_key, value=theta_value)
                if status >= 1:
                    self.write(json.dumps({'status': 'success'}))
                else:
                    raise ExceptionHandler(
                        reason=
                        "Theta_key or theta_value could not be validated.",
                        status_code=401)
            else:
                raise ExceptionHandler(
                    reason="Experiment could not be validated.",
                    status_code=401)
        else:
            raise ExceptionHandler(reason="Could not validate user.",
                                   status_code=401)
예제 #12
0
    def get(self, exp_id):
        """ Get an action given a context for a specific exp_id
        
        +----------------------------------------------------------------+
        | Example                                                        |
        +================================================================+
        |http://example.com/1/getAction.json?key=XXXX&context={'age': 25}|
        +----------------------------------------------------------------+
        
        
        :param int exp_id: Experiment ID as specified in the url
        :param JSON context: The context to be evaluated. 
        :param string key: The key corresponding to the experiment.
        
        :returns: A JSON of the form: {"action": XX}
        :raises AuthErorr: 401 Invalid key
        """
        key = self.get_argument("key", default=False)

        if not key:
            self.set_status(401)
            self.write("invalid key")
            return

        __EXP__ = Experiment(exp_id, key)

        if __EXP__.is_valid():
            context = json.loads(self.get_argument("context", default="{}"))
            response = __EXP__.run_action_code(context)

            if self.settings['debug']:
                self.write(json.dumps({
                    'action': response,
                    'context': context
                }))
            else:
                self.write(json.dumps({'action': response}))

        else:
            self.set_status(401)  # Needs proper error handling
            self.write("invalid key")
            return
예제 #13
0
    def get(self, exp_id):
        """ Get a dict of all logged thetas

        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        | http://example.com/stats/1/getHourlyTheta.json                     |
        +--------------------------------------------------------------------+
        
        :requires: A secure cookie, obtained by logging in.
        :param int exp_id: The experiment ID for the thetas that are to be retrieved.
        :returns: A JSON of JSONs of hourly logged thetas.
        :raises: AUTH_ERROR if there is no secure cookie available.
        """
        if self.get_secure_cookie("user"):
            exp = Experiment(exp_id)
            response = exp.get_hourly_theta()
            self.write(json.dumps(response))
        else:
            self.write("AUTH_ERROR")
예제 #14
0
    def get(self, exp_id):
        """ Get the current theta for experiment id exp_id
        
        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        | http://example.com/stats/1/getCurrentTheta.json                    |
        +--------------------------------------------------------------------+

        :requires: A secure cookie, obtained by logging in.
        :param int exp_id: The experiment ID for the theta that is to be retrieved.
        :returns: A JSON of current theta.
        :raises: AUTH_ERROR if there is no secure cookie available.
        """
        if self.get_secure_cookie("user"):
            exp = Experiment(exp_id)
            response = exp.get_theta()
            self.write(json.dumps(response))
        else:
            self.write("AUTH_ERROR")
예제 #15
0
    def get(self, exp_id):
        """ Get the current theta for experiment id exp_id
        
        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        | http://example.com/stats/1/getCurrentTheta.json                    |
        +--------------------------------------------------------------------+

        :requires: A secure cookie, obtained by logging in.
        :param int exp_id: The experiment ID for the theta that is to be retrieved.
        :returns: A JSON of current theta.
        :raises: AUTH_ERROR if there is no secure cookie available.
        """
        if self.get_secure_cookie("user"):
            exp = Experiment(exp_id)
            response = exp.get_theta()
            self.write(json.dumps(response))
        else:
            self.write("AUTH_ERROR")
예제 #16
0
    def get(self, exp_id):
        """ Get a dict of all logged thetas

        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        | http://example.com/stats/1/getHourlyTheta.json                     |
        +--------------------------------------------------------------------+
        
        :requires: A secure cookie, obtained by logging in.
        :param int exp_id: The experiment ID for the thetas that are to be retrieved.
        :returns: A JSON of JSONs of hourly logged thetas.
        :raises: AUTH_ERROR if there is no secure cookie available.
        """
        if self.get_secure_cookie("user"):
            exp = Experiment(exp_id)
            response = exp.get_hourly_theta()
            self.write(json.dumps(response))
        else:
            self.write("AUTH_ERROR")
예제 #17
0
    def get(self, exp_id):
        """ Get all the (manually) logged data for experiment exp_id.

        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        | http://example.com/stats/1/getLog.json                             |
        +--------------------------------------------------------------------+

        :requires: A secure cookie, obtained by logging in.
        :param int exp_id: The experiment ID for the logs that are to be retrieved.
        :returns: A JSON of JSONs of the logs.
        :raises: AUTH_ERROR if there is no secure cookie available.
        """
        if self.get_secure_cookie("user"):
            exp = Experiment(exp_id)
            response = exp.get_log_data()
            self.write(json.dumps(response))
        else:
            self.write("AUTH_ERROR")
예제 #18
0
    def get(self, exp_id):
        """ Get all the (manually) logged data for experiment exp_id.

        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        | http://example.com/stats/1/getLog.json                             |
        +--------------------------------------------------------------------+

        :requires: A secure cookie, obtained by logging in.
        :param int exp_id: The experiment ID for the logs that are to be retrieved.
        :returns: A JSON of JSONs of the logs.
        :raises: AUTH_ERROR if there is no secure cookie available.
        """
        if self.get_secure_cookie("user"):
            exp = Experiment(exp_id)
            response = exp.get_log_data()
            self.write(json.dumps(response))
        else:
            self.write("AUTH_ERROR")
예제 #19
0
    def get(self, exp_id):
        """ Update the parameters for a given experiment

        +----------------------------------------------------------------+
        | Example                                                        |
        +================================================================+
        |http://example.com/1/setReward.json?key=XXXX&context={'age': 25}|
        |&action={'action':'A'}&reward={'click':1}                       |
        +----------------------------------------------------------------+

        :param int exp_id: Experiment ID as specified in the url
        :param JSON context: The context to train on.
        :param JSON action: The action to train on.
        :param JSON reward: The reward for the experiment.
        :param string key: The key corresponding to the experiment.

        :returns: A JSON of the form: {"status":true}
        :raises KeyError: 400 Error if Key is not valid.
        """
        key = self.get_argument("key", default=False)
        __EXP__ = Experiment(exp_id, key)

        if __EXP__.is_valid():
            context = json.loads(self.get_argument("context", default="{}"))
            action = json.loads(self.get_argument("action", default="{}"))
            reward = json.loads(self.get_argument("reward", default="{}"))

            __EXP__.run_reward_code(context, action, reward)

            if self.settings['debug']:
                self.write(
                    json.dumps({
                        'status': 'success',
                        'action': action,
                        'context': context,
                        'reward': reward
                    }))
            else:
                self.write(json.dumps({'status': 'success'}))
        else:
            self.write_error(400)  # Needs proper error handling
    def test_entity_name_escaped(self):
        experiment = Experiment.create_draft(
            name='test',
            variants=[
                Variant('anyvar', 100),
                ]).start()

        storage = DictStorage()
        service = AssignmentService(storage=storage)

        service.assign(experiment, 'name-with-colon:')
        assert_not_equal(':', storage.list()[0][-1])
예제 #21
0
 def get(self, exp_id):
     """ Get an action given a context for a specific exp_id
     
     +----------------------------------------------------------------+
     | Example                                                        |
     +================================================================+
     |http://example.com/1/getAction.json?key=XXXX&context={'age': 25}|
     +----------------------------------------------------------------+
     
     
     :param int exp_id: Experiment ID as specified in the url
     :param JSON context: The context to be evaluated. 
     :param string key: The key corresponding to the experiment.
     
     :returns: A JSON of the form: {"action": XX}
     :raises AuthErorr: 401 Invalid key
     """
     key = self.get_argument("key", default = False)
     
     if not key:
         self.set_status(401)
         self.write("invalid key")
         return
     
     __EXP__ = Experiment(exp_id, key)
     
     if __EXP__.is_valid():
         context = json.loads(self.get_argument("context", default="{}"))  
         response = __EXP__.run_action_code(context)
         
         if self.settings['debug']:
             self.write(json.dumps({'action':response, 'context':context}))
         else:
             self.write(json.dumps({'action':response}))
             
     else:
         self.set_status(401)       # Needs proper error handling
         self.write("invalid key")
         return
예제 #22
0
def advice_time_out():
    """ For every experiment, if the advice_id flag is set, we want to
    check whether certain advice_id's have timed out according to the
    experiment's own settings.
    """
    redis_db = Database()
    advice_db = Advice()
    experiment_ids = redis_db.get_experiment_ids()
    for experiment_id in experiment_ids:
        # Check experiment properties
        exp = Experiment(experiment_id)
        if exp.properties["advice_id"] == "True":
            # Get all the advices for this experiment
            # Check whether or not the date has exceeded the time-out rate
            delta_hours = int(exp.properties["delta_hours"])
            advices_retrieved = advice_db.advices.find({
                "date": {
                    "$lt": datetime.utcnow() - timedelta(hours=delta_hours)
                }
            })
            for adv in advices_retrieved:
                log = exp.get_by_advice_id(str(adv["_id"]))
                reward = ast.literal_eval(exp.properties["default_reward"])
                exp.run_reward_code(adv["context"], adv["action"], reward)
    def test_manual_assignment_overwrites_previous_assignment(self):
        ALLOCATION_100 = 'var1'
        ALLOCATION_0 = 'var2'
        ANY_ENTITY = 1

        experiment = Experiment.create_draft(
            name='test',
            variants=[
                Variant(ALLOCATION_100, 100),
                Variant(ALLOCATION_0, 0),
            ]).start()

        service = AssignmentService(storage=DictStorage())

        assert_equal(service.assign(experiment, ANY_ENTITY), ALLOCATION_100)
        service.manual_assign(experiment, ANY_ENTITY, ALLOCATION_0)

        assert_equal(service.assign(experiment, ANY_ENTITY), ALLOCATION_0)
    def test_when_variants_are_changed_existing_assignments_stay_the_same(self):
        START = 'var1'
        AFTER_UPDATE = 'var2'
        ANY_ENTITY = 1

        experiment = Experiment.create_draft(
            name='test',
            variants=[
                Variant(START, 100),
                Variant(AFTER_UPDATE, 0)
            ]).start()

        service = AssignmentService(storage=DictStorage())

        assert_equal(service.assign(experiment, ANY_ENTITY), START)

        experiment.update_variants([
            Variant(START, 0),
            Variant(AFTER_UPDATE, 100)
        ])

        assert_equal(service.assign(experiment, ANY_ENTITY), START)
예제 #25
0
],
            [
                "satellite", "wine_quality", "image_recognition", "ship",
                "knowledge_graph", "wikipedia_visualisation"
            ],
            [
                "knowledge_graph", "wikipedia_visualisation", "ship",
                "satellite", "wine_quality", "image_recognition"
            ]]
alternatives5 = {
    "knowledge_graph", "wikipedia_visualisation", "ship", "satellite",
    "wine_quality", "image_recognition"
}

profile6 = [["train", "bus", "airplane", "walk"],
            ["bus", "airplane", "train", "walk"],
            ["airplane", "train", "bus", "walk"]]
alternatives6 = {"train", "bus", "airplane", "walk"}

profile7 = [["ios", "windows", "android"], ["android", "windows", "ios"],
            ["windows", "ios", "android"]]
alternatives7 = {"ios", "windows", "android"}

# Build dataset
alternatives = [alternatives3]
profiles = [(3, profile3)]

css_regret = CurrentSolutionHeuristicProtocol()
test_experiment = Experiment(alternatives, profiles, borda_name, css_regret,
                             save_result_to_file)
test_experiment.execute()
예제 #26
0
    def get(self, exp_id):  # Documentation needs update to advice_id
        """ Update the parameters and set a reward for a given experiment.

        For parameters, there are two options (next to the mandatory key 
        and exp_id). The first option is supplying all the information manually, 
        meaning that you supply the following parameters:
            -   Context
            -   Action
            -   Reward

        +----------------------------------------------------------------+
        | Example                                                        |
        +================================================================+
        |http://example.com/setreward/EXP_ID?key=KEY&context=CONTEXT     |
        |&action=ACTION&reward=REWARD                                    |
        +----------------------------------------------------------------+

        When you have toggled the Advice ID in the experiment properties (second option), 
        and have received an Advice ID from the getaction call, you only have
        to supply the following parameters:
            -   Advice ID
            -   Reward

        +----------------------------------------------------------------+
        | Example                                                        |
        +================================================================+
        |http://example.com/setreward/EXP_ID?key=KEY                     |
        |&advice_id=ADVICE_ID&reward=REWARD                              |
        +----------------------------------------------------------------+

        :param int exp_id: Experiment ID as specified in the url.
        :param string key: The key corresponding to the experiment.

        :param JSON context (optional): The context for the current update.
        :param JSON action (optional): The action for the current update.
        :param string advice_id (optional): The advice_id for the current \
        full update loop.
        :param JSON reward: The reward for the current update.
        :returns: A JSON indicating success.
        :raises 400: If key is not supplied.
        :raises 401: If the key or exp_id is invalid.
        """
        key = self.get_argument("key", default=False)

        if not key:
            raise ExceptionHandler(reason="Key not supplied.", status_code=400)

        __EXP__ = Experiment(exp_id, key)

        if __EXP__.is_valid():
            if self.get_argument("advice_id", default="") == "":
                context = json.loads(self.get_argument("context",
                                                       default="{}"))
                action = json.loads(self.get_argument("action", default="{}"))
            else:
                advice_id = self.get_argument("advice_id", default="")
                log = __EXP__.get_by_advice_id(advice_id)
                if log == False:
                    self.finish("Advice ID does not exist!")
                else:
                    context = log['context']
                    action = log['action']
            reward = json.loads(self.get_argument("reward", default="{}"))
            __EXP__.run_reward_code(context, action, reward)
            __EXP__.log_setreward_data(context, action, reward)

            if self.settings['debug']:
                self.write(
                    json.dumps({
                        'status': 'success',
                        'action': action,
                        'context': context,
                        'reward': reward
                    }))
            else:
                self.write(json.dumps({'status': 'success'}))
        else:
            raise ExceptionHandler(reason="Key or exp_id is invalid.",
                                   status_code=401)
예제 #27
0
import sqlite3
from core.voting_rules import rules_global_dict
from core.elicitation_protocols import elicitation_protocols_global_dict
from core.dataset_reader import read_dataset
from core.experiment import Experiment
from core.output_writers import save_result_to_db, create_results_table_with_name

DB_PATH = ".\\db\\data.db"

retrieve_scheduled_experiments = """
                                 SELECT * FROM experiments
                                 WHERE Scheduled = 1
                                 """

conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute(retrieve_scheduled_experiments)
scheduled_experiments = c.fetchall()
conn.close()

for experiment_row in scheduled_experiments:
    rule = rules_global_dict[experiment_row[1]]
    protocol = elicitation_protocols_global_dict[experiment_row[2]]()
    dataset_name = experiment_row[3]
    alternatives, profiles = read_dataset(dataset_name, DB_PATH)
    results_table = experiment_row[4]
    create_results_table_with_name(DB_PATH, results_table, dataset_name)
    experiment = Experiment(alternatives, profiles, rule, protocol,
                            save_result_to_db(DB_PATH, results_table))
    experiment.execute()
예제 #28
0
            [
                "knowledge_graph", "wikipedia_visualisation", "ship",
                "satellite", "wine_quality", "image_recognition"
            ]]
alternatives5 = {
    "knowledge_graph", "wikipedia_visualisation", "ship", "satellite",
    "wine_quality", "image_recognition"
}

profile6 = [["train", "bus", "airplane", "walk"],
            ["bus", "airplane", "train", "walk"],
            ["airplane", "train", "bus", "walk"]]
alternatives6 = {"train", "bus", "airplane", "walk"}

profile7 = [["ios", "windows", "android"], ["android", "windows", "ios"],
            ["windows", "ios", "android"]]
alternatives7 = {"ios", "windows", "android"}

# Build dataset
alternatives = [
    alternatives1, alternatives2, alternatives3, alternatives4, alternatives5,
    alternatives6, alternatives7
]
profiles = [(1, profile1), (2, profile2), (3, profile3), (4, profile4),
            (5, profile5), (6, profile6), (7, profile7)]

random_pairwise = RandomPairwiseElicitationProtocol()
test_experiment = Experiment(alternatives, profiles, borda_name,
                             random_pairwise, save_result_to_file)
test_experiment.execute()
예제 #29
0
    def get(self, exp_id):  # Documentation needs update to advice_id
        """ Get an action given a context for a specific exp_id.
        
        +----------------------------------------------------------------+
        | Example                                                        |
        +================================================================+
        |http://example.com/getaction/EXP_ID?key=KEY&context=CONTEXT     |
        +----------------------------------------------------------------+
        
        :param int exp_id: Experiment ID as specified in the url.
        :param string key: The key corresponding to the experiment.
        :param JSON context (optional): The context to be evaluated. 
        :returns: A JSON that standard only includes action. When debug is toggled, \
        it returns the context as well. When Advice ID is toggled, it will also return \
        an Advice ID that can be used to set the reward later on.
        :raises 400: If the key is not supplied.
        :raises 401: If the key or exp_id is invalid.
        """
        key = self.get_argument("key", default=False)
        context = json.loads(self.get_argument("context", default="{}"))

        if not key:
            raise ExceptionHandler(reason="Key not supplied.", status_code=400)

        __EXP__ = Experiment(exp_id, key)

        if __EXP__.is_valid():
            response = __EXP__.run_action_code(context)

            if __EXP__.properties['advice_id'] == "True":
                advice_id = __EXP__.gen_advice_id(response.copy(),
                                                  context.copy())

            __EXP__.log_getaction_data(context, response)

            if self.settings['debug'] and __EXP__.properties[
                    'advice_id'] == "True":
                self.write(
                    json.dumps(
                        {
                            'action': response,
                            'context': context,
                            'advice_id': advice_id
                        },
                        default=json_util.default))
            elif __EXP__.properties['advice_id'] == "True":
                self.write(
                    json.dumps({
                        'action': response,
                        'advice_id': advice_id
                    },
                               default=json_util.default))
            elif self.settings['debug']:
                self.write(json.dumps({
                    'action': response,
                    'context': context
                }))
            else:
                self.write(json.dumps({'action': response}))
        else:
            raise ExceptionHandler(reason="Key or exp_id is invalid.",
                                   status_code=401)
예제 #30
0
    def get(self, exp_id):
        """ Simulate your experiment based on four scripts, which create a closed feedback loop.

        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        |http://example.com/eval/EXP_ID/simulate?N=1000&log_stats=True       |
        |&verbose=True&seed=10                                               |
        +--------------------------------------------------------------------+

        :requires: A secure cookie, obtained by logging in.
        :param int exp_id: Experiment ID as specified in the url.
        :param int N: The number of simulation draws.
        :param bool log_stats: Flag for logging the results in the database (default is False)
        :param bool verbose: Flag for displaying the results in the returning JSON object (default is True)
        :param int seed (optional): Set numpy seed.
        :returns: A JSON indicating success when verbose flag is False, and a JSON with all the data when verbose flag is True.
        :raises 400: If the experiment does not belong to this user or the exp_id is wrong.
        :raises 401: If user is not logged in or if there is no secure cookie available.
        """
        if self.get_current_user():
            if self.validate_user_experiment(exp_id):

                N = int(self.get_argument("N", default=1000))
                log_stats = self.get_argument("log_stats", default=False)
                verbose = self.get_argument("verbose", default=True)
                seed = self.get_argument("seed", default=None)
                if seed is None:
                    seed = np.random.randint(2**32 - 1, dtype=np.uint32)
                if verbose == "True":
                    verbose = True
                else:
                    verbose = False
                if log_stats == "True":
                    log_stats = True
                else:
                    log_stats = False

                __EXP__ = Experiment(exp_id)

                data = {}

                with self.temp_seed(int(seed)):
                    for i in range(N):
                        # Generate context
                        context = __EXP__.run_context_code()

                        # Get action
                        action = __EXP__.run_action_code(context)

                        # Generate reward
                        reward = __EXP__.run_get_reward_code(context, action)

                        # Set reward
                        __EXP__.run_reward_code(context, action, reward)

                        # Get theta
                        theta = __EXP__.get_theta()

                        # Save stats
                        data[str(i)] = {
                            'context': context.copy(),
                            'action': action.copy(),
                            'reward': reward.copy(),
                            'theta': theta.copy()
                        }

                        context.clear()
                        action.clear()
                        reward.clear()

                if log_stats == True:
                    __EXP__.log_simulation_data(data.copy())
                data_tmp = data.copy()
                data.clear()

                if verbose == True:
                    self.write(
                        json.dumps({
                            'simulate': 'success',
                            'experiment': exp_id,
                            'data': data_tmp
                        }))
                else:
                    self.write(
                        json.dumps({
                            'simulate': 'success',
                            'experiment': exp_id,
                            'theta': theta
                        }))
            else:
                raise ExceptionHandler(
                    reason="Experiment could not be validated.",
                    status_code=401)
        else:
            raise ExceptionHandler(reason="Could not validate user.",
                                   status_code=401)
예제 #31
0
    def get(self, exp_id):
        """ Simulate your experiment on a simple model
        The model that is drawn from is:

        y = -(x - c)**2 + c2 + rnorm(mu,var)

        Currently there is no context. Make sure that the action of your
        experiment results in:

        {"x" : x}

        This is how the model currently expects your action to be formulated.
        This might become more flexible later on.

        +--------------------------------------------------------------------+
        | Example                                                            |
        +====================================================================+
        |http://example.com/eval/5/simulate?key=XXX&N=10&c=5&c2=10&mu=0&var=1|
        +--------------------------------------------------------------------+

        :param int exp_id: Experiment ID as specified in the url
        :param string key: The key corresponding to the experiment
        :param int N: The number of simulation draws
        :param int c: The size of the parabola
        :param int c2: The height of the parabola
        :param int mu: The mean of the noise on the model
        :param int var: The variance of the noise on the model
        :param string log_stats: Flag for logging the results in the database
        
        :returns: A JSON of the form: {"simulate":"success"}
        :raises AuthError: 401 Invalid Key

        """

        key = self.get_argument("key", default = False)
        
        # Number of draws
        N = int(self.get_argument("N", default = 1000))

        log_stats = self.get_argument("log_stats", default = True)

        # Parameterset for the simulator
        c = float(self.get_argument("c", default = 5))
        c2 = float(self.get_argument("c2", default = 10))
        mu = float(self.get_argument("mu", default = 0))
        var = float(self.get_argument("var", default = .1))

        if not key:
            self.set_status(401)
            self.write("Key not given")
            return

        __EXP__ = Experiment(exp_id, key)

        rewards = np.array([0])
        reward_over_time = np.array([])
        regret = np.array([0])

        if __EXP__.is_valid():
            for i in range(N):
                # Generate context
                context = {}

                # Get action

                action = __EXP__.run_action_code(context)

                # Generate reward

                y = -(action["x"] - c)**2 + c2 + np.random.normal(mu, var)
                #y = 15 + 8*action["x"] + 10*action["x"]**2 + np.random.normal(mu, var)

                reward = {"y" : y}

                # Set reward
                __EXP__.run_reward_code(context, action, reward)
                
                # Save stats
                rewards = np.append(rewards, y)
                tmp_rot = (rewards[-1] + y) / (i+1)
                reward_over_time = np.append(reward_over_time, tmp_rot)
                regret = np.append(regret, (regret[-1] + (c2 - y)))

                #self.write("n = {}, Regret is: {}, reward = {} <br>".format(i,regret[-1], rewards[-1]))


            # Now save the data together with a timestamp in the logs
            # To read out the Numpy array data out again, use array =
            # pickle.loads(record['feature'])

            # FOR FUTURE, the json_tricks package might be interesting
            if log_stats == True:
                print("Logging data")
                __EXP__.log_data({
                    "type" : "evaluation",
                    "time" : int(time.time()),
                    "experiment" : exp_id,
                    "N" : N,
                    "c" : c,
                    "c2" : c2,
                    "rewards" : Binary(_pickle.dumps(rewards, protocol = 2), subtype = 128),
                    "reward_over_time" : Binary(_pickle.dumps(reward_over_time, protocol = 2), subtype = 128),
                    "regret" : Binary(_pickle.dumps(regret, protocol = 2), subtype = 128)
                    })

                self.write(json.dumps({'simulate':'success','experiment':exp_id}))
        else:
            self.set_status(401)
            self.write("Key is not valid for this experiment")
            return
예제 #32
0
파일: demo.py 프로젝트: RomanKoshelev/zoo
from __future__ import print_function

from config import config
from core.experiment import Experiment

config['exp.id'] = "002"
config['env.target_range_xz'] = [[-.7, +.7], [+.5, +1.0]]
config['view.width'] = 1200
config['view.height'] = 800

exp = Experiment(config)
exp.demo()
예제 #33
0
 def prepare(self):
     Experiment.prepare(self)
예제 #34
0
 def clean(self):
     Experiment.clean(self)