def launch():
"""Launch the experiment."""
exp = experiment(db.init_db(drop_all=False))
exp.log("Launching experiment...", "-----")
init_db()
exp.recruiter().open_recruitment(n=exp.initial_recruitment_size)
session_psiturk.commit()
session.commit()
return success_response(request_type="launch")
def launch():
"""Launch the experiment."""
exp = experiment(db.init_db(drop_all=False))
exp.log("Launch route hit, initializing tables and opening recruitment.", "-----")
init_db()
exp.recruiter().open_recruitment(n=exp.initial_recruitment_size)
session_psiturk.commit()
session.commit()
exp.log("Experiment successfully launched!", "-----")
data = {"status": "success"}
js = dumps(data)
return Response(js, status=200, mimetype='application/json')
def debug(verbose):
"""Run the experiment locally."""
(id, tmp) = setup_experiment(debug=True, verbose=verbose)
# Drop all the tables from the database.
db.init_db(drop_all=True)
# Switch to the temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
# Set the mode to debug.
config.set("Experiment Configuration", "mode", "debug")
config.set("Shell Parameters", "launch_in_sandbox_mode", "true")
config.set("Server Parameters", "logfile",
os.path.join(cwd, config.get("Server Parameters", "logfile")))
# Swap in the HotAirRecruiter
os.rename("wallace_experiment.py", "wallace_experiment_tmp.py")
with open("wallace_experiment_tmp.py", "r+") as f:
with open("wallace_experiment.py", "w+") as f2:
f2.write("from wallace.recruiters import HotAirRecruiter\n")
for idx, line in enumerate(f):
if re.search("\s*self.recruiter = (.*)", line):
p = line.partition("self.recruiter =")
f2.write(p[0] + p[1] + ' HotAirRecruiter\n')
else:
f2.write(line)
os.remove("wallace_experiment_tmp.py")
# Set environment variables.
aws_vars = ['aws_access_key_id', 'aws_secret_access_key', 'aws_region']
for var in aws_vars:
if var not in os.environ:
os.environ[var] = config.get('AWS Access', var)
pt_vars = ['psiturk_access_key_id', 'psiturk_secret_access_id']
for var in pt_vars:
if var not in os.environ:
os.environ[var] = config.get('psiTurk Access', var)
if "HOST" not in os.environ:
os.environ["HOST"] = config.get('Server Parameters', 'host')
# Start up the local server
log("Starting up the server...")
# Try opening the psiTurk shell.
try:
p = pexpect.spawn("psiturk")
p.expect_exact("]$")
p.sendline("server on")
p.expect_exact("Experiment server launching...")
# Launche the experiment.
time.sleep(4)
host = config.get("Server Parameters", "host")
port = config.get("Server Parameters", "port")
subprocess.call('curl --data "" http://{}:{}/launch'.format(
host, port),
shell=True)
log("Here's the psiTurk shell...")
p.interact()
except Exception:
click.echo("\nCouldn't open psiTurk shell. Internet connection okay?")
log("Completed debugging of experiment " + id + ".")
os.chdir(cwd)
def setup(self):
self.db = db.init_db(drop_all=True)
def setup(self):
self.db = db.init_db(drop_all=True)
def setup(self):
"""Set up the environment by resetting the tables."""
self.db = db.init_db(drop_all=True)
def setup(self):
"""Set up the environment by resetting the tables."""
self.db = db.init_db(drop_all=True)
def debug(verbose):
"""Run the experiment locally."""
(id, tmp) = setup(debug=True, verbose=verbose)
# Drop all the tables from the database.
db.init_db(drop_all=True)
# Switch to the temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Load psiTurk configuration.
config = PsiturkConfig()
config.load_config()
# Set the mode to debug.
config.set("Experiment Configuration", "mode", "debug")
config.set("Shell Parameters", "launch_in_sandbox_mode", "true")
config.set(
"Server Parameters",
"logfile",
os.path.join(cwd, config.get("Server Parameters", "logfile")))
# Swap in the HotAirRecruiter
os.rename("wallace_experiment.py", "wallace_experiment_tmp.py")
with open("wallace_experiment_tmp.py", "r+") as f:
with open("wallace_experiment.py", "w+") as f2:
f2.write("from wallace.recruiters import HotAirRecruiter\n")
for idx, line in enumerate(f):
if re.search("\s*self.recruiter = (.*)", line):
p = line.partition("self.recruiter =")
f2.write(p[0] + p[1] + ' HotAirRecruiter\n')
else:
f2.write(line)
os.remove("wallace_experiment_tmp.py")
# Set environment variables.
aws_vars = ['aws_access_key_id', 'aws_secret_access_key', 'aws_region']
for var in aws_vars:
if var not in os.environ:
os.environ[var] = config.get('AWS Access', var)
pt_vars = ['psiturk_access_key_id', 'psiturk_secret_access_id']
for var in pt_vars:
if var not in os.environ:
os.environ[var] = config.get('psiTurk Access', var)
if "HOST" not in os.environ:
os.environ["HOST"] = config.get('Server Parameters', 'host')
# Start up the local server
log("Starting up the server...")
# Try opening the psiTurk shell.
try:
p = pexpect.spawn("psiturk")
p.expect_exact("]$")
p.sendline("server on")
p.expect_exact("Experiment server launching...")
# Launche the experiment.
time.sleep(4)
host = config.get("Server Parameters", "host")
port = config.get("Server Parameters", "port")
subprocess.call(
'curl --data "" http://{}:{}/launch'.format(host, port),
shell=True)
log("Here's the psiTurk shell...")
p.interact()
except Exception:
print "\nCouldn't open the psiTurk shell. Internet connection okay?"
log("Completed debugging of experiment " + id + ".")
os.chdir(cwd)
def test_run_bandit(self):
self.db = db.init_db(drop_all=True)
"""
SIMULATE THE BANDIT GAME
"""
hit_id = str(random.random())
overall_start_time = timenow()
print("Running simulated experiment...", end="\r")
sys.stdout.flush()
# initialize the experiment
exp_setup_start = timenow()
exp = BanditGame(self.db)
exp_setup_stop = timenow()
# reload it for timing purposes
exp_setup_start2 = timenow()
exp = BanditGame(self.db)
exp_setup_stop2 = timenow()
# variables to store timing data
p_ids = []
p_times = []
dum = timenow()
assign_time = dum - dum
process_time = dum - dum
# while there is space
while exp.networks(full=False):
# update the print out
num_completed_participants = len(exp.networks()[0].nodes(type=Agent))
if p_times:
print("Running simulated experiment... participant {} of {}, {} participants failed. Prev time: {}".format(
num_completed_participants+1,
exp.networks()[0].max_size,
len(exp.networks()[0].nodes(failed=True)),
p_times[-1]),
end="\r")
else:
print("Running simulated experiment... participant {} of {}, {} participants failed.".format(
num_completed_participants+1,
exp.networks()[0].max_size,
len(exp.networks()[0].nodes(failed=True))),
end="\r")
sys.stdout.flush()
# generate a new participant
worker_id = str(random.random())
assignment_id = str(random.random())
p = models.Participant(worker_id=worker_id, assignment_id=assignment_id, hit_id=hit_id)
self.db.add(p)
self.db.commit()
p_id = p.unique_id
p_ids.append(p_id)
p_start_time = timenow()
# apply for new nodes
while True:
assign_start_time = timenow()
# get a network
network = exp.get_network_for_participant(participant_id=p_id)
if network is None:
break
else:
# make a node
agent = exp.make_node_for_participant(
participant_id=p_id,
network=network)
# add it to the network
exp.add_node_to_network(
participant_id=p_id,
node=agent,
network=network)
self.db.commit()
assign_stop_time = timenow()
assign_time += (assign_stop_time - assign_start_time)
# play the experiment
process_start_time = timenow()
# get their genes
memory = int(agent.infos(type=MemoryGene)[0].contents)
curiosity = int(agent.infos(type=CuriosityGene)[0].contents)
# variables to store memory
bandit_memory = []
decision_memory = []
# get all bandits
bandits = Bandit.query.filter_by(network_id=agent.network_id).all()
# create a mapping
mapping = range(len(bandits))
for m in range(len(bandits)):
mapping[m] = random.sample(range(1, exp.n_options + 1), exp.n_options)
# play all trials
for trial in range(exp.n_trials):
# pick a bandit
bandit = random.sample(bandits, 1)[0]
bandit_id = bandit.bandit_id
# probabilistically remap bandits
for m in range(len(bandits)):
if random.random() < 0.4:
mapping[m] = random.sample(range(1, exp.n_options + 1), exp.n_options)
# do they remember it?
if memory > 0 and bandit_id in bandit_memory[-memory:]:
remember_bandit = "true"
else:
remember_bandit = "false"
# print("remember bandit: {}".format(remember_bandit))
# if they dont, sample, then choose
if remember_bandit == "false":
# which arms do they pull?
vals = random.sample(range(1, exp.n_options + 1), curiosity)
# pull them!
found_treasure = False
for val in vals:
# print("checking!")
pull = Pull(origin=agent, contents=mapping[bandit_id][val-1])
pull.check = "true"
pull.bandit_id = bandit_id
pull.remembered = remember_bandit
pull.trial = trial
pull.tile = val
# if you pull the right one stop pulling now
if mapping[bandit_id][val-1] == bandit.good_arm:
decision = val
found_treasure = True
break
if not found_treasure:
# otherwise pick a random other arm
decision = random.sample([v for v in range(1, exp.n_options+1) if v not in vals], 1)[0]
# if they do remember it
else:
# what decision can they remember making
remembered_decision = [d for b, d in zip(bandit_memory[-memory:], decision_memory[-memory:]) if b == bandit_id][-1]
# add a chance to misremember
if random.random() < 0.2:
remembered_decision = random.sample([v for v in range(1, exp.n_options+1)], 1)[0]
smart_agents = True
if smart_agents:
# test this decision
pull = Pull(origin=agent, contents=mapping[bandit_id][remembered_decision-1])
pull.check = "true"
pull.bandit_id = bandit_id
pull.remembered = remember_bandit
pull.trial = trial
pull.tile = remembered_decision
# if this is right commit to it
if mapping[bandit_id][remembered_decision-1] == bandit.good_arm:
decision = remembered_decision
# if it isnt pull some more arms
else:
vals = random.sample([v for v in range(1, exp.n_options+1) if v != remembered_decision], (curiosity-1))
found_treasure = False
for val in vals:
pull = Pull(origin=agent, contents=mapping[bandit_id][val-1])
pull.check = "true"
pull.bandit_id = bandit_id
pull.remembered = remember_bandit
pull.trial = trial
pull.tile = val
if mapping[bandit_id][val-1] == bandit.good_arm:
decision = val
found_treasure = True
break
if not found_treasure:
decision = random.sample([v for v in range(1, exp.n_options+1) if v not in vals and v != remembered_decision], 1)[0]
else:
decision = remembered_decision
# now commit that decision
pull = Pull(origin=agent, contents=mapping[bandit_id][decision-1])
pull.check = "false"
pull.bandit_id = bandit_id
pull.remembered = remember_bandit
pull.trial = trial
pull.tile = decision
bandit_memory.append(bandit_id)
decision_memory.append(decision)
self.db.commit()
# calculate fitness
agent.calculate_fitness()
self.db.commit()
process_stop_time = timenow()
process_time += (process_stop_time - process_start_time)
worked = exp.data_check(participant=p)
assert worked
bonus = exp.bonus(participant=p)
assert bonus >= 0
assert bonus <= 1
attended = exp.attention_check(participant=p)
if not attended:
participant_nodes = models.Node.query\
.filter_by(participant_id=p_id, failed=False)\
.all()
p.status = "did_not_attend"
for node in participant_nodes:
node.fail()
self.db.commit()
else:
p.status = "approved"
self.db.commit()
exp.submission_successful(participant=p)
self.db.commit()
p_stop_time = timenow()
p_times.append(p_stop_time - p_start_time)
print("Running simulated experiment... done! ")
sys.stdout.flush()
overall_stop_time = timenow()
assert len(exp.networks()) == exp.practice_repeats + exp.experiment_repeats
"""
TEST NODES
"""
print("Testing nodes...", end="\r")
sys.stdout.flush()
for network in exp.networks():
agents = network.nodes(type=Agent)
assert len(agents) == network.max_size
for generation in range(network.generations):
assert len([a for a in agents if a.generation == generation]) == network.generation_size
sources = network.nodes(type=Source)
assert len(sources) == 1 + exp.n_bandits
genetic_source = network.nodes(type=GeneticSource)
assert len(genetic_source) == 1
bandits = network.nodes(type=Bandit)
assert len(bandits) == exp.n_bandits
genetic_source = genetic_source[0]
for agent in agents:
assert type(agent) == BanditAgent
print("Testing nodes... done!")
sys.stdout.flush()
"""
TEST VECTORS
"""
print("Testing vectors...", end="\r")
sys.stdout.flush()
for network in exp.networks():
agents = network.nodes(type=Agent)
vectors = network.vectors()
genetic_source = network.nodes(type=GeneticSource)[0]
for agent in agents:
if agent.generation == 0:
assert len(agent.vectors(direction="incoming")) == 1
assert agent.is_connected(direction="from", whom=genetic_source)
else:
assert len(agent.vectors(direction="incoming")) == 1
assert len(agent.neighbors(type=BanditAgent, connection="from")) == 1
for v in vectors:
if v.origin == genetic_source:
assert isinstance(v.destination, BanditAgent)
for agent in agents:
assert len(agent.vectors(direction="incoming")) == 1
if agent.generation == exp.generations-1:
assert len(agent.vectors(direction="all")) == 1
print("Testing vectors... done!")
sys.stdout.flush()
"""
TEST INFOS
"""
print("Testing infos...", end="\r")
sys.stdout.flush()
for network in exp.networks():
agents = network.nodes(type=Agent)
bandits = network.nodes(type=Bandit)
genetic_source = network.nodes(type=GeneticSource)[0]
assert len(genetic_source.infos()) == 2
assert len(genetic_source.infos(type=Gene)) == 2
assert len(genetic_source.infos(type=MemoryGene)) == 1
assert len(genetic_source.infos(type=CuriosityGene)) == 1
for bandit in bandits:
assert len(bandit.infos()) == 0
for agent in agents:
infos = agent.infos()
assert (len([i for i in infos if isinstance(i, Gene)])) == 2
assert (len([i for i in infos if isinstance(i, MemoryGene)])) == 1
assert (len([i for i in infos if isinstance(i, CuriosityGene)])) == 1
assert (len([i for i in infos if isinstance(i, Pull)])) == len(infos) - 2
curiosity = int([i for i in infos if isinstance(i, CuriosityGene)][0].contents)
pulls = [i for i in infos if isinstance(i, Pull)]
for pull in pulls:
if pull.check == "true":
assert len([pp for pp in pulls if pp.trial == pull.trial and pp.check == "false"]) == 1
else:
if smart_agents:
assert len([pp for pp in pulls if pp.trial == pull.trial and pp.check == "true"]) > 0
else:
assert len([pp for pp in pulls if pp.trial == pull.trial and pp.check == "true"]) >= 0
assert len([pp for pp in pulls if pp.trial == pull.trial and pp.check == "true"]) <= curiosity
received_infos = agent.received_infos()
assert len(received_infos) == 2
print("Testing infos... done!")
sys.stdout.flush()
"""
TEST TRANSMISSIONS
"""
print("Testing transmissions...", end="\r")
sys.stdout.flush()
for network in exp.networks():
agents = network.nodes(type=Agent)
genetic_source = network.nodes(type=GeneticSource)[0]
infos = network.infos()
assert len(genetic_source.transmissions(direction="all", status="pending")) == 0
assert len(genetic_source.transmissions(direction="incoming", status="all")) == 0
assert len(genetic_source.transmissions(direction="outgoing", status="received")) == 2*exp.generation_size
for agent in agents:
assert len(agent.transmissions(direction="all", status="pending")) == 0
assert len(agent.transmissions(direction="incoming", status="received")) == 2
assert len(agent.transmissions(direction="outgoing", status="received")) % 2 == 0
if agent.generation == exp.generations-1:
assert len(agent.transmissions(direction="outgoing", status="received")) == 0
print("Testing transmissions... done!")
"""
TEST TRANSFORMATIONS
"""
print("Testing transformations...", end="\r")
sys.stdout.flush()
for network in exp.networks():
agents = network.nodes(type=Agent)
genetic_source = network.nodes(type=GeneticSource)[0]
infos = network.infos()
assert len(genetic_source.transformations()) == 0
for agent in agents:
ts = agent.transformations()
assert len(ts) == 2
for t in ts:
assert isinstance(t, Mutation)
print("Testing transformations... done!")
"""
TEST FITNESS
"""
print("Testing fitness...", end="\r")
sys.stdout.flush()
for network in exp.networks():
agents = network.nodes(type=Agent)
sources = network.nodes(type=GeneticSource)
for agent in agents:
assert isinstance(agent.fitness, float)
for source in sources:
assert not isinstance(source.property4, float)
print("Testing fitness... done!")
sys.stdout.flush()
print("All tests passed: good job!")
print("Timings:")
overall_time = overall_stop_time - overall_start_time
print("Overall time to simulate experiment: {}".format(overall_time))
setup_time = exp_setup_stop - exp_setup_start
print("Experiment setup(): {}".format(setup_time))
print("Experiment load: {}".format(exp_setup_stop2 - exp_setup_start2))
print("Participant assignment: {}".format(assign_time))
print("Participant processing: {}".format(process_time))
# for i in range(len(p_times)):
# if i == 0:
# total_time = p_times[i]
# else:
# total_time += p_times[i]
# print("Participant {}: {}, total: {}".format(i, p_times[i], total_time))
print("#########")
agents = BanditAgent.query.filter_by(failed=False).all()
memory_genes = MemoryGene.query.all()
curiosity_genes = CuriosityGene.query.all()
mean_memory = range(exp.generations)
mean_curiosity = range(exp.generations)
for generation in range(exp.generations):
generation_agents = [a for a in agents if int(a.property2) == generation]
generation_agents_ids = [a.id for a in generation_agents]
generation_memory = [int(m.contents) for m in memory_genes if m.origin_id in generation_agents_ids]
mean_memory[generation] = round(1.0*sum(generation_memory)/(1.0*len(generation_memory)), 2)
generation_curiosity = [int(c.contents) for c in curiosity_genes if c.origin_id in generation_agents_ids]
mean_curiosity[generation] = round(1.0*sum(generation_curiosity)/(1.0*len(generation_curiosity)), 2)
print(mean_curiosity)
print(mean_memory)
def setup(self):
self.db = db.init_db(drop_all=True)
os.chdir(os.path.join("examples", "bartlett1932"))
def test_run_linegame(self):
"""
SIMULATE THE BANDIT GAME
"""
self.db = db.init_db(drop_all=True) #this line is important for setup
hit_id = str(random.random())
overall_start_time = timenow()
print("Running simulated experiment...", end="\r")
sys.stdout.flush()
# initialize the experiment
exp_setup_start = timenow()
exp = LineGame(self.db)
exp_setup_stop = timenow()
# reload it for timing purposes
exp_setup_start2 = timenow()
exp = LineGame(self.db)
exp_setup_stop2 = timenow()
# variables to store timing data
p_ids = []
p_times = []
dum = timenow()
assign_time = dum - dum
process_time = dum - dum
bonus=0
num_completed_participants = 0
# while there is space
while exp.networks(full=False, role="experiment"):
chain_lengths = [n.size(type=LineAgent) for n in exp.networks()]
# update the print out
chain_dic = dict([(n.id,n.size(type=LineAgent)) for n in exp.networks()])
chain_list= [ chain_dic[k] for k in chain_dic.keys()]
print("*******")
print(chain_list)
print("*******")
if p_times:
print("Running simulated experiment... participant {} bonus {}, sum of chain length {}, {} nodes failed. Shortest chain: {}, longest chain: {}. Prev time: {}".format(
num_completed_participants+1,
bonus,
sum(chain_lengths),
len(Node.query.filter_by(failed=True).all()),
min(chain_lengths),
max(chain_lengths),
p_times[-1]),
end="\r")
else:
print("Running simulated experiment... participant {} bonus {}, sum of chain length {}, {} nodes failed. Shortest chain: {}, longest chain: {}.".format(
num_completed_participants+1,
bonus,
sum(chain_lengths),
len(Node.query.filter_by(failed=True).all()),
min(chain_lengths),
max(chain_lengths)),
end="\r")
sys.stdout.flush()
# generate a new participant
worker_id = str(random.random())
assignment_id = str(random.random())
from psiturk.models import Participant
p = Participant(workerid=worker_id, assignmentid=assignment_id, hitid=hit_id)
self.db.add(p)
self.db.commit()
p_id = p.uniqueid
p_ids.append(p_id)
p_start_time = timenow()
# apply for new nodes
while True:
assign_start_time = timenow()
# get a network
network = exp.get_network_for_participant(participant_id=p_id)
if network is None:
break
else:
# make a node
agent = exp.make_node_for_participant(
participant_id=p_id,
network=network)
# add it to the network
exp.add_node_to_network(
participant_id=p_id,
node=agent,
network=network)
exp.node_post_request(participant_id=p_id, node=agent)
self.db.commit()
assign_stop_time = timenow()
assign_time += (assign_stop_time - assign_start_time)
# play the experiment
process_start_time = timenow()
# get their true ratio
TrueRatio = int(agent.received_infos()[0].contents)
#print(agent.received_infos()[0].generation)
generation = int(agent.received_infos()[0].generation)
ResponseRatio = int(TrueRatio + round(5*numpy.random.normal()))
ResponseRatio = max(min (ResponseRatio,100), 0)
if (abs(ResponseRatio-TrueRatio)>=exp.UI_PROX_T):
ResponseRatio='NaN'
p_wrong=0.1
if random.random() < p_wrong:
ResponseRatio='NaN'
p_more_trials=0.1
p_more_mistakes=0.5
num_attempts=1
reaction_time=random.random()*1000
true_seed=TrueRatio
result_list=repr([ResponseRatio])
#simulate multiple trials
if unicode(ResponseRatio).isnumeric():
if random.random() < p_more_trials:
num_attempts+=1
result_list=str([ResponseRatio,100-ResponseRatio])
if random.random() < p_more_mistakes:
num_attempts+=1
result_list=([ResponseRatio,100-ResponseRatio,100-TrueRatio])
if random.random() < p_more_mistakes:
num_attempts+=1
result_list=([100-TrueRatio,100-TrueRatio,100-TrueRatio])
ResponseRatio='NaN'
info = LineInfo(contents=ResponseRatio, origin=agent)
info.num_attempts=num_attempts
info.reaction_time=reaction_time
info.true_seed=true_seed
info.result_list=result_list
#info.generation=generation+1
info.generation=agent.generation
# prob of 'Nan', prob wierd number, prob of right number
# get all bandits
exp.info_post_request(node=agent, info=info)
self.db.commit()
process_stop_time = timenow()
process_time += (process_stop_time - process_start_time)
worked = exp.data_check(participant=p)
assert worked
bonus = exp.bonus(participant=p)
p.bonus=bonus
assert bonus >= 0*exp.bonus_payment
assert bonus <= 1*exp.bonus_payment
attended = exp.attention_check(participant=p)
if not attended:
participant_nodes = models.Node.query\
.filter_by(participant_id=p_id, failed=False)\
.all()
p.status = 102
for node in participant_nodes:
node.fail()
self.db.commit()
else:
p.status = 101
self.db.commit()
exp.submission_successful(participant=p)
self.db.commit()
p_stop_time = timenow()
num_completed_participants += 1
p_times.append(p_stop_time - p_start_time)
print("Running simulated experiment... done! ")
sys.stdout.flush()
overall_stop_time = timenow()
assert len(exp.networks()) == exp.practice_repeats + exp.experiment_repeats
"""
TEST NODES
"""
print("Testing nodes...", end="\r")
sys.stdout.flush()
for network in exp.networks(role="experiment"):
agents = network.nodes(type=LineAgent)
assert len(agents) == network.max_size
sources = network.nodes(type=LineSource)
assert len(sources) == 1 # only one source for every network
print("Testing nodes... done!")
sys.stdout.flush()
"""
TEST VECTORS
"""
print("Testing vectors...", end="\r")
sys.stdout.flush()
for network in exp.networks():
agents = network.nodes(type=LineAgent)
vectors = network.vectors()
source = network.nodes(type=LineSource)[0]
for agent in agents:
incomes=[v for v in vectors if v.destination_id == agent.id] #incomes=agent.vectors(direction="incoming")
fromes=[v for v in vectors if v.origin_id == agent.id] #fromes=agent.vectors(direction="outgoing")
assert len(incomes) == 1
assert len(fromes) in [0, 1]
if network.role == "experiment":
assert len([v for v in vectors if v.origin_id == source.id]) == 1
print("Testing vectors... done!")
sys.stdout.flush()
"""
TEST INFOS
"""
print("Testing infos...", end="\r")
sys.stdout.flush()
for network in exp.networks():
agents = network.nodes(type=LineAgent)
source = network.nodes(type=LineSource)[0]
assert len(source.infos(LineInfo)) == 1
for agent in agents:
infos = agent.infos(LineInfo)
assert len(infos) == 1
info=infos[0]
true_seed=info.true_seed
try:
contents=int(info.contents)
except:
assert (contents=='NaN')
else:
assert (abs(contents-true_seed)<exp.UI_PROX_T)
print("Testing infos... done!")
sys.stdout.flush()
"""
TEST TRANSMISSIONS
"""
print("Testing transmissions...", end="\r")
sys.stdout.flush()
for network in exp.networks():
agents = network.nodes(type=LineAgent)
source = network.nodes(type=LineSource)[0]
infos = network.infos(LineInfo)
assert len(source.transmissions(direction="all", status="pending")) == 0
assert len(source.transmissions(direction="incoming", status="all")) == 0
if agents:
assert len(source.transmissions(direction="outgoing", status="received")) == 1
else:
assert len(source.transmissions(direction="outgoing", status="received")) == 0
for agent in agents:
assert len(agent.transmissions(direction="all", status="pending")) == 0
assert len(agent.transmissions(direction="incoming", status="received")) == 1
assert len(agent.transmissions(direction="outgoing", status="received")) in [0, 1]
# if agent.generation == exp.generations-1:
# assert len(agent.transmissions(direction="outgoing", status="received")) == 0
print("Testing transmissions... done!")
print("Print contents. <<<<<<<<<<<<<<<<<<<<<<<<")
number_of_participants_to_check=3
print ("Print contents of {} last particiapnts...".format(number_of_participants_to_check))
networks = exp.networks()
my_part_id=None
for network in networks:
agents=network.nodes(type=LineAgent)
for agent in agents:
my_part_id = agent.participant_id
if not(my_part_id==None):
break
participants = Participant.query.filter_by(uniqueid=my_part_id).all()
print(participants)
print('myhitid:' + my_part_id)
my_hit_id=participants[0].hitid
print('myhitid:' + my_part_id)
# for network in networks:
# agents = network.nodes(type=LineAgent)
# if agents:
# hit_ids=[agent.participant_id for agent in agents]
# else:
# hit_ids=None
# print(hit_ids)
# print(hit_ids)
# hit_id=hit_ids[0]
participants = Participant.query.filter_by(hitid=my_hit_id).all()
# participants = Participant.query.all(assignment_id=assignment_id)
dates={p.beginhit:p for p in participants}
sorted_dates_tuples=sorted(dates.items())
# check that number_of_participants_to_check
if number_of_participants_to_check in [0,None]:
number_of_participants_to_check= len(sorted_dates_tuples)
my_num_parts= min ([number_of_participants_to_check, len(sorted_dates_tuples)])
selected_participants_ids=[k[1].uniqueid for k in sorted_dates_tuples[(-my_num_parts):]]
#print(selected_participants_ids)
#nodes(self, type=None, failed=False, participant_id=None):
# print("Testing infos...", end="\r")
# sys.stdout.flush()
for network in networks:
print(network)
agents = network.nodes(type=LineAgent)
relevant_agents = [agent for agent in agents if agent.participant_id in selected_participants_ids]
if not(relevant_agents): # if this network do not touch the list of particiapnts
continue
source = network.nodes(type=LineSource)[0]
for agent in agents:
infos = agent.infos(Info)
for info in infos:
try:
true_seed=info.true_seed
try:
contents=int(info.contents)
is_fail=False
except:
contents==None
is_fail=True
except:
contents=None
true_seed=None
is_fail=True
net_role=network.role
net_id=network.id
part_id=agent.participant_id
target=contents
source=true_seed
summary={"net_role": net_role, "net_id": net_id, "is_fail": is_fail, "target": target, "source": source}
#print(summary)
str_summary="net_role: {} net_id: {} is_fail: {} part_id: {} target: {} source: {}".format(net_role,net_id,is_fail,part_id,target,source)
print(str_summary)
chain_lengths = [n.size(type=LineAgent) for n in exp.networks()]
chain_dic = dict([(n.id,n.size(type=LineAgent)) for n in exp.networks()])
chain_list= [ chain_dic[k] for k in chain_dic.keys()]
selected_participants_bonus=sum([k[1].bonus for k in sorted_dates_tuples[(-my_num_parts):]])
all_participants_bonus=sum([k[1].bonus for k in sorted_dates_tuples])
all_chain_length=sum(chain_lengths)
min_chain_length=min(chain_lengths)
max_chain_length=max(chain_lengths)
num_failed_nodes=len(Node.query.filter_by(failed=True).all())
selected_participants_num=len(selected_participants_ids)
str_general_summary="all_bonus={} selected_part_num={} part_bonus={} all_chain_length={} max_chain_length={} min_chain_length={} num_failed_nodes={}".format(all_participants_bonus, selected_participants_num, selected_participants_bonus, all_chain_length, max_chain_length, min_chain_length, num_failed_nodes)
summary_general={"all_bonus": all_participants_bonus, "selected_part_num": selected_participants_num, "selected_part_bonus": selected_participants_bonus, "all_chain_length": all_chain_length, "max_chain_length":max_chain_length, "min_chain_length": min_chain_length, "num_failed_nodes": num_failed_nodes, "all_chain_lengths": str(chain_list)}
print(str_general_summary)
print(summary_general)
print("TEST DICTIONARIES!!! ")
number_of_current_participants=1;
number_of_participants_to_check=0;
participants = Participant.query.filter_by(hitid=my_hit_id).all()
dates={p.beginhit:p for p in participants}
sorted_dates_tuples=sorted(dates.items())
# check that number_of_participants_to_check
if number_of_participants_to_check==0:
number_of_participants_to_check= len(sorted_dates_tuples)
my_num_parts= min ([number_of_participants_to_check, len(sorted_dates_tuples)])
number_of_current_participants=min([number_of_current_participants,my_num_parts])
selected_participants_ids=[k[1].uniqueid for k in sorted_dates_tuples[(-my_num_parts):]]
current_participants_ids=[k[1].uniqueid for k in sorted_dates_tuples[(-number_of_current_participants):]]
summary_node_list=[]
summary_node_dict={}
summary_node_dict_fail=dict([(p,{}) for p in selected_participants_ids])
for network in networks:
agents = network.nodes(type=LineAgent)
relevant_agents = [agent for agent in agents if agent.participant_id in selected_participants_ids]
if not(relevant_agents): # if this network do not touch the list of particiapnts
continue
summary_node_dict[network.id]={}
summary_node_dict[network.id]['label']=network.id
summary_node_dict[network.id]['role']=network.role
summary_node_dict[network.id]['current']=False
if network.role == "experiment":
summary_node_dict[network.id]['x']=[-1] * (exp.M_length + 1)
summary_node_dict[network.id]['y']=[-1] * (exp.M_length + 1)
else:
summary_node_dict[network.id]['x']=[-1] * (max_chain_length + 1)
summary_node_dict[network.id]['y']=[-1] * (max_chain_length + 1)
#print('summary_node_dict at initialization')
#print (summary_node_dict)
#source = network.nodes(type=LineSource)[0]
#print("relevant network")
#print(network)
for agent in agents:
agent_p_id=agent.participant_id
if agent_p_id in current_participants_ids:
summary_node_dict[network.id]['label']=str(network.id) + "|" + str(agent_p_id)
summary_node_dict[network.id]['current']=True
#print("-->relevant agent")
#print(agent)
infos = agent.infos()
#infos = Info.query.filter_by(origin_id=agent.id).all()
#print("---->>relevant infos")
#print(infos)
#print(LineInfo.query.filter_by(origin_id=agent.id).all())
for info in infos:
#print("---->>>>relevant info")
#print(info)
#print(">>>>>>>>info true seed:")
#print(info.true_seed)
is_fail=info.failed
generation=info.generation
try:
true_seed=info.true_seed
try:
contents=int(info.contents)
except:
contents==0
is_fail=True
except:
contents=None
true_seed=-10
is_fail=True
generation=0
#print("---->>here")
net_role=network.role
net_id=network.id
part_id=agent.participant_id
target=contents
value_source=true_seed
summary_node={"net_role": net_role, "net_id": net_id, "is_fail": is_fail, "target": target, "source": value_source}
#print(summary_node)
str_summary="net_role: {} net_id: {} is_fail: {} part_id: {} target: {} source: {}".format(net_role,net_id,is_fail,part_id,target,value_source)
print("is_fail:"+ str(is_fail) + " contents:" + str(contents) )
if is_fail:
assert(not(is_fail))
# if agent_p_id in current_participants_ids:
# gen = 1.0*(max([generation-1, 0])) + 0.8*random.random()
# summary_node_dict_fail[agent_p_id]['x'].append(gen)
# summary_node_dict_fail[agent_p_id]['y'].append(true_seed)
else:
gen = 1.0*(generation) + 0.8*random.random()
gen = generation
#print("generation=" + str(generation) + " gen=" + str(gen))
summary_node_dict[network.id]['x'][generation]=gen
summary_node_dict[network.id]['y'][generation]=target
if generation==1:
gen=1.0*(0)+0.8*random.random()
gen = 0
summary_node_dict[network.id]['x'][0]=gen
summary_node_dict[network.id]['y'][0]=value_source
agents = Node.query.filter_by(failed=True, network_id=network.id).all()
for agent in agents:
agent_p_id=agent.participant_id
if agent_p_id in selected_participants_ids:
if not(summary_node_dict_fail[agent_p_id]):
summary_node_dict_fail[agent_p_id]['label']=agent_p_id
summary_node_dict_fail[agent_p_id]['current']=False
summary_node_dict_fail[agent_p_id]['x']=[]
summary_node_dict_fail[agent_p_id]['y']=[]
summary_node_dict_fail[agent_p_id]['role']=[]
generation = agent.generation
true_seed = info.true_seed
gen = max([generation-1, 0])
print("*****")
print(agent)
print("gen:"+ str(generation) + " true_seed:" + str(true_seed) + "failed:" + str(agent.failed))
if network.role=="experiment":
assert(generation<=exp.M_length)
if (gen and true_seed):
summary_node_dict_fail[agent_p_id]['x'].append(gen)
summary_node_dict_fail[agent_p_id]['y'].append(true_seed)
summary_node_dict_fail[agent_p_id]['role'].append("e" if network.role=="experiment" else "p")
#print('summary_node_dict_fail at initialization')
#print (summary_node_dict_fail)
if agent_p_id in current_participants_ids:
summary_node_dict_fail[agent_p_id]['label']=str(agent_p_id)+ "<" + str(network.id)
summary_node_dict_fail[agent_p_id]['current']=True
print('****> summary_node_dict final')
print (dumps(summary_node_dict))
print('****> summary_node_dict_fail final')
print (dumps(summary_node_dict_fail))
# print('****> summary_node_dict_fail prooned')
# # summary_node_dict_fail={k: summary_node_dict_fail[k] for k in summary_node_dict_fail.keys() if len(summary_node_dict_fail[k]['x'])>0}
# temp=[summary_node_dict_fail[k]['x'] for k in summary_node_dict_fail.keys()]
# print(temp)
# summary_node_dict_fail={k: summary_node_dict_fail[k] for k in summary_node_dict_fail.keys() if summary_node_dict_fail[k]['x']}
# print (summary_node_dict_fail)
"""
TEST TRANSFORMATIONS
"""
print("Not testing transformations (there are none)...", end="\r")
sys.stdout.flush()
# for network in exp.networks():
# agents = network.nodes(type=Agent)
# genetic_source = network.nodes(type=GeneticSource)[0]
# infos = network.infos()
# assert len(genetic_source.transformations()) == 0
# for agent in agents:
# ts = agent.transformations()
# assert len(ts) == 2
# for t in ts:
# assert isinstance(t, Mutation)
print("Testing transformations (did nothing)... done!")
"""
TEST FITNESS
"""
# print("Testing fitness...", end="\r")
# sys.stdout.flush()
# for network in exp.networks():
# agents = network.nodes(type=Agent)
# sources = network.nodes(type=GeneticSource)
# for agent in agents:
# assert isinstance(agent.fitness, float)
# for source in sources:
# assert not isinstance(source.property4, float)
print("TODO:Testing fitness... done!")
sys.stdout.flush()
print("All tests passed: good job!")
print("Timings:")
overall_time = overall_stop_time - overall_start_time
print("Overall time to simulate experiment: {}".format(overall_time))
setup_time = exp_setup_stop - exp_setup_start
print("Experiment setup(): {}".format(setup_time))
print("Experiment load: {}".format(exp_setup_stop2 - exp_setup_start2))
print("Participant assignment: {}".format(assign_time))
print("Participant processing: {}".format(process_time))
# for i in range(len(p_times)):
# if i == 0:
# total_time = p_times[i]
# else:
# total_time += p_times[i]
# print("Participant {}: {}, total: {}".format(i, p_times[i], total_time))
print("#########")