def participant(**kwargs):
defaults = {
'worker_id': '1',
'hit_id': '1',
'assignment_id': '1',
'mode': 'test',
}
defaults.update(kwargs)
part = Participant(**defaults)
part.creation_time = datetime.datetime.now()
return part
def nodes(self, topology, db_session, n=3):
nodes = []
for i in range(n):
participant = Participant(
recruiter_id='testing',
worker_id='worker_%d' % i,
assignment_id='assignment_%d' % i,
hit_id='hit_%d' % i,
mode='testing',
fingerprint_hash='fingerprint_%d' % i,
)
db_session.add(participant)
participant.status = 'working'
node = Agent(network=topology, participant=participant)
db_session.add(node)
topology.add_node(node)
nodes.append(node)
return nodes
def write(session):
count = session.query(Participant).count()
counts.append(count)
session.add(
Participant(
worker_id='serialized_{}'.format(count + 1),
assignment_id='test',
hit_id='test',
mode='test',
))
def test_current_hit_id_with_active_experiment(self):
from dallinger.models import Participant
participant = Participant(worker_id='1',
hit_id='the hit!',
assignment_id='1',
mode="test")
self.db.add(participant)
recruiter = self.make_one()
assert recruiter.current_hit_id() == 'the hit!'
def test_open_recruitment_is_noop_if_experiment_in_progress(self):
from dallinger.models import Participant
participant = Participant(worker_id='1',
hit_id='1',
assignment_id='1',
mode="test")
self.db.add(participant)
recruiter = self.make_one()
recruiter.open_recruitment()
recruiter.mturkservice.check_credentials.assert_not_called()
def test_statistics_show_working(self, logged_in, db_session):
from dallinger.models import Participant
participant = Participant(
recruiter_id="hotair",
worker_id="1",
hit_id="1",
assignment_id="1",
mode="test",
)
db_session.add(participant)
resp = logged_in.get("/dashboard/monitoring")
assert resp.status_code == 200
resp_text = resp.data.decode("utf8")
assert "<h3>Participants</h3>" in resp_text
assert "<li>working: 1</li>" in resp_text
def add_participant(session, interruptor=None):
count = session.query(Participant).count()
counts.append(count)
# This is for the sake of the test,
# to let us change the count from a different session
# on the first try.
if interruptor:
interruptor()
session.add(
Participant(
recruiter_id="hotair",
worker_id="serialized_{}".format(count + 1),
assignment_id="test",
hit_id="test",
mode="test",
))
def test_run_rogers(self):
"""
SIMULATE ROGERS
"""
hit_id = str(random.random())
overall_start_time = timenow()
print("Running simulated experiment...", end="\r")
sys.stdout.flush()
exp_setup_start = timenow()
exp = RogersExperiment(self.db)
exp_setup_stop = timenow()
exp_setup_start2 = timenow()
exp = RogersExperiment(self.db)
exp_setup_stop2 = timenow()
p_ids = []
p_times = []
dum = timenow()
assign_time = dum - dum
process_time = dum - dum
while exp.networks(full=False):
num_completed_participants = len(exp.networks()[0].nodes(type=Agent))
if p_times:
print(
"Running simulated experiment... participant {} of {}, "
"{} participants failed. Prev time: {}".format(
num_completed_participants + 1,
exp.networks()[0].max_size,
len(exp.networks()[0].nodes(failed=True)),
p_times[-1],
),
end="\r",
)
else:
print(
"Running simulated experiment... participant {} of {}, "
"{} participants failed.".format(
num_completed_participants + 1,
exp.networks()[0].max_size,
len(exp.networks()[0].nodes(failed=True)),
),
end="\r",
)
sys.stdout.flush()
worker_id = str(random.random())
assignment_id = str(random.random())
from dallinger.models import Participant
p = Participant(
recruiter_id="hotair",
worker_id=worker_id,
assignment_id=assignment_id,
hit_id=hit_id,
mode="debug",
)
self.db.add(p)
self.db.commit()
p_id = p.id
p_ids.append(p_id)
p_start_time = timenow()
while True:
assign_start_time = timenow()
network = exp.get_network_for_participant(participant=p)
if network is None:
break
else:
agent = exp.create_node(participant=p, network=network)
exp.add_node_to_network(node=agent, network=network)
self.db.commit()
exp.node_post_request(participant=p, node=agent)
self.db.commit()
assign_stop_time = timenow()
assign_time += assign_stop_time - assign_start_time
process_start_time = timenow()
agent.receive()
from operator import attrgetter
current_state = max(
network.nodes(type=RogersEnvironment)[0].infos(),
key=attrgetter("id"),
).contents
if float(current_state) >= 0.5:
right_answer = "blue"
wrong_answer = "yellow"
else:
right_answer = "yellow"
wrong_answer = "blue"
if num_completed_participants == 0:
info = Meme(origin=agent, contents=right_answer)
else:
if random.random() < 0.9:
info = Meme(origin=agent, contents=right_answer)
else:
info = Meme(origin=agent, contents=wrong_answer)
self.db.commit()
exp.info_post_request(node=agent, info=info)
# print("state: {}, answer: {}, score: {}, fitness {}".format(
# current_state, info.contents, agent.score, agent.fitness))
process_stop_time = timenow()
process_time += process_stop_time - process_start_time
worked = exp.data_check(participant=p)
assert worked
bonus = exp.bonus(participant=p)
assert bonus >= 0
assert bonus <= 1
attended = exp.attention_check(participant=p)
if not attended:
participant_nodes = models.Node.query.filter_by(
participant_id=p_id, failed=False
).all()
p.status = 102
for node in participant_nodes:
node.fail()
self.db.commit()
else:
p.status = "approved"
exp.submission_successful(participant=p)
p_stop_time = timenow()
p_times.append(p_stop_time - p_start_time)
print("Running simulated experiment... done!")
sys.stdout.flush()
overall_stop_time = timenow()
assert len(exp.networks()) == exp.practice_repeats + exp.experiment_repeats
"""
TEST NODES
"""
print("Testing nodes...", end="\r")
sys.stdout.flush()
for network in [exp.networks()[0]]:
agents = network.nodes(type=Agent)
sources = network.nodes(type=Source)
assert len(sources) == 2
assert len(network.nodes(type=RogersSource)) == 1
assert len(network.nodes(type=RogersEnvironment)) == 1
assert len(agents) + len(sources) == network.max_size
source = network.nodes(type=RogersSource)
assert len(source) == 1
source = source[0]
assert type(source) == RogersSource
environment = network.nodes(type=RogersEnvironment)
assert len(environment) == 1
environment = environment[0]
assert type(environment) == RogersEnvironment
vectors = network.vectors()
for agent in agents:
assert type(agent) == RogersAgent
for agent in agents:
if agent.generation == 0:
assert len(agent.vectors(direction="incoming")) == 2
assert agent.is_connected(direction="from", whom=source)
assert agent.is_connected(direction="from", whom=environment)
else:
assert len(agent.vectors(direction="incoming")) in [2, 3]
assert not agent.is_connected(direction="from", whom=source)
assert agent.is_connected(direction="from", whom=environment)
assert RogersAgent in [
type(a) for a in agent.neighbors(direction="from")
]
print("Testing nodes... done!")
sys.stdout.flush()
"""
TEST VECTORS
"""
print("Testing vectors...", end="\r")
sys.stdout.flush()
for network in [exp.networks()[0]]:
agents = network.nodes(type=Agent)
vectors = network.vectors()
source = network.nodes(type=RogersSource)[0]
environment = network.nodes(type=RogersEnvironment)[0]
for v in vectors:
if isinstance(v.origin, Agent):
assert v.origin.generation == v.destination.generation - 1
else:
assert isinstance(v.origin, Source) or isinstance(
v.origin, RogersEnvironment
)
for agent in agents:
if agent.generation == 0:
assert (
len(
models.Vector.query.filter_by(
origin_id=source.id, destination_id=agent.id
).all()
)
== 1
)
else:
assert (
len(
models.Vector.query.filter_by(
origin_id=source.id, destination_id=agent.id
).all()
)
== 0
)
for agent in agents:
assert (
len(
[
v
for v in vectors
if v.origin_id == environment.id
and v.destination_id == agent.id
]
)
== 1
)
for v in [v for v in vectors if v.origin_id == source.id]:
assert isinstance(v.destination, RogersAgent)
print("Testing vectors... done!")
sys.stdout.flush()
"""
TEST INFOS
"""
print("Testing infos...", end="\r")
sys.stdout.flush()
for network in [exp.networks()[0]]:
agents = network.nodes(type=Agent)
vectors = network.vectors()
source = network.nodes(type=RogersSource)[0]
environment = network.nodes(type=RogersEnvironment)[0]
infos = network.infos()
for agent in agents:
assert len([i for i in infos if i.origin_id == agent.id]) == 2
assert (
len(
[
i
for i in infos
if i.origin_id == agent.id and isinstance(i, Gene)
]
)
== 1
)
assert (
len(
[
i
for i in infos
if i.origin_id == agent.id and isinstance(i, LearningGene)
]
)
== 1
)
assert (
len(
[
i
for i in infos
if i.origin_id == agent.id and isinstance(i, Meme)
]
)
== 1
)
print("Testing infos... done!")
sys.stdout.flush()
"""
TEST TRANSMISSIONS
"""
print("Testing transmissions...", end="\r")
sys.stdout.flush()
for network in [exp.networks()[0]]:
agents = network.nodes(type=Agent)
vectors = network.vectors()
source = network.nodes(type=RogersSource)[0]
environment = network.nodes(type=RogersEnvironment)[0]
infos = network.infos()
transmissions = network.transmissions()
for agent in agents:
in_ts = [t for t in transmissions if t.destination_id == agent.id]
types = [type(t.info) for t in in_ts]
assert len(in_ts) == 2
assert (
len(
[
t
for t in transmissions
if t.destination_id == agent.id and t.status == "pending"
]
)
== 0
)
lg = [
i
for i in infos
if i.origin_id == agent.id and isinstance(i, LearningGene)
]
assert len(lg) == 1
lg = lg[0]
if lg.contents == "asocial":
assert State in types
assert LearningGene in types
assert Meme not in types
else:
assert State not in types
assert LearningGene in types
assert Meme in types
print("Testing transmissions... done!")
"""
TEST FITNESS
"""
print("Testing fitness...", end="\r")
sys.stdout.flush()
p0_nodes = models.Node.query.filter_by(participant_id=p_ids[0]).all()
assert len(p0_nodes) == len(exp.networks())
is_asocial = True
e = 2
b = 1
c = 0.3 * b
baseline = c + 0.0001
for n in p0_nodes:
assert n.fitness == (baseline + 1 * b - is_asocial * c) ** e
for network in [exp.networks()[0]]:
agents = network.nodes(type=Agent)
for agent in agents:
is_asocial = agent.infos(type=LearningGene)[0].contents == "asocial"
assert agent.fitness == (
(baseline + agent.score * b - is_asocial * c) ** e
)
print("Testing fitness... done!")
sys.stdout.flush()
"""
TEST BONUS
"""
print("Testing bonus payments...", end="\r")
sys.stdout.flush()
assert (
exp.bonus(participant=Participant.query.filter_by(id=p_ids[0]).all()[0])
== exp.bonus_payment
)
print("Testing bonus payments... done!")
sys.stdout.flush()
print("All tests passed: good job!")
print("Timings:")
overall_time = overall_stop_time - overall_start_time
print("Overall time to simulate experiment: {}".format(overall_time))
setup_time = exp_setup_stop - exp_setup_start
print("Experiment setup(): {}".format(setup_time))
print("Experiment load: {}".format(exp_setup_stop2 - exp_setup_start2))
print("Participant assignment: {}".format(assign_time))
print("Participant processing: {}".format(process_time))
for i in range(len(p_times)):
if i == 0:
total_time = p_times[i]
else:
total_time += p_times[i]
print("Participant {}: {}, total: {}".format(i, p_times[i], total_time))
print("#########")
test = [p_time.total_seconds() for p_time in p_times]
print(test)