def test_current_hit_id_with_active_experiment(self): from dallinger.models import Participant participant = Participant(worker_id='1', hit_id='the hit!', assignment_id='1', mode="test") self.db.add(participant) recruiter = self.make_one() assert recruiter.current_hit_id() == 'the hit!'
def write(session): count = session.query(Participant).count() counts.append(count) session.add( Participant( worker_id='serialized_{}'.format(count + 1), assignment_id='test', hit_id='test', mode='test', ))
def test_open_recruitment_is_noop_if_experiment_in_progress(self): from dallinger.models import Participant participant = Participant(worker_id='1', hit_id='1', assignment_id='1', mode="test") self.db.add(participant) recruiter = self.make_one() recruiter.open_recruitment() recruiter.mturkservice.check_credentials.assert_not_called()
def participant(**kwargs): defaults = { 'worker_id': '1', 'hit_id': '1', 'assignment_id': '1', 'mode': 'test', } defaults.update(kwargs) part = Participant(**defaults) part.creation_time = datetime.datetime.now() return part
def test_statistics_show_working(self, logged_in, db_session): from dallinger.models import Participant participant = Participant( recruiter_id="hotair", worker_id="1", hit_id="1", assignment_id="1", mode="test", ) db_session.add(participant) resp = logged_in.get("/dashboard/monitoring") assert resp.status_code == 200 resp_text = resp.data.decode("utf8") assert "<h3>Participants</h3>" in resp_text assert "<li>working: 1</li>" in resp_text
def nodes(self, topology, db_session, n=3): nodes = [] for i in range(n): participant = Participant( recruiter_id='testing', worker_id='worker_%d' % i, assignment_id='assignment_%d' % i, hit_id='hit_%d' % i, mode='testing', fingerprint_hash='fingerprint_%d' % i, ) db_session.add(participant) participant.status = 'working' node = Agent(network=topology, participant=participant) db_session.add(node) topology.add_node(node) nodes.append(node) return nodes
def add_participant(session, interruptor=None): count = session.query(Participant).count() counts.append(count) # This is for the sake of the test, # to let us change the count from a different session # on the first try. if interruptor: interruptor() session.add( Participant( recruiter_id="hotair", worker_id="serialized_{}".format(count + 1), assignment_id="test", hit_id="test", mode="test", ))
def test_run_rogers(self): """ SIMULATE ROGERS """ hit_id = str(random.random()) overall_start_time = timenow() print("Running simulated experiment...", end="\r") sys.stdout.flush() exp_setup_start = timenow() exp = RogersExperiment(self.db) exp_setup_stop = timenow() exp_setup_start2 = timenow() exp = RogersExperiment(self.db) exp_setup_stop2 = timenow() p_ids = [] p_times = [] dum = timenow() assign_time = dum - dum process_time = dum - dum while exp.networks(full=False): num_completed_participants = len(exp.networks()[0].nodes(type=Agent)) if p_times: print( "Running simulated experiment... participant {} of {}, " "{} participants failed. Prev time: {}".format( num_completed_participants + 1, exp.networks()[0].max_size, len(exp.networks()[0].nodes(failed=True)), p_times[-1], ), end="\r", ) else: print( "Running simulated experiment... participant {} of {}, " "{} participants failed.".format( num_completed_participants + 1, exp.networks()[0].max_size, len(exp.networks()[0].nodes(failed=True)), ), end="\r", ) sys.stdout.flush() worker_id = str(random.random()) assignment_id = str(random.random()) from dallinger.models import Participant p = Participant( recruiter_id="hotair", worker_id=worker_id, assignment_id=assignment_id, hit_id=hit_id, mode="debug", ) self.db.add(p) self.db.commit() p_id = p.id p_ids.append(p_id) p_start_time = timenow() while True: assign_start_time = timenow() network = exp.get_network_for_participant(participant=p) if network is None: break else: agent = exp.create_node(participant=p, network=network) exp.add_node_to_network(node=agent, network=network) self.db.commit() exp.node_post_request(participant=p, node=agent) self.db.commit() assign_stop_time = timenow() assign_time += assign_stop_time - assign_start_time process_start_time = timenow() agent.receive() from operator import attrgetter current_state = max( network.nodes(type=RogersEnvironment)[0].infos(), key=attrgetter("id"), ).contents if float(current_state) >= 0.5: right_answer = "blue" wrong_answer = "yellow" else: right_answer = "yellow" wrong_answer = "blue" if num_completed_participants == 0: info = Meme(origin=agent, contents=right_answer) else: if random.random() < 0.9: info = Meme(origin=agent, contents=right_answer) else: info = Meme(origin=agent, contents=wrong_answer) self.db.commit() exp.info_post_request(node=agent, info=info) # print("state: {}, answer: {}, score: {}, fitness {}".format( # current_state, info.contents, agent.score, agent.fitness)) process_stop_time = timenow() process_time += process_stop_time - process_start_time worked = exp.data_check(participant=p) assert worked bonus = exp.bonus(participant=p) assert bonus >= 0 assert bonus <= 1 attended = exp.attention_check(participant=p) if not attended: participant_nodes = models.Node.query.filter_by( participant_id=p_id, failed=False ).all() p.status = 102 for node in participant_nodes: node.fail() self.db.commit() else: p.status = "approved" exp.submission_successful(participant=p) p_stop_time = timenow() p_times.append(p_stop_time - p_start_time) print("Running simulated experiment... done!") sys.stdout.flush() overall_stop_time = timenow() assert len(exp.networks()) == exp.practice_repeats + exp.experiment_repeats """ TEST NODES """ print("Testing nodes...", end="\r") sys.stdout.flush() for network in [exp.networks()[0]]: agents = network.nodes(type=Agent) sources = network.nodes(type=Source) assert len(sources) == 2 assert len(network.nodes(type=RogersSource)) == 1 assert len(network.nodes(type=RogersEnvironment)) == 1 assert len(agents) + len(sources) == network.max_size source = network.nodes(type=RogersSource) assert len(source) == 1 source = source[0] assert type(source) == RogersSource environment = network.nodes(type=RogersEnvironment) assert len(environment) == 1 environment = environment[0] assert type(environment) == RogersEnvironment vectors = network.vectors() for agent in agents: assert type(agent) == RogersAgent for agent in agents: if agent.generation == 0: assert len(agent.vectors(direction="incoming")) == 2 assert agent.is_connected(direction="from", whom=source) assert agent.is_connected(direction="from", whom=environment) else: assert len(agent.vectors(direction="incoming")) in [2, 3] assert not agent.is_connected(direction="from", whom=source) assert agent.is_connected(direction="from", whom=environment) assert RogersAgent in [ type(a) for a in agent.neighbors(direction="from") ] print("Testing nodes... done!") sys.stdout.flush() """ TEST VECTORS """ print("Testing vectors...", end="\r") sys.stdout.flush() for network in [exp.networks()[0]]: agents = network.nodes(type=Agent) vectors = network.vectors() source = network.nodes(type=RogersSource)[0] environment = network.nodes(type=RogersEnvironment)[0] for v in vectors: if isinstance(v.origin, Agent): assert v.origin.generation == v.destination.generation - 1 else: assert isinstance(v.origin, Source) or isinstance( v.origin, RogersEnvironment ) for agent in agents: if agent.generation == 0: assert ( len( models.Vector.query.filter_by( origin_id=source.id, destination_id=agent.id ).all() ) == 1 ) else: assert ( len( models.Vector.query.filter_by( origin_id=source.id, destination_id=agent.id ).all() ) == 0 ) for agent in agents: assert ( len( [ v for v in vectors if v.origin_id == environment.id and v.destination_id == agent.id ] ) == 1 ) for v in [v for v in vectors if v.origin_id == source.id]: assert isinstance(v.destination, RogersAgent) print("Testing vectors... done!") sys.stdout.flush() """ TEST INFOS """ print("Testing infos...", end="\r") sys.stdout.flush() for network in [exp.networks()[0]]: agents = network.nodes(type=Agent) vectors = network.vectors() source = network.nodes(type=RogersSource)[0] environment = network.nodes(type=RogersEnvironment)[0] infos = network.infos() for agent in agents: assert len([i for i in infos if i.origin_id == agent.id]) == 2 assert ( len( [ i for i in infos if i.origin_id == agent.id and isinstance(i, Gene) ] ) == 1 ) assert ( len( [ i for i in infos if i.origin_id == agent.id and isinstance(i, LearningGene) ] ) == 1 ) assert ( len( [ i for i in infos if i.origin_id == agent.id and isinstance(i, Meme) ] ) == 1 ) print("Testing infos... done!") sys.stdout.flush() """ TEST TRANSMISSIONS """ print("Testing transmissions...", end="\r") sys.stdout.flush() for network in [exp.networks()[0]]: agents = network.nodes(type=Agent) vectors = network.vectors() source = network.nodes(type=RogersSource)[0] environment = network.nodes(type=RogersEnvironment)[0] infos = network.infos() transmissions = network.transmissions() for agent in agents: in_ts = [t for t in transmissions if t.destination_id == agent.id] types = [type(t.info) for t in in_ts] assert len(in_ts) == 2 assert ( len( [ t for t in transmissions if t.destination_id == agent.id and t.status == "pending" ] ) == 0 ) lg = [ i for i in infos if i.origin_id == agent.id and isinstance(i, LearningGene) ] assert len(lg) == 1 lg = lg[0] if lg.contents == "asocial": assert State in types assert LearningGene in types assert Meme not in types else: assert State not in types assert LearningGene in types assert Meme in types print("Testing transmissions... done!") """ TEST FITNESS """ print("Testing fitness...", end="\r") sys.stdout.flush() p0_nodes = models.Node.query.filter_by(participant_id=p_ids[0]).all() assert len(p0_nodes) == len(exp.networks()) is_asocial = True e = 2 b = 1 c = 0.3 * b baseline = c + 0.0001 for n in p0_nodes: assert n.fitness == (baseline + 1 * b - is_asocial * c) ** e for network in [exp.networks()[0]]: agents = network.nodes(type=Agent) for agent in agents: is_asocial = agent.infos(type=LearningGene)[0].contents == "asocial" assert agent.fitness == ( (baseline + agent.score * b - is_asocial * c) ** e ) print("Testing fitness... done!") sys.stdout.flush() """ TEST BONUS """ print("Testing bonus payments...", end="\r") sys.stdout.flush() assert ( exp.bonus(participant=Participant.query.filter_by(id=p_ids[0]).all()[0]) == exp.bonus_payment ) print("Testing bonus payments... done!") sys.stdout.flush() print("All tests passed: good job!") print("Timings:") overall_time = overall_stop_time - overall_start_time print("Overall time to simulate experiment: {}".format(overall_time)) setup_time = exp_setup_stop - exp_setup_start print("Experiment setup(): {}".format(setup_time)) print("Experiment load: {}".format(exp_setup_stop2 - exp_setup_start2)) print("Participant assignment: {}".format(assign_time)) print("Participant processing: {}".format(process_time)) for i in range(len(p_times)): if i == 0: total_time = p_times[i] else: total_time += p_times[i] print("Participant {}: {}, total: {}".format(i, p_times[i], total_time)) print("#########") test = [p_time.total_seconds() for p_time in p_times] print(test)