def test_closes_not_active(self, close_mock, open_mock, put_mock): Experiment(name = 'e1', file = 'file1.txt', since = datetime.now(), filters = {'type': '1'}).save() e2 = Experiment(name = 'e2', file = 'file2.txt', since = datetime.now(), filters = {'type': '2'}).save() RecordingPDU.FILES_PURGE_THRESHOLD = 0.1 pdu = RecordingPDU() pdu.process_message({'type': '3', 'id': '0'}) pdu.process_message({'type': '1', 'id': '1'}) pdu.process_message({'type': '2', 'id': '2'}) e2.active = False e2.save() time.sleep(0.3) pdu.process_message({'type': '1', 'id': '3'}) pdu.process_message({'type': '2', 'id': '4'}) pdu.process_message({'type': '3', 'id': '5'}) eq_(3, put_mock.call_count, "2 put expected!") put_mock.assert_any_call(Matcher("file1.txt", lambda o: o.fname), Matcher('1', lambda o: o['type'])) put_mock.assert_any_call(Matcher("file2.txt", lambda o: o.fname), Matcher('2', lambda o: o['type'])) eq_(1, close_mock.call_count, "1 close expected!") close_mock.assert_called_once_with(Matcher("file2.txt", lambda o: o.fname))
def test_inactive_matching_experiment_is_not_returned(self): Experiment.objects.all().delete() e1 = Experiment(name = 'e1', file = 'file4.txt', filters = {'a': 'b'}, active=False) e1.save() matches = Experiment.get_active_experiments_matching({'a': 'b'}) eq_(len(matches), 0)
def test_experiment_matches(self): e = Experiment(file = 'file3.txt', filters = {'a': 'b', 'm': ['e', 'f', 'g']}, name='test3') ok_(not e.matches({'c': 'd', 'a': 'b'})) ok_(not e.matches({'c': 'd', 'a': 'b', 'm':'h'})) ok_(e.matches({'c': 'd', 'a': 'b', 'm':'f'})) ok_(not e.matches({'c': 'd'})) ok_(not e.matches({'a': 'c'}))
def test_no_active(self, close_mock, open_mock, put_mock): e = Experiment(name = 'e1', file = 'file2.txt', since = datetime.now()) e.active = False e.save() pdu = RecordingPDU() pdu.process_message({'id': 1}) pdu.process_message({'id': 1}) eq_(0, close_mock.call_count, "no calls expected!") eq_(0, open_mock.call_count, "no calls expected!") eq_(0, put_mock.call_count, "no calls expected!")
def repeat(id): experiment = Experiment.query.get_or_404(id) # Search to modify the name properly (ending in *.RepNo) match = re.match(r'(.+\.)(\d+)', experiment.name) # If already a similar ending cause of older repeats or # randomly if match: repExpName = match.group(1) + str(int(match.group(2)) + 1) # For sure the first repeat else: repExpName = experiment.name + ".1" try: repNodes = [i for i in experiment.nodes if i.available] if not repNodes: flash("No node is ready to repeat the experiment") return redirect(url_for(".show", id=id)) repExp = Experiment(repExpName, experiment.overlay, repNodes) db.session.add(repExp) db.session.commit() flash("Experiment '%s' added successfully" % repExpName) return redirect(url_for(".show", id=repExp.id)) except Exception as e: db.session.rollback() flash("Failed to add experiment '%s' %s" % (repExpName, str(e))) return redirect(url_for(".show", id=id))
def process_message(self, message): # Route messages towards mongo-writer # created_at should be normally set as close to the data generation # as possible and should be the timestamp that the measurement # was phisically generated. Since this is not always possible, the # default is the time of entry in the pipeline. message['created_at'] = message.get('created_at', int(time.time())) self.send_to('mongo-writer', message) self.log('sensor type %s' % message['sensor_type']) # Images & skeletons should be sent to head-crop if message['type'] in ['image_rgb', 'skeleton']: self.send_to('head-crop', message) # If there is at least one active experiment, send them to # recorder. Otherwise, prevent bandwidth waste :) active_experiments = Experiment.get_active_experiments() if len(active_experiments) > 0: self.send_to('recorder', message) # Only send to room position if it's a Kinect skeleton if (message['sensor_type'] == 'kinect' and message['type'] == 'skeleton'): self.send_to('room-position', message) self.send_to('posture-classifier', message) # Only send to dashboard if it's a Kinect RGB image if (message['sensor_type'] == 'kinect' and message['type'] in ['image_rgb', 'skeleton']): self.send_to('dashboard', message)
def test_save_and_fetch(self): """ Test that an experiment can be saved and fetched correctly. """ e = Experiment(name='e1') e.since = datetime.now() e.until = datetime.now() e.file = 'file.txt' e.filters = {} e.name = 'test1' e.save() e2 = Experiment.objects.get(id = e.id) eq_(e2.file, e.file, "File of retrieved experiment should be equal")
def test_get_active_experiments_returns_active_experiment(self): """ Test that get_active_experiments works correctly indeed. """ e = Experiment(name = "e1", file = 'file2.txt', since = datetime.now()) e.save() active = Experiment.get_active_experiments() eq_(active.count(), 1, "There should be exactly one active experiment") e.active = False e.save() active = Experiment.get_active_experiments() eq_(active.count(), 0, "There should be no active experiments")
def test_get_active_experiments_matching(self): Experiment.objects.all().delete() e1 = Experiment(name = 'e1', file = 'file3.txt', filters = {'a': 'b'}) e1.save() e2 = Experiment(name = 'e2', file = 'file3.txt', filters = {'c': 'd'}) e2.save() matches = Experiment.get_active_experiments_matching({'a': 'b', 'c': 'd'}) ok_(set(matches), set([e1, e2])) matches = Experiment.get_active_experiments_matching({'a': 'b'}) ok_(set(matches), set([e1])) matches = Experiment.get_active_experiments_matching({'c': 'd'}) ok_(set(matches), set([e2])) matches = Experiment.get_active_experiments_matching({'e': 'f'}) eq_(len(matches), 0)
def add(): if request.method == "GET": overlays = enumerate(sorted(os.listdir(app.config["OVERLAY_DIR"]))) nodes = get_nodes(None) freeNodes = [ node for node in nodes if (node.reachable == True and node.available == True) ] return render_template("experiment/add.html", freeNodes=freeNodes, overlays=overlays) elif request.method == "POST": try: try: name = request.form["name"] if not name: raise Exception("Invalid name") except KeyError: raise Exception("Name not specified") try: overlay = request.form["overlay"] except KeyError: raise Exception("Overlay not selected") overlayFile = None if overlay == "NEW": try: overlayFile = request.files["overlay_file"] overlay = secure_filename(overlayFile.filename) overlayFile.save( os.path.join(app.config["OVERLAY_DIR"], overlay)) except KeyError: raise Exception("Invalid overlay uploaded") try: nodes = Node.query.filter( Node.id.in_(request.form.getlist("nodeIds"))).all() if not nodes: raise KeyError except KeyError: raise Exception("Can't setup an experiment with no nodes") experiment = Experiment(name, overlay, nodes) db.session.add(experiment) db.session.commit() flash("Experiment '%s' added successfully" % name) return redirect(url_for(".show", id=experiment.id)) except Exception as e: db.session.rollback() flash("Failed to add experiment: %s" % str(e)) return redirect(url_for(".add"))
def create_experiment(self, name="anexperiment", nodes=[], overlay=""): if not nodes: nodes = [self.create_node()] experiment = Experiment( name=name, overlay=overlay, nodes=nodes ) db.session.add(experiment) db.session.commit() return experiment
def worker(i, working_queue, output_q, cv_splits, ds_collection): while True: try: exp_params = working_queue.get_nowait() exp = Experiment.from_params(exp_params) exp.cv_splits = cv_splits exp.set_datasets(ds_collection) Print.progress("{}: Running Experiment".format(i)) exp.run() output_q.put(exp.report) except Empty: Print.info("Queue Empty") break return
def process_message(self, message): # Purge open files once in a while current_time = time.time() if current_time - self._last_files_purge >= self.FILES_PURGE_THRESHOLD: self.purge_files() # Get active experiments matching this message experiments = Experiment.get_active_experiments_matching(message) experiment_ids = [str(e.id) for e in experiments] if len(experiment_ids) > 0: self.logger.info("Measurement %r matches experiments %r" %\ (json_dumps(message), experiment_ids)) # Write the measurement to each experiment file for e in experiments: efile = self.get_file_for_experiment(e) efile.put(message) self.logger.info("Written measurement to file %s" % e.file)
def purge_files(self): ''' Compare the list of active exercises with the list of open files to determine which are to be closed''' self.logger.info("Starting to purge files ...") active_ids = set([e.id for e in Experiment.get_active_experiments()]) self.logger.info("Active experiment IDs: %r" % active_ids) open_files = {} for (ident, efile) in self._open_files.iteritems(): if ident not in active_ids: efile.close() self.logger.info("Closing down file for experiment %s," "because its experiment is no longer active" %\ str(ident)) else: open_files[ident] = efile self._open_files = open_files
def run_experiments(self, fast_datasets=False): time.sleep(1) start_run_time = time.time() ds_collection = DatasetCollection.from_params(self.params, self.cv_splits, fast=fast_datasets) if self.multiprocessing == "exp": self.run_multi(ds_collection) else: for i, exp_params in enumerate( tqdm(self.exp_params_list, desc="Running Experiments")): exp = Experiment.from_params(exp_params) exp.cv_splits = self.cv_splits exp.index = i exp.set_datasets(ds_collection) exp.multiprocessing = (self.multiprocessing == "cv") exp.run() if self.best_exp is None or exp.report[ "accuracy"] > self.best_exp.report["accuracy"]: Print.good("New best: {}".format( np.round(exp.report["accuracy"], 3))) self.best_exp = exp self.exp_reports.append(exp.report) self.run_time = time.time() - start_run_time self.generate_report() if self.save_best: from sklearn.externals import joblib fp = Path.classifiers + '/' + "classifier1.pkl" joblib.dump(self.best_exp.pipeline, fp)
def probe(self): # Receive message from SQS queue response = self.sqs.receive_message( QueueUrl=self.sqs_url, AttributeNames=['All'], MaxNumberOfMessages=1, MessageAttributeNames=['StimuliType'], VisibilityTimeout=0, WaitTimeSeconds=10) if not response or response.get('Messages', None) is None: return None message = response['Messages'][0] receipt_handle = message['ReceiptHandle'] message_id = message['MessageId'] body = message['Body'] msg_attrs = message['MessageAttributes'] stimuli = msg_attrs['StimuliType']['StringValue'] if stimuli == WORD_LIST: info = json.loads(body) self.sqs.delete_message(QueueUrl=self.sqs_url, ReceiptHandle=receipt_handle) Experiment()(info) return message_id
print "%20s %20s %20s" % ("name", "file", "active") print "--------------------------------------------------------------" for e in experiments: print "%20s %20s %20s" % (e.name, e.file, e.active) elif args.operation == 'start': # See if there is an existing Experiment with that name try: connect('experiments', host=settings.MONGO_SERVER) e = Experiment.objects.get(name=args.name) logger.error("There is already an experiment with the name %s!" % args.name) except DoesNotExist: e = Experiment(name=args.name, filters=json.loads(args.filters), file=args.file, since=datetime.now(), active=True) e.save() except MultipleObjectsReturned: logger.error("There is more than one experiment with name %s! " "Database is corrupted." % args.name) elif args.operation == 'stop': try: connect('experiments', host=settings.MONGO_SERVER) e = Experiment.objects.get(name=args.name) if not e.active: logger.error("The experiment %s is not active!" % args.name) else: e.active = False
def onStart(self, input_list): probabilistic_interleavings_list = input_list[0]["probabilistic"] team_draft_interleavings_list = input_list[0]["team_draft"] probabilistic_click_model = input_list[1]["probabilistic"] random_click_model = input_list[1]["random"] experiment_1 = Experiment((probabilistic_interleavings_list), probabilistic_click_model, 1) experiment_2 = Experiment((probabilistic_interleavings_list), random_click_model, 2) experiment_3 = Experiment((team_draft_interleavings_list), probabilistic_click_model, 3) experiment_4 = Experiment((team_draft_interleavings_list), random_click_model, 4) save_and_load = Saver("data/") experiments = [experiment_1, experiment_2, experiment_3, experiment_4] ignores = [] q = mp.Queue() processes = [ mp.Process(target=self.experimenting, args=(exp, q)) for exp in experiments ] results = [None] * 4 try: print("Running experiments: 1/4") result = save_and_load.load_python_obj("experiment1") results[0] = result ignores.append(0) except: print("started multiprocessing " + str(1)) # result_1 = experiment_1.run() # save_and_load.save_python_obj(result_1, "experiment_1") processes[0].start() try: print("Running experiments: 2/4") result = save_and_load.load_python_obj("experiment2") results[1] = result ignores.append(1) except: print("started multiprocessing " + str(2)) # result_2 = experiment_2.run() # save_and_load.save_python_obj(result_2, "experiment_2") processes[1].start() try: print("Running experiments: 3/4") result = save_and_load.load_python_obj("experiment3") results[2] = result ignores.append(2) except: print("started multiprocessing " + str(3)) # result_3 = experiment_3.run() # save_and_load.save_python_obj(result_3, "experiment_3") processes[2].start() try: print("Running experiments: 4/4") result = save_and_load.load_python_obj("experiment4") results[3] = result ignores.append(3) except: print("started multiprocessing " + str(4)) # result_4 = experiment_4.run() # save_and_load.save_python_obj(result_4, "experiment_4") processes[3].start() for experiment_index in range(4): if (experiment_index in ignores): continue print("Attempting get\n") result = q.get() index = result["name"] results[index - 1] = result del result["name"] print("Got {}\n".format(index)) for i, p in enumerate(processes): if (i in ignores): continue print("Attempting join\n") p.join() print("joined {}\n".format(i)) for i, res in zip([1, 2, 3, 4], results): save_and_load.save_python_obj(res, "experiment{}".format(i)) result_1, result_2, result_3, result_4 = results[0], results[ 1], results[2], results[3] print("\rRunning experiments: Done!") result = { "pbm": { "probabilistic_interleaving": result_1, "team_draft": result_3 }, "random": { "probabilistic_interleaving": result_2, "team_draft": result_4 } } return result
def store_new_experiment(self, name): exp = Experiment() exp.name = name self.session.add(exp) self.session.commit() self.experiment_list.refresh_experiments()