def test_closes_not_active(self, close_mock, open_mock, put_mock):
Experiment(name = 'e1',
file = 'file1.txt',
since = datetime.now(),
filters = {'type': '1'}).save()
e2 = Experiment(name = 'e2',
file = 'file2.txt',
since = datetime.now(),
filters = {'type': '2'}).save()
RecordingPDU.FILES_PURGE_THRESHOLD = 0.1
pdu = RecordingPDU()
pdu.process_message({'type': '3', 'id': '0'})
pdu.process_message({'type': '1', 'id': '1'})
pdu.process_message({'type': '2', 'id': '2'})
e2.active = False
e2.save()
time.sleep(0.3)
pdu.process_message({'type': '1', 'id': '3'})
pdu.process_message({'type': '2', 'id': '4'})
pdu.process_message({'type': '3', 'id': '5'})
eq_(3, put_mock.call_count, "2 put expected!")
put_mock.assert_any_call(Matcher("file1.txt", lambda o: o.fname),
Matcher('1', lambda o: o['type']))
put_mock.assert_any_call(Matcher("file2.txt", lambda o: o.fname),
Matcher('2', lambda o: o['type']))
eq_(1, close_mock.call_count, "1 close expected!")
close_mock.assert_called_once_with(Matcher("file2.txt", lambda o: o.fname))
def test_inactive_matching_experiment_is_not_returned(self):
Experiment.objects.all().delete()
e1 = Experiment(name = 'e1', file = 'file4.txt', filters = {'a': 'b'}, active=False)
e1.save()
matches = Experiment.get_active_experiments_matching({'a': 'b'})
eq_(len(matches), 0)
def test_experiment_matches(self):
e = Experiment(file = 'file3.txt', filters = {'a': 'b', 'm': ['e', 'f', 'g']}, name='test3')
ok_(not e.matches({'c': 'd', 'a': 'b'}))
ok_(not e.matches({'c': 'd', 'a': 'b', 'm':'h'}))
ok_(e.matches({'c': 'd', 'a': 'b', 'm':'f'}))
ok_(not e.matches({'c': 'd'}))
ok_(not e.matches({'a': 'c'}))
def test_no_active(self, close_mock, open_mock, put_mock):
e = Experiment(name = 'e1', file = 'file2.txt', since = datetime.now())
e.active = False
e.save()
pdu = RecordingPDU()
pdu.process_message({'id': 1})
pdu.process_message({'id': 1})
eq_(0, close_mock.call_count, "no calls expected!")
eq_(0, open_mock.call_count, "no calls expected!")
eq_(0, put_mock.call_count, "no calls expected!")
def repeat(id):
experiment = Experiment.query.get_or_404(id)
# Search to modify the name properly (ending in *.RepNo)
match = re.match(r'(.+\.)(\d+)', experiment.name)
# If already a similar ending cause of older repeats or
# randomly
if match:
repExpName = match.group(1) + str(int(match.group(2)) + 1)
# For sure the first repeat
else:
repExpName = experiment.name + ".1"
try:
repNodes = [i for i in experiment.nodes if i.available]
if not repNodes:
flash("No node is ready to repeat the experiment")
return redirect(url_for(".show", id=id))
repExp = Experiment(repExpName, experiment.overlay, repNodes)
db.session.add(repExp)
db.session.commit()
flash("Experiment '%s' added successfully" % repExpName)
return redirect(url_for(".show", id=repExp.id))
except Exception as e:
db.session.rollback()
flash("Failed to add experiment '%s' %s" % (repExpName, str(e)))
return redirect(url_for(".show", id=id))
def process_message(self, message):
# Route messages towards mongo-writer
# created_at should be normally set as close to the data generation
# as possible and should be the timestamp that the measurement
# was phisically generated. Since this is not always possible, the
# default is the time of entry in the pipeline.
message['created_at'] = message.get('created_at', int(time.time()))
self.send_to('mongo-writer', message)
self.log('sensor type %s' % message['sensor_type'])
# Images & skeletons should be sent to head-crop
if message['type'] in ['image_rgb', 'skeleton']:
self.send_to('head-crop', message)
# If there is at least one active experiment, send them to
# recorder. Otherwise, prevent bandwidth waste :)
active_experiments = Experiment.get_active_experiments()
if len(active_experiments) > 0:
self.send_to('recorder', message)
# Only send to room position if it's a Kinect skeleton
if (message['sensor_type'] == 'kinect' and
message['type'] == 'skeleton'):
self.send_to('room-position', message)
self.send_to('posture-classifier', message)
# Only send to dashboard if it's a Kinect RGB image
if (message['sensor_type'] == 'kinect' and
message['type'] in ['image_rgb', 'skeleton']):
self.send_to('dashboard', message)
def test_save_and_fetch(self):
""" Test that an experiment can be saved and fetched correctly. """
e = Experiment(name='e1')
e.since = datetime.now()
e.until = datetime.now()
e.file = 'file.txt'
e.filters = {}
e.name = 'test1'
e.save()
e2 = Experiment.objects.get(id = e.id)
eq_(e2.file, e.file, "File of retrieved experiment should be equal")
def test_get_active_experiments_returns_active_experiment(self):
""" Test that get_active_experiments works correctly indeed. """
e = Experiment(name = "e1", file = 'file2.txt', since = datetime.now())
e.save()
active = Experiment.get_active_experiments()
eq_(active.count(), 1, "There should be exactly one active experiment")
e.active = False
e.save()
active = Experiment.get_active_experiments()
eq_(active.count(), 0, "There should be no active experiments")
def test_get_active_experiments_matching(self):
Experiment.objects.all().delete()
e1 = Experiment(name = 'e1', file = 'file3.txt', filters = {'a': 'b'})
e1.save()
e2 = Experiment(name = 'e2', file = 'file3.txt', filters = {'c': 'd'})
e2.save()
matches = Experiment.get_active_experiments_matching({'a': 'b', 'c': 'd'})
ok_(set(matches), set([e1, e2]))
matches = Experiment.get_active_experiments_matching({'a': 'b'})
ok_(set(matches), set([e1]))
matches = Experiment.get_active_experiments_matching({'c': 'd'})
ok_(set(matches), set([e2]))
matches = Experiment.get_active_experiments_matching({'e': 'f'})
eq_(len(matches), 0)
def add():
if request.method == "GET":
overlays = enumerate(sorted(os.listdir(app.config["OVERLAY_DIR"])))
nodes = get_nodes(None)
freeNodes = [
node for node in nodes
if (node.reachable == True and node.available == True)
]
return render_template("experiment/add.html",
freeNodes=freeNodes,
overlays=overlays)
elif request.method == "POST":
try:
try:
name = request.form["name"]
if not name:
raise Exception("Invalid name")
except KeyError:
raise Exception("Name not specified")
try:
overlay = request.form["overlay"]
except KeyError:
raise Exception("Overlay not selected")
overlayFile = None
if overlay == "NEW":
try:
overlayFile = request.files["overlay_file"]
overlay = secure_filename(overlayFile.filename)
overlayFile.save(
os.path.join(app.config["OVERLAY_DIR"], overlay))
except KeyError:
raise Exception("Invalid overlay uploaded")
try:
nodes = Node.query.filter(
Node.id.in_(request.form.getlist("nodeIds"))).all()
if not nodes:
raise KeyError
except KeyError:
raise Exception("Can't setup an experiment with no nodes")
experiment = Experiment(name, overlay, nodes)
db.session.add(experiment)
db.session.commit()
flash("Experiment '%s' added successfully" % name)
return redirect(url_for(".show", id=experiment.id))
except Exception as e:
db.session.rollback()
flash("Failed to add experiment: %s" % str(e))
return redirect(url_for(".add"))
def create_experiment(self, name="anexperiment", nodes=[], overlay=""):
if not nodes:
nodes = [self.create_node()]
experiment = Experiment(
name=name,
overlay=overlay,
nodes=nodes
)
db.session.add(experiment)
db.session.commit()
return experiment
def worker(i, working_queue, output_q, cv_splits, ds_collection):
while True:
try:
exp_params = working_queue.get_nowait()
exp = Experiment.from_params(exp_params)
exp.cv_splits = cv_splits
exp.set_datasets(ds_collection)
Print.progress("{}: Running Experiment".format(i))
exp.run()
output_q.put(exp.report)
except Empty:
Print.info("Queue Empty")
break
return
def process_message(self, message):
# Purge open files once in a while
current_time = time.time()
if current_time - self._last_files_purge >= self.FILES_PURGE_THRESHOLD:
self.purge_files()
# Get active experiments matching this message
experiments = Experiment.get_active_experiments_matching(message)
experiment_ids = [str(e.id) for e in experiments]
if len(experiment_ids) > 0:
self.logger.info("Measurement %r matches experiments %r" %\
(json_dumps(message), experiment_ids))
# Write the measurement to each experiment file
for e in experiments:
efile = self.get_file_for_experiment(e)
efile.put(message)
self.logger.info("Written measurement to file %s" % e.file)
def purge_files(self):
''' Compare the list of active exercises with the list of open files to
determine which are to be closed'''
self.logger.info("Starting to purge files ...")
active_ids = set([e.id for e in Experiment.get_active_experiments()])
self.logger.info("Active experiment IDs: %r" % active_ids)
open_files = {}
for (ident, efile) in self._open_files.iteritems():
if ident not in active_ids:
efile.close()
self.logger.info("Closing down file for experiment %s,"
"because its experiment is no longer active" %\
str(ident))
else:
open_files[ident] = efile
self._open_files = open_files
def run_experiments(self, fast_datasets=False):
time.sleep(1)
start_run_time = time.time()
ds_collection = DatasetCollection.from_params(self.params,
self.cv_splits,
fast=fast_datasets)
if self.multiprocessing == "exp":
self.run_multi(ds_collection)
else:
for i, exp_params in enumerate(
tqdm(self.exp_params_list, desc="Running Experiments")):
exp = Experiment.from_params(exp_params)
exp.cv_splits = self.cv_splits
exp.index = i
exp.set_datasets(ds_collection)
exp.multiprocessing = (self.multiprocessing == "cv")
exp.run()
if self.best_exp is None or exp.report[
"accuracy"] > self.best_exp.report["accuracy"]:
Print.good("New best: {}".format(
np.round(exp.report["accuracy"], 3)))
self.best_exp = exp
self.exp_reports.append(exp.report)
self.run_time = time.time() - start_run_time
self.generate_report()
if self.save_best:
from sklearn.externals import joblib
fp = Path.classifiers + '/' + "classifier1.pkl"
joblib.dump(self.best_exp.pipeline, fp)
def probe(self):
# Receive message from SQS queue
response = self.sqs.receive_message(
QueueUrl=self.sqs_url,
AttributeNames=['All'],
MaxNumberOfMessages=1,
MessageAttributeNames=['StimuliType'],
VisibilityTimeout=0,
WaitTimeSeconds=10)
if not response or response.get('Messages', None) is None:
return None
message = response['Messages'][0]
receipt_handle = message['ReceiptHandle']
message_id = message['MessageId']
body = message['Body']
msg_attrs = message['MessageAttributes']
stimuli = msg_attrs['StimuliType']['StringValue']
if stimuli == WORD_LIST:
info = json.loads(body)
self.sqs.delete_message(QueueUrl=self.sqs_url,
ReceiptHandle=receipt_handle)
Experiment()(info)
return message_id
print "%20s %20s %20s" % ("name", "file", "active")
print "--------------------------------------------------------------"
for e in experiments:
print "%20s %20s %20s" % (e.name, e.file, e.active)
elif args.operation == 'start':
# See if there is an existing Experiment with that name
try:
connect('experiments', host=settings.MONGO_SERVER)
e = Experiment.objects.get(name=args.name)
logger.error("There is already an experiment with the name %s!" %
args.name)
except DoesNotExist:
e = Experiment(name=args.name,
filters=json.loads(args.filters),
file=args.file,
since=datetime.now(),
active=True)
e.save()
except MultipleObjectsReturned:
logger.error("There is more than one experiment with name %s! "
"Database is corrupted." % args.name)
elif args.operation == 'stop':
try:
connect('experiments', host=settings.MONGO_SERVER)
e = Experiment.objects.get(name=args.name)
if not e.active:
logger.error("The experiment %s is not active!" % args.name)
else:
e.active = False
def onStart(self, input_list):
probabilistic_interleavings_list = input_list[0]["probabilistic"]
team_draft_interleavings_list = input_list[0]["team_draft"]
probabilistic_click_model = input_list[1]["probabilistic"]
random_click_model = input_list[1]["random"]
experiment_1 = Experiment((probabilistic_interleavings_list),
probabilistic_click_model, 1)
experiment_2 = Experiment((probabilistic_interleavings_list),
random_click_model, 2)
experiment_3 = Experiment((team_draft_interleavings_list),
probabilistic_click_model, 3)
experiment_4 = Experiment((team_draft_interleavings_list),
random_click_model, 4)
save_and_load = Saver("data/")
experiments = [experiment_1, experiment_2, experiment_3, experiment_4]
ignores = []
q = mp.Queue()
processes = [
mp.Process(target=self.experimenting, args=(exp, q))
for exp in experiments
]
results = [None] * 4
try:
print("Running experiments: 1/4")
result = save_and_load.load_python_obj("experiment1")
results[0] = result
ignores.append(0)
except:
print("started multiprocessing " + str(1))
# result_1 = experiment_1.run()
# save_and_load.save_python_obj(result_1, "experiment_1")
processes[0].start()
try:
print("Running experiments: 2/4")
result = save_and_load.load_python_obj("experiment2")
results[1] = result
ignores.append(1)
except:
print("started multiprocessing " + str(2))
# result_2 = experiment_2.run()
# save_and_load.save_python_obj(result_2, "experiment_2")
processes[1].start()
try:
print("Running experiments: 3/4")
result = save_and_load.load_python_obj("experiment3")
results[2] = result
ignores.append(2)
except:
print("started multiprocessing " + str(3))
# result_3 = experiment_3.run()
# save_and_load.save_python_obj(result_3, "experiment_3")
processes[2].start()
try:
print("Running experiments: 4/4")
result = save_and_load.load_python_obj("experiment4")
results[3] = result
ignores.append(3)
except:
print("started multiprocessing " + str(4))
# result_4 = experiment_4.run()
# save_and_load.save_python_obj(result_4, "experiment_4")
processes[3].start()
for experiment_index in range(4):
if (experiment_index in ignores):
continue
print("Attempting get\n")
result = q.get()
index = result["name"]
results[index - 1] = result
del result["name"]
print("Got {}\n".format(index))
for i, p in enumerate(processes):
if (i in ignores):
continue
print("Attempting join\n")
p.join()
print("joined {}\n".format(i))
for i, res in zip([1, 2, 3, 4], results):
save_and_load.save_python_obj(res, "experiment{}".format(i))
result_1, result_2, result_3, result_4 = results[0], results[
1], results[2], results[3]
print("\rRunning experiments: Done!")
result = {
"pbm": {
"probabilistic_interleaving": result_1,
"team_draft": result_3
},
"random": {
"probabilistic_interleaving": result_2,
"team_draft": result_4
}
}
return result
def store_new_experiment(self, name):
exp = Experiment()
exp.name = name
self.session.add(exp)
self.session.commit()
self.experiment_list.refresh_experiments()
def store_new_experiment(self, name):
exp = Experiment()
exp.name = name
self.session.add(exp)
self.session.commit()
self.experiment_list.refresh_experiments()
