def _process_experiment(reduction_obj, expt_string):
"""
Process the experiment string of a form and find/create
the appropriate Experiment object
@param reduction_obj: ReductionProcess or ReductionConfiguration object
@param expt_string: string taken from the reduction form
"""
# Find experiment
uncategorized_expt = Experiment.objects.get_uncategorized('eqsans')
expts = expt_string.split(',')
for item in expts:
# Experiments have unique names of no more than 24 characters
expt_objs = Experiment.objects.filter(name=item.upper().strip()[:24])
if len(expt_objs)>0:
if expt_objs[0] not in reduction_obj.experiments.all():
reduction_obj.experiments.add(expt_objs[0])
else:
expt_obj = Experiment(name=item.upper().strip()[:24])
expt_obj.save()
reduction_obj.experiments.add(expt_obj)
# Clean up the uncategorized experiment object if we found
# at least one suitable experiment to associate with this reduction
if len(expts)>0:
if uncategorized_expt in reduction_obj.experiments.all():
try:
reduction_obj.experiments.remove(uncategorized_expt)
except:
logger.error("Could not remote uncategorized expt: %s" % sys.exc_value)
else:
reduction_obj.experiments.add(uncategorized_expt)
def async_job(qp: QuantumProgram,
experiment: Experiment,
block_missing_credits=True,
use_sim=True):
running_jobs: List[RunningExperimentData] = []
done_jobs: List[RunningExperimentData] = []
for algorithm_input in experiment.get_inputs():
qasm, qobj, expected = experiment.create_experiment(
qp, algorithm_input)
executing_backend = models.backend_online_simulator if use_sim else experiment.backend
my_credits = qp.get_api().get_my_credits()
backend_status = qp.get_api().backend_status(executing_backend)
log.debug("Current credits: %s" % my_credits["remaining"])
log.debug("Current executing backend status: %s" % backend_status)
while my_credits["remaining"] < 3 and block_missing_credits:
time.sleep(10)
my_credits = qp.get_api().get_my_credits()
log.debug("Current credits: %s" % my_credits["remaining"])
log.debug("Current executing backend status: %s" % backend_status)
job_result = qp.get_api().run_job([{
"qasm": qasm
}],
executing_backend,
experiment.get_shots(),
max_credits=3,
seed=None)
job_id = job_result["id"]
job = RunningExperimentData(job_id, executing_backend,
experiment.get_shots(), algorithm_input,
expected, experiment)
running_jobs.append(job)
log.debug("Added job %s (%s)..." % (job_id, str(algorithm_input)))
while len(running_jobs) > 0:
for job_entry in running_jobs:
job_result = qp.get_api().get_job(job_entry.job_id)
log.debug("Checking job %s..." % (job_entry.job_id))
if "status" in job_result and job_result["status"] == "COMPLETED":
log.debug("Done job %s..." % (job_entry.job_id))
running_jobs.remove(job_entry)
job_entry.complete_job(job_result)
done_jobs.append(job_entry)
time.sleep(2)
for job_entry in done_jobs:
log.info(job_entry.to_log())
commands.update_config_file()
def test_db(req):
print 'hit inside test_db'
data = {
"cpus":"1",
"dogs":"3"
}
experiment = Experiment(name="experiment a", details=data)
experiment.save()
return {}
def create_experiment(username, experiment):
author = username
date_time = datetime.now(pytz.timezone(country_timezones["AR"][1]))
resume = json.loads(experiment)["settings"]["a_description"]
status = "created"
e = Experiment(author=author,
experiment=experiment,
date_time=date_time,
resume=resume,
status=status)
e.save()
return e.id
def participate(experiment,
alternatives,
client_id,
force=None,
bucket=None,
traffic_fraction=None,
prefetch=False,
datetime=None,
redis=None):
exp = Experiment.find_or_create(experiment,
alternatives,
traffic_fraction=traffic_fraction,
redis=redis)
alt = None
if force and force in alternatives:
alt = Alternative(force, exp, redis=redis)
elif not cfg.get('enabled', True):
alt = exp.control
elif exp.winner is not None:
alt = exp.winner
else:
client = Client(client_id, redis=redis)
alt = exp.get_alternative(client,
dt=datetime,
prefetch=prefetch,
force=bucket)
return alt
def experiment():
form = ExperimentForm()
if request.method == "POST":
if form.validate() == False:
return render_template("experiment.html", form=form)
else:
title = form.title.data
description = form.description.data
etype = form.etype.data
protocol = form.protocol.data
duration = form.duration.data
username = session['username']
user = User.query.filter_by(username=username).first()
exp = Experiment(uid=user.id,
title=title,
duration=duration,
protocol=protocol,
description=description)
db.session.add(exp)
db.session.flush()
session['current_experiment'] = exp.eid
db.session.commit()
print(exp.eid)
return redirect(url_for('node'))
elif request.method == "GET":
return render_template("experiment.html", form=form)
def new_experiment(url, name):
raw = get_dataset_from_url(url)
with database.atomic():
experiment = Experiment.create(source_url=url, name=name)
Data.bulk_create([Data(**r, experiment=experiment) for r in raw])
return experiment
def on_experiment_details(self, request, name):
exp = Experiment.find(name, redis=self.redis)
if exp is None:
return json_error({'message': 'experiment not found'}, request,
404)
return json_success(exp.objectify_by_period('day', True), request)
def on_traffic():
experiment_name = request.args.get('experiment')
traffic_fraction = request.args.get('traffic_fraction')
if traffic_fraction is None or experiment_name is None:
return utils.json_error(
{'message': 'set traffic fail because of missing arguments'},
request, 400)
traffic_fraction = float(traffic_fraction)
try:
exp = Experiment.find(experiment_name, redis=db.REDIS)
exp.set_traffic_fraction(traffic_fraction)
exp.save()
except ValueError as e:
return utils.json_error({'message': str(e)}, request, 400)
resp = {
'experiment': {
'name': experiment_name,
},
'traffic_fraction': traffic_fraction,
'status': 'ok'
}
return utils.json_success(resp, request)
def experiment_list():
experiments = Experiment.all(redis=db.REDIS)
period = determine_period()
experiments = [
simple_markdown(exp.objectify_by_period(period)) for exp in experiments
]
return jsonify({'experiments': experiments})
def participate(experiment, alternatives, client_id,
force=None,
record_force=False,
traffic_fraction=None,
prefetch=False,
datetime=None,
redis=None):
exp = Experiment.find_or_create(experiment, alternatives, traffic_fraction=traffic_fraction, redis=redis)
alt = None
if force and force in alternatives:
alt = Alternative(force, exp, redis=redis)
if record_force:
client = Client(client_id, redis=redis)
alt.record_participation(client, datetime)
elif not cfg.get('enabled', True):
alt = exp.control
elif exp.winner is not None:
alt = exp.winner
else:
client = Client(client_id, redis=redis)
alt = exp.get_alternative(client, dt=datetime, prefetch=prefetch)
return alt
def on_alt_fractions(self, request):
experiment_name = request.args.get('experiment')
if experiment_name is None:
return json_error(
{
'message':
'set alt_fractions fail because of missing experiment name'
}, request, 400)
try:
exp = Experiment.find(experiment_name, redis=self.redis)
alt_fractions = []
for alternabive in exp.alternatives:
alt_fractions.append(request.args.get(alternabive.name))
exp.set_alt_fractions(alt_fractions)
exp.save()
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'experiment': {
'name': experiment_name,
},
'alt_fraction': exp.alt_fractions,
'status': 'ok'
}
return json_success(resp, request)
def on_participate(self, request):
#获取分组列表
alts = request.args.getlist('alternatives')
# alt_fractions = request.args.getlist('alt_fractions')
experiment_name = request.args.get('experiment') #实验名称
force = request.args.get('force')
#获取record_force参数，并进行to_bool判断若为[y,ture,yes],返回Ture,否则False
record_force = to_bool(request.args.get('record_force', 'false'))
client_id = request.args.get('client_id') #获取client_id
traffic_fraction = request.args.get('traffic_fraction') #获取参与比例
if traffic_fraction is not None:
traffic_fraction = float(
traffic_fraction) #若traffic_fraction存在则转为float()
prefetch = to_bool(request.args.get(
'prefetch',
'false')) #获取prefetch，并进行to_bool判断若为[y,ture,yes],返回Ture,否则Fals
if client_id is None or experiment_name is None or alts is None: #三个必要参数，否则返回400
return json_error({'message': 'missing arguments'}, request, 400)
dt = None
if request.args.get(
"datetime"): #获取datetime参数，转化为datetime.datetime()类型，没有则默认None
dt = dateutil.parser.parse(request.args.get("datetime"))
try:
if should_exclude_visitor(
request): #判断user_agent是爬虫或者IP为指定排除的ip,则拒绝参与
exp = Experiment.find(experiment_name, redis=self.redis)
if exp.winner is not None: #判断是否已经胜出
alt = exp.winner
else:
alt = exp.control #没有胜出，返回分组中的第一个赋值给alt
else:
alt = participate(
experiment_name,
alts,
client_id,
force=force,
record_force=record_force,
traffic_fraction=traffic_fraction,
# alt_fraction=alt_fractions,
prefetch=prefetch,
datetime=dt,
redis=self.redis)
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'alternative': {
'name': alt.name
},
'experiment': {
'name': alt.experiment.name,
},
'client_id': client_id,
'status': 'ok'
}
return json_success(resp, request)
def find_or_404(experiment_name):
try:
exp = Experiment.find(experiment_name, db.REDIS)
if request.args.get('kpi'):
exp.set_kpi(request.args.get('kpi'))
return exp
except ValueError:
abort(404)
def find_or_404(experiment_name):
try:
exp = Experiment.find(experiment_name, db.REDIS)
if request.args.get("kpi"):
exp.set_kpi(request.args.get("kpi"))
return exp
except:
abort(404)
def find_or_404(experiment_name):
try:
exp = Experiment.find(experiment_name, db.REDIS)
if request.args.get('kpi'):
exp.set_kpi(request.args.get('kpi'))
return exp
except ValueError:
abort(404)
def config_upload(req):
#print 'req is: '+str(req)
#print 'req.post is: '+str(req.POST)
#print 'req files is '+str(req.FILES)
#data = req.FILES['uploadedFile'].read()
data = json.load(req.FILES['uploadedFile'])
filename = req.FILES['uploadedFile']
#json_data = json.load(data)
#for s in data:
#print s
#TODO filename != name...
experiment = Experiment(name=filename, details=data)
experiment.save()
return render_to_response(
'index.html', {
}, context_instance=RequestContext(req)
)
def find_or_404(experiment_name):
try:
experiment_name = url=urllib.unquote(experiment_name).decode('utf8')
exp = Experiment.find(experiment_name, db.REDIS)
if request.args.get('kpi'):
exp.set_kpi(request.args.get('kpi'))
return exp
except ValueError:
abort(404)
def run(self, experiment: Experiment) -> SPIAResult:
"""
Returns: a list of pathways: pathway id, pathway name, pNDE, pPERT, pG, FDR correction,
Bonferroni correction, status for each pathway
"""
pvalue = ttest(experiment) <= self.threshold
calc_f = experiment.calculate_fold_change()
all = pvalue.index.tolist()
for a in range(len(all)):
all[a] = all[a].name
p = pvalue[pvalue == True].index.tolist()
de = {}
for i in p:
de[i.name] = calc_f["FC"][i]
json = {}
if len(de) == 0:
# if there are no DEGs anywhere, the problem of finding the impact on various pathways is meaningless
print('No differentialy expressed genes.')
return SPIAResult([])
db = KEGGPathways(self.organism)
pathways = {}
for gene in de.keys():
ps = db.search_by_gene(gene)
for (k, v) in ps.items():
if k not in pathways.keys():
pathways[k] = v
if not pathways:
print('No pathways found in database.')
return SPIAResult([])
for (id, descr) in pathways.items():
pathway = db.get_pathway(id)
path_genes = set(pathway.nodes)
path_genes = list(path_genes)
interaction_list = {i: [] for i in rel}
x = get_edge_attributes(pathway, 'type')
for gene1, interaction in x.items():
interaction = '_'.join(interaction)
if interaction in interaction_list.keys():
interaction_list[interaction].append([
path_genes.index(gene1[0]),
path_genes.index(gene1[1])
])
else:
interaction_list[interaction] = [[
path_genes.index(gene1[0]),
path_genes.index(gene1[1])
]]
interaction_list['row_names'] = path_genes
json[id] = interaction_list
json['id2name'] = pathways
s = SPIA.calculate_spia(de, all, json)
result = SPIAResult(s)
if self.markdown:
result.generate_markdown(self.markdown, 'Results of SPIA:')
return result
def find_or_404(experiment_name):
try:
experiment_name = url = urllib.unquote(experiment_name).decode('utf8')
exp = Experiment.find(experiment_name, db.REDIS)
if request.args.get('kpi'):
exp.set_kpi(request.args.get('kpi'))
return exp
except ValueError:
abort(404)
def add_experiment(req):
experiment_name = req.POST['experiment_name']
filename = req.FILES['uploadedFile']
data = json.load(req.FILES['uploadedFile'])
experiment = Experiment(experiment_name=experiment_name, filename=filename, config_options=data)
experiment.save()
print str(experiment.id)
# chop off fractional second
left = experiment.last_modified.isoformat()[0:19]
right = experiment.last_modified.isoformat()[26:]
last_modified_fix = left + right
return {
'id': experiment.id,
'experiment_name': experiment_name,
'filename': str(filename),
'last_modified': last_modified_fix,
}
def convert(experiment, client_id, kpi=None, datetime=None, redis=None):
exp = Experiment.find(experiment, redis=redis)
if cfg.get('enabled', True):
client = Client(client_id, redis=redis)
alt = exp.convert(client, dt=datetime, kpi=kpi)
else:
alt = exp.control
return alt
def experiment_new():
form=ExperimentF(request.form)
if form.cancel.data:
return redirect(url_for('dashboard'))
if form.validate_on_submit():
exp=Experiment()
form.populate_obj(exp)
db_session.add(exp)
db_session.commit()
return redirect(url_for('.experiment_info',expid=exp.id))
else:
return render_template('experiment_edit.html',form=form,type='create',recent=recent.get())
def on_participate(self, request):
alts = request.args.getlist('alternatives')
experiment_name = request.args.get('experiment')
bucket = request.args.get('bucket')
force = request.args.get('force')
client_id = request.args.get('client_id')
traffic_fraction = request.args.get('traffic_fraction')
if traffic_fraction is not None:
traffic_fraction = float(traffic_fraction)
prefetch = to_bool(request.args.get('prefetch', 'false'))
if client_id is None or experiment_name is None or alts is None:
return json_error({'message': 'missing arguments'}, request, 400)
dt = None
if request.args.get("datetime"):
dt = dateutil.parser.parse(request.args.get("datetime"))
if should_exclude_visitor(request):
exp = Experiment.find(experiment_name, redis=self.redis)
if exp.winner is not None:
alt = exp.winner
else:
alt = exp.control
else:
try:
alt = participate(experiment_name,
alts,
client_id,
force=force,
bucket=bucket,
traffic_fraction=traffic_fraction,
prefetch=prefetch,
datetime=dt,
redis=self.redis)
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'alternative': {
'name': alt.name
},
'experiment': {
'name': alt.experiment.name,
},
'client_id': client_id,
'status': 'ok'
}
return json_success(resp, request)
def convert(experiment, client_id,
kpi=None,
datetime=None,
redis=None):
exp = Experiment.find(experiment, redis=redis)
if cfg.get('enabled', True):
client = Client(client_id, redis=redis)
alt = exp.convert(client, dt=datetime, kpi=kpi)
else:
alt = exp.control
return alt
def convert(experiment, client_id,
kpi=None,
datetime=None,
redis=None):
""""""
exp = Experiment.find(experiment, redis=redis)#查找实验，得到实验对象
if cfg.get('enabled', True):#判断是否为启用状态
client = Client(client_id, redis=redis)#创建client对象
alt = exp.convert(client, dt=datetime, kpi=kpi)#调用exp对象的convert方法
else:
alt = exp.control
return alt
def client_experiments(api_key,
client_id,
kpi=None,
redis=None,
exclude_paused=True,
exclude_archived=True):
client = Client(client_id, redis=redis)
alternatives = []
running_experiments = Experiment.all(api_key,
redis=redis,
exclude_paused=exclude_paused,
exclude_archived=exclude_archived)
for experiment in running_experiments:
alternatives.append(experiment.get_alternative(client))
return alternatives
def on_participate(self, request):
opts = {}
alts = request.args.getlist('alternatives')
experiment_name = request.args.get('experiment')
force = request.args.get('force')
client_id = request.args.get('client_id')
distribution = request.args.get('traffic_dist')
if client_id is None or experiment_name is None or alts is None:
return json_error({'message': 'missing arguments'}, request, 400)
if distribution:
opts['distribution'] = distribution
try:
experiment = Experiment.find_or_create(experiment_name, alts,
self.redis, opts)
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
alternative = None
if force and force in alts:
alternative = force
elif not cfg.get('enabled', True):
alternative = alts[0]
elif experiment.winner is not None:
alternative = experiment.winner
elif should_exclude_visitor(request):
alternative = alts[0]
else:
dt = None
if request.args.get("datetime"):
dt = dateutil.parser.parse(request.args.get("datetime"))
client = Client(client_id, self.redis)
alternative = experiment.get_alternative(client, dt=dt).name
resp = {
'alternative': {
'name': alternative
},
'experiment': {
'name': experiment.name,
},
'client_id': client_id,
'status': 'ok'
}
return json_success(resp, request)
def on_participate(self, request):
opts = {}
alts = request.args.getlist('alternatives')
experiment_name = request.args.get('experiment')
force = request.args.get('force')
client_id = request.args.get('client_id')
distribution = request.args.get('traffic_dist')
if client_id is None or experiment_name is None or alts is None:
return json_error({'message': 'missing arguments'}, request, 400)
if distribution:
opts['distribution'] = distribution
try:
experiment = Experiment.find_or_create(experiment_name, alts, self.redis, opts)
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
alternative = None
if force and force in alts:
alternative = force
elif not cfg.get('enabled', True):
alternative = alts[0]
elif experiment.winner is not None:
alternative = experiment.winner
elif should_exclude_visitor(request):
alternative = alts[0]
else:
dt = None
if request.args.get("datetime"):
dt = dateutil.parser.parse(request.args.get("datetime"))
client = Client(client_id, self.redis)
alternative = experiment.get_alternative(client, dt=dt).name
resp = {
'alternative': {
'name': alternative
},
'experiment': {
'name': experiment.name,
},
'client_id': client_id,
'status': 'ok'
}
return json_success(resp, request)
def ttest(experiment: Experiment):
"""This is a two-sided test for the null hypothesis that 2 independent samples have identical average (expected) values.
This test assumes that the populations have identical variances by default.
Args:
experiment: Experiment object with case and control sample(s)
Returns:
one-dimensional labeled array of p-values with Gene objects as labels
"""
pvals = {}
for (idx, row) in experiment.get_all().as_array().iterrows():
control = [row[label] for label in experiment.control.labels]
case = [row[label] for label in experiment.case.labels]
pvals[idx] = ttest_ind(control, case).pvalue
return pd.Series(pvals, name='p-value')
def on_user_experiment_alternatives(self, request, name):
api_key = request.args.get('api_key', None)
start = request.args.get('sequence_id_start') or 1
page_size = request.args.get('page_size') or 5000
end = int(start) + int(page_size) - 1
exp = Experiment.find(api_key, name, redis=self.redis)
if exp is None:
return json_error({'message': 'experiment not found'}, request, 404)
user_alternatives = experiment_user_alternatives(
api_key, exp, redis=self.redis, start=start, end=end)
resp = {
'alternatives': [
{'user_id': user_id, 'alternative_name': alt} for user_id, alt in user_alternatives.iteritems()
]
}
return json_success(resp, request)
def experiment_clone(expid=None):
if not expid:
expid = request.cookies.get('project')
if not expid:
return render_template('error.html',message="Proyecto no seleccionado",recent=recent.get())
expid=int(expid)
exp_=db_session.query(Experiment).get(expid)
if not exp_:
return render_template('error.html',message="Experimento no definido",recent=recent.get())
exp=Experiment()
exp.name="CLONE:"+exp_.name
exp.definition=exp_.definition
exp.description=exp_.description
exp.instructions=exp_.instructions
exp.invitation=exp_.invitation
exp.reinvitation=exp_.reinvitation
db_session.add(exp)
db_session.commit()
return redirect(url_for('.experiment_info',expid=exp.id))
def on_participate(self, request):
alts = request.args.getlist('alternatives')
experiment_name = request.args.get('experiment')
force = request.args.get('force')
record_force = to_bool(request.args.get('record_force', 'false'))
client_id = request.args.get('client_id')
traffic_fraction = request.args.get('traffic_fraction')
if traffic_fraction is not None:
traffic_fraction = float(traffic_fraction)
prefetch = to_bool(request.args.get('prefetch', 'false'))
if client_id is None or experiment_name is None or alts is None:
return json_error({'message': 'missing arguments'}, request, 400)
dt = None
if request.args.get("datetime"):
dt = dateutil.parser.parse(request.args.get("datetime"))
try:
if should_exclude_visitor(request):
exp = Experiment.find(experiment_name, redis=self.redis)
if exp.winner is not None:
alt = exp.winner
else:
alt = exp.control
else:
alt = participate(experiment_name, alts, client_id,
force=force, record_force=record_force,
traffic_fraction=traffic_fraction,
prefetch=prefetch, datetime=dt, redis=self.redis)
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'alternative': {
'name': alt.name
},
'experiment': {
'name': alt.experiment.name,
},
'client_id': client_id,
'status': 'ok'
}
return json_success(resp, request)
def on_convert(self, request):
if should_exclude_visitor(request):
return json_success({'excluded': 'true'}, request)
experiment_name = request.args.get('experiment')
client_id = request.args.get('client_id')
kpi = request.args.get('kpi', None)
if client_id is None or experiment_name is None:
return json_error({'message': 'missing arguments'}, request, 400)
client = Client(client_id, self.redis)
try:
experiment = Experiment.find(experiment_name, self.redis)
if cfg.get('enabled', True):
dt = None
if request.args.get("datetime"):
dt = dateutil.parser.parse(request.args.get("datetime"))
alternative = experiment.convert(client, dt=dt, kpi=kpi)
else:
alternative = experiment.control.name
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'alternative': {
'name': alternative
},
'experiment': {
'name': experiment.name,
},
'conversion': {
'value': None,
'kpi': kpi
},
'client_id': client_id
}
return json_success(resp, request)
def log_file_analyse():
from models import Experiment, Queue
print("Name:", request.form['name-input'])
print("Tags:", request.form['tags-input'])
print("Notes", request.form['name-input'])
print("Author", request.form['author-input'])
print("Dataset alias", request.form['alias-input'])
print("Dataset:", request.files['datasetInputFile'])
name = request.form['name-input']
tags_s = request.form['tags-input']
notes = request.form['name-input']
authors_s = request.form['author-input']
alias = request.form['alias-input']
file = request.files['datasetInputFile']
tags = tags_s.split(',')
authors = authors_s.split(',')
filename = None
if file.filename == '':
print("PROOOBLEM!!!!")
return redirect(url_for("home"))
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
print("here")
e = Experiment(name=name, data_file_path=filename, authors=authors, tags=tags, notes=notes, alias=alias)
q = Queue(e)
db.session.add(e)
db.session.add(q)
db.session.commit()
print("New task")
queue.create_new_task(q)
return redirect(url_for('queue'))
def on_convert(self, request):
if should_exclude_visitor(request):
return json_success({'excluded': 'true'}, request)
experiment_name = request.args.get('experiment')
client_id = request.args.get('client_id')
kpi = request.args.get('kpi', None)
if client_id is None or experiment_name is None:
return json_error({'message': 'missing arguments'}, request, 400)
client = Client(client_id, self.redis)
try:
experiment = Experiment.find(experiment_name, self.redis)
if cfg.get('enabled', True):
dt = None
if request.args.get("datetime"):
dt = dateutil.parser.parse(request.args.get("datetime"))
alternative = experiment.convert(client, dt=dt, kpi=kpi)
else:
alternative = experiment.control.name
except ValueError as e:
return json_error({'message': str(e)}, request, 400)
resp = {
'alternative': {
'name': alternative
},
'experiment': {
'name': experiment.name,
},
'conversion': {
'value': None,
'kpi': kpi
},
'client_id': client_id
}
return json_success(resp, request)
def copyProjectExperiments(project, new_project):
t_experiments = Experiment.objects.filter(project=project)
for experiment in t_experiments:
new_experiment = Experiment(project=new_project,
name=str(experiment.name),
notes=str(experiment.notes))
new_experiment.save()
t_conditions = Condition.objects.filter(experiment=experiment)
for condition in t_conditions:
new_condition = Condition(experiment=new_experiment,
name=str(condition.name),
notes=str(condition.notes))
new_condition.save()
t_observations = Observation.objects.filter(condition=condition)
for t_observation in t_observations:
new_observation = Observation(
condition=new_condition,
species=t_observation.species,
time=t_observation.time,
value=t_observation.value,
stddev=t_observation.stddev,
steady_state=t_observation.steady_state,
min_steady_state=t_observation.min_steady_state,
max_steady_state=t_observation.max_steady_state)
new_observation.save()
t_treatments = Treatment.objects.filter(condition=condition)
for t_treatment in t_treatments:
new_treatment = Treatment(condition=new_condition,
species=t_treatment.species,
time=t_treatment.time,
value=t_treatment.value)
new_treatment.save()
new_condition.save()
new_experiment.save()
def updateTaskInDb(name, status, pk=None):
# acquire lock
dbUpdateLock.acquire()
try:
# check if entry in db
if pk is None:
# if not in db, make new one
ne = Experiment(name=name, status=status)
ne.save()
pk = ne.pk
else:
# if present, retrieve and update it
ne = Experiment.objects.get(pk=pk)
ne.status = status
ne.save()
pk = ne.pk
except:
print "Error in updating database entry"
pk = None
finally:
dbUpdateLock.release()
return pk
def experiment_list():
experiments = Experiment.all(db.REDIS)
period = determine_period()
experiments = [simple_markdown(exp.objectify_by_period(period)) for exp in experiments]
return jsonify({'experiments': experiments})
def archived():
experiments = Experiment.all(db.REDIS, False)
experiments = [exp.name for exp in experiments if exp.is_archived()]
return render_template('dashboard.html', experiments=experiments, page='archived')
def hello():
experiments = Experiment.all(db.REDIS)
experiments = [exp.name for exp in experiments]
return render_template('dashboard.html', experiments=experiments, page='home')
def paused():
experiments = Experiment.paused(redis=db.REDIS)
experiments = [exp.name for exp in experiments]
return render_template('dashboard.html', experiments=experiments, page='paused')
def __init__(self,protocol,nickname):
dt = datetime.now()
self.experiment = Experiment(name=nickname,protocol=protocol,time_start=dt)
self.experiment.save()
self.executioner = Executioner()
libarian.init_db_cache(self.experiment)
class Dictator(object):
executioner = None
experiment = None
eventObjects = None
def __init__(self,protocol,nickname):
dt = datetime.now()
self.experiment = Experiment(name=nickname,protocol=protocol,time_start=dt)
self.experiment.save()
self.executioner = Executioner()
libarian.init_db_cache(self.experiment)
# self.lock = Lock()
def current_trial(self):
return poke_cache(str(self.experiment.id)+'.trial_current',self.experiment.current_trial(),secs=60)
def start(self):
self.executioner.start()
def complete(self):
current_trial = self.experiment.current_trial()
if current_trial!=None:
current_trial.duration = libarian.time_since_trial(self.experiment.id)
current_trial.save()
self.experiment.total_duration = libarian.time_since_exp(self.experiment.id)
self.experiment.set_trials_completed()
self.experiment.time_complete = datetime.now()
self.experiment.save()
# libarian.clear_db_cache()
def new_trial(self):
total_time = libarian.time_since_exp(self.experiment.id)
trial_time = libarian.time_since_trial(self.experiment.id)
new_trial = Trial(experiment=self.experiment, duration=Decimal(0.0),completed=False,time_start=datetime.now())
new_trial.save()
old_trial = libarian.get_trial_current(self.experiment.id)
if old_trial!=None:
w.NextTrialThread(old_trial,new_trial,trial_time,total_time,self.experiment).start()
else:
w.NewHappening('TRL','New Trial',total_time,self.experiment).start()
libarian.set_trial_current(new_trial,self.experiment.id)
self.executioner.interval_pointer = 0
self.executioner.is_new_trial = True
libarian.set_interval_start(datetime.now(),self.experiment.id)
def check_emulate_action(self,action_type):
if em.Action.objects.filter(type=action_type).count()==1:
act = em.Action.objects.filter(type=action_type)[0]
try:
ea = EmulateAction.objects.filter(experiment=self.experiment, action=act).latest('id')
self.action_happen('%s'%(action_type),action_type,ea.time_occurred)
ea.delete()
return True
except ObjectDoesNotExist:
return False
else:
return False
def run_simulate_events(self):
event_ids = SimEvent.objects.filter(experiment=self.experiment)
for se in event_ids:
self.eventObjects[se.eventid].perform(time=se.time_occurred)
event_ids.delete()
def action_happen(self,description,keyname,given_time=None):
if given_time==None:
time = libarian.time_since_exp(self.experiment.id)
else:
time = given_time
thready = w.NewHappening('ACT',keyname,description,time,self.experiment)
thready.start()
def event_happen(self,description,keyname,given_time=None):
if given_time==None:
time = libarian.time_since_exp(self.experiment.id)
else:
time = given_time
thready = w.NewHappening('EVT',keyname,description,time,self.experiment)
thready.start()
def interval_happen(self,description,keyname):
time = libarian.time_since_exp(self.experiment.id)
thready = w.NewHappening('ITL',keyname,description,time,self.experiment)
thready.start()
def paused():
experiments = Experiment.paused(redis=db.REDIS)
experiments = [exp.name for exp in experiments]
return render_template('dashboard.html',
experiments=experiments,
page='paused')
def hello():
experiments = Experiment.all(redis=db.REDIS)
experiments = [exp.name for exp in experiments]
return render_template('dashboard.html',
experiments=experiments,
page='home')
def load_experiment(experiment_id):
experiment = Experiment.gql('WHERE __key__ = KEY(:1)', experiment_id).get()
treatments = Treatment.gql('WHERE ANCESTOR IS KEY(:1)', experiment_id)
properties = Property.gql('WHERE ANCESTOR IS KEY(:1)', experiment_id)
return (experiment, treatments, properties)
def on_experiment_details(self, request, name):
exp = Experiment.find(name, redis=self.redis)
if exp is None:
return json_error({'message': 'experiment not found'}, request, 404)
return json_success(exp.objectify_by_period('day', True), request)
def archived():
set_redis_prefix()
experiments = Experiment.all(exclude_archived=False, redis=db.REDIS)
experiments = [exp.name for exp in experiments if exp.is_archived()]
return render_template('dashboard.html', experiments=experiments, page='archived')
def hello():
experiments = Experiment.all(exclude_archived=True, redis=db.REDIS)
experiments = [exp.name for exp in experiments]
return render_template('dashboard.html', experiments=experiments, page='home')
def get(self):
user = users.get_current_user()
exps = Experiment.gql('WHERE owner = :1', user)
template = jinja_environment.get_template('index.html')
self.response.out.write(template.render(user=user, experiments=exps))
def run(self, experiment: Experiment) -> ImpactAnalysisResult:
"""
Returns:
list of pathways sorted by their impact factor. Each pathway in the list has values of FDR and
Bonferroni corrections assigned.
"""
self.experiment_genes = set(
[gene.name for gene in experiment.get_all().genes])
# calculate fold change
self.FC = experiment.calculate_fold_change()
# remove genes for witch fold change cannot be calculated correctly
experiment.exclude_genes(
list(self.FC['FC'][isnan(self.FC['FC'])].index))
if self.degs:
self.degs = pd.Series({
Gene(x): True
for x in self.degs if Gene(x) not in self.experiment_genes
})
else:
# select differentialy expressed genes
pvalue = ttest(experiment) <= self.threshold
self.degs = pvalue[pvalue == True]
if self.degs.size == 0:
# if there are no DEGs anywhere, the problem of finding the impact on various pathways is meaningless
print('No differentialy expressed genes.')
return ImpactAnalysisResult([])
db = KEGGPathways(self.org)
pathways = {}
for gene in [g.name for g in list(self.degs.index)]:
ps = db.search_by_gene(gene)
for (k, v) in ps.items():
if k not in pathways.keys():
pathways[k] = v
if not pathways:
print('No pathways found in database.')
return ImpactAnalysisResult([])
res = pd.DataFrame(columns=['name', 'IF', 'pvalue'])
for (code, descr) in pathways.items():
pathway = db.get_pathway(code)
impact_factor, pval = self.calculate_impact_factor(
experiment, pathway)
if impact_factor is not None and pval is not None:
res.loc[len(res.index)] = [descr, impact_factor, pval]
res['FDR'], res['Bonferroni'] = self.calculate_corrections(
res['pvalue'])
ifp_pathways = [IAPathway(res.loc[i]) for i in range(len(res.index))]
ifp_pathways.sort(key=lambda x: x.IF if not isnan(x.IF) else 0,
reverse=True)
result = ImpactAnalysisResult(ifp_pathways)
if self.markdown:
result.generate_markdown(self.markdown,
'Results of Impact Analysis:')
return result