def __init__(self):
'Create a new experiment with given sample locations for water and WASTE; totalTime is expected run time in seconds, if known'
worklist.comment("Generated %s" % (datetime.now().ctime()))
worklist.userprompt(
"The following reagent tubes should be present: %s" %
Sample.getAllLocOnPlate(decklayout.REAGENTPLATE))
worklist.userprompt(
"The following eppendorf tubes should be present: %s" %
Sample.getAllLocOnPlate(decklayout.EPPENDORFS))
worklist.email(dest='*****@*****.**',
subject='Run started (Generate: %s)' %
(datetime.now().ctime()))
worklist.email(dest='*****@*****.**',
subject='Tecan error',
onerror=1)
self.cleanTips = 0
# self.sanitize() # Not needed, TRP does it, also first use of tips will do this
self.useDiTis = False
self.ptcrunning = False
self.overrideSanitize = False
self.pgmStartTime = None
self.pgmEndTime = None
# Access PTC and RIC early to be sure they are working
worklist.pyrun("PTC\\ptctest.py")
# worklist.periodicWash(15,4)
worklist.userprompt(
"Verify that PTC thermocycler lid pressure is set to '2'.")
self.idlePgms = []
self.timerStartTime = [None] * 8
def test_calc_speed(self):
sample = Sample()
self.assertEqual(sample.calc_speed(distance=10.0, elapsed_time=2.0),
5.0)
self.assertEqual(sample.calc_speed(distance=10, elapsed_time=2), 5.0)
self.assertEqual(sample.calc_speed(distance=10, elapsed_time=4), 2.5)
self.assertEqual(sample.calc_speed(distance=0, elapsed_time=2), 0.0)
def initialDBScanSciLearn(self):
db = DBSCAN(eps=8, min_samples=self.minPts,
algorithm='brute').fit(self.buffer)
clusters = db.labels_
self.buffer['clusters'] = clusters
clusterNumber = np.unique(clusters)
for clusterId in clusterNumber:
if (clusterId != -1):
cl = self.buffer[self.buffer['clusters'] == clusterId]
cl = cl.drop('clusters', axis=1)
sample = Sample(cl.iloc[0].tolist())
mc = MicroCluster(sample, self.currentTimestamp, self.lamb)
for sampleNumber in range(len(cl[1:])):
sample = Sample(cl.iloc[sampleNumber].tolist())
mc.insertSample(sample, self.currentTimestamp)
self.pMicroCluster.insert(mc)
def make_ready_for_test(network_dir, test_name, condition, test_sub_name):
"""
>>> test_files_directory = get_test_files_directory()
>>> result = make_ready_for_test(test_files_directory, "normal", "test1","aggregate")
>>> len(result) == 2
True
"""
sample_name = get_sample_name(test_name)
sample_file_path = os.path.join(network_dir, sample_name)
# There should be sample files
assert os.path.exists(sample_file_path), "No sample file at %s" % sample_file_path
net_file_path = os.path.join(network_dir, test_name + ".txt")
dot_file_path = net_file_path + ".dot"
if os.path.exists(net_file_path):
if not os.path.exists(dot_file_path):
n = Network(net_file_path)
dumb = n.dot_gen(dot_file_path)
# get the target root file
test_report_directory = network_dir + os.sep + condition
test_report_sub_directory = test_report_directory + os.sep + test_sub_name
if os.path.exists(test_report_sub_directory):
shutil.rmtree(test_report_sub_directory)
os.makedirs(test_report_sub_directory)
sample = Sample()
sample.read(sample_file_path)
return test_report_sub_directory, sample
def test_find_next_pixel(self):
Nx = 3
Ny = 3
Jc = np.zeros((Nx, Ny))
sample = Sample(Jc)
sample.boolean_matrix[0, 1] = False
sample.boolean_matrix[0, 0] = False
nxt = find_next_pixel(0, 0)
self.assertEqual(nxt, (1, 0))
sample.boolean_matrix[nxt] = False
nxt = find_next_pixel(nxt)
self.assertEqual(nxt, (2, 0))
sample.boolean_matrix[nxt] = False
nxt = find_next_pixel(nxt)
self.assertEqual(nxt, (2, 1))
sample.boolean_matrix[nxt] = False
nxt = find_next_pixel(nxt)
self.assertEqual(nxt, (2, 2))
sample.boolean_matrix[nxt] = False
nxt = find_next_pixel(nxt)
self.assertEqual(nxt, (1, 2))
sample.boolean_matrix[nxt] = False
nxt = find_next_pixel(nxt)
self.assertEqual(nxt, (0, 2))
sample.boolean_matrix[nxt] = False
nxt = find_next_pixel(nxt)
self.assertEqual(nxt, ())
def __init__(self):
'Create a new experiment with given sample locations for water and WASTE; totalTime is expected run time in seconds, if known'
self.checksum=md5sum(sys.argv[0])
self.checksum=self.checksum[-4:]
pyTecan=os.path.dirname(os.path.realpath(__file__))
self.gitlabel=strip(subprocess.check_output(["git", "describe","--always"],cwd=pyTecan))
worklist.comment("Generated %s (%s-%s pyTecan-%s)"%(datetime.now().ctime(),sys.argv[0],self.checksum,self.gitlabel))
worklist.userprompt("The following reagent tubes should be present: %s"%Sample.getAllLocOnPlate(decklayout.REAGENTPLATE))
worklist.userprompt("The following eppendorf tubes should be present: %s"%Sample.getAllLocOnPlate(decklayout.EPPENDORFS))
worklist.email(dest='*****@*****.**',subject='Run started (Generate: %s) expected runtime %.0f minutes'%(datetime.now().ctime(),clock.totalTime/60.0 if clock.totalTime is not None else 0.0 ) )
worklist.email(dest='*****@*****.**',subject='Tecan error',onerror=1)
self.cleanTips=0
# self.sanitize() # Not needed, TRP does it, also first use of tips will do this
self.useDiTis=False
self.ptcrunning=False
self.overrideSanitize=False
self.pgmStartTime=None
self.pgmEndTime=None
# Access PTC and RIC early to be sure they are working
worklist.pyrun("PTC\\ptctest.py")
# worklist.periodicWash(15,4)
worklist.userprompt("Verify that PTC thermocycler lid pressure is set to '2'.")
self.idlePgms=[]
self.timerStartTime=[None]*8
def __init__(self, samp0_is_population=None, samp1_is_population=None):
self.samp0 = Sample(title='samp0', is_population=samp0_is_population)
"""The first sample, is usually the population, or the pre-test sample"""
self.samp1 = Sample(title='samp1', is_population=samp1_is_population)
"""The second sample, the post-test sample."""
# Parameters
self.alpha = 0.05
"""Requested confidence level"""
self.dir = None # see StatTool.xxx_TEST constants
"""Directionality of the test."""
# Expected difference
self.expected_difference = 0.0
"""The expected difference between the two sample means.
For two tailed test, usually we write the hypothesis as μ1 != μ2.
This can be rewritten as μ1 - μ2 != 0. And actually the general
expression is μ1 - μ2 != expected_difference.
"""
# Description
self.treatment_title = "treatment"
"""The name of the treatment"""
self.results_title = "results"
"""The name of the dependent variable"""
def test1(self):
'Neutrons_from above'
rq = lambda Q: np.exp(-Q * Q / 25)
s = Sample('sample', 5, 5, rq)
from mcni import neutron_buffer, neutron
N = 8
nb = neutron_buffer(N)
t = 1.
for i in range(N):
vi = np.array((0, -(i + 1) * 100, (i + 1) * 100))
ri = -vi * t
nb[i] = neutron(r=ri, v=vi, s=(0, 0), time=0., prob=1.)
continue
nb2 = s.process(nb)
for i, (n, n2) in enumerate(zip(nb, nb2)):
# print "input:", n
# print "output:", n2
vf = np.array((0, (i + 1) * 100, (i + 1) * 100))
np.allclose(vf, n2.state.velocity)
np.allclose([0, 0, 0], n2.state.position)
np.isclose(n2.time, 1.)
vy = vf[1]
Q = np.abs(vy * 2) * conv.V2K
np.isclose(rq(Q), n2.probability)
return
def runpgm(self,pgm,duration,waitForCompletion=True,volume=10,hotlidmode="TRACKING",hotlidtemp=1):
if self.ptcrunning:
logging.error("Attempt to start a progam on PTC when it is already running")
if len(pgm)>8:
logging.error("PTC program name (%s) too long (max is 8 char)"%pgm)
# move to thermocycler
worklist.flushQueue()
self.lihahome()
cmt="run %s"%pgm
worklist.comment(cmt)
#print "*",cmt
worklist.pyrun("PTC\\ptclid.py OPEN")
self.moveplate(decklayout.SAMPLEPLATE,"PTC")
worklist.vector("Hotel 1 Lid",decklayout.HOTELPOS,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.CLOSE)
worklist.vector("PTC200lid",decklayout.PTCPOS,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.OPEN)
worklist.romahome()
worklist.pyrun("PTC\\ptclid.py CLOSE")
# pgm="PAUSE30" # For debugging
assert hotlidmode=="TRACKING" or hotlidmode=="CONSTANT"
assert (hotlidmode=="TRACKING" and hotlidtemp>=0 and hotlidtemp<=45) or (hotlidmode=="CONSTANT" and hotlidtemp>30)
worklist.pyrun('PTC\\ptcrun.py %s CALC %s,%d %d'%(pgm,hotlidmode,hotlidtemp,volume))
self.pgmStartTime=clock.pipetting
self.pgmEndTime=duration*60+clock.pipetting
self.ptcrunning=True
Sample.addallhistory("{%s}"%pgm,addToEmpty=False,onlyplate=decklayout.SAMPLEPLATE.name)
if waitForCompletion:
self.waitpgm()
def create_deck(self):
'''Lesson4-1
'''
# カウント用変数初期化
i = 0
j = 0
# デッキ用リストを初期化
DECK = []
try:
# スーツをリスト化
Suits = Sample().get_suits()
# ランクをリスト化
Ranks = Sample().get_ranks()
# カードを設定
while i < 4:
while j < 13:
Sample().set_trumpcard(Ranks[j], Suits[i])
DECK.append([Ranks[j], Suits[i]])
j += 1
i += 1
j = 0
# デッキを設定
Sample().set_deck(DECK)
# 設定
self.deck = DECK
except ValueError:
# 強制終了
sys.exit()
def test(model, optimizer, test_iterator, device, BATCH_SIZE):
model.eval()
test_loss = 0
kld_loss = 0
rcl_loss = 0
kl_per_lt = {
'Latent_Dimension': [],
'KL_Divergence': [],
'Latent_Mean': [],
'Latent_Variance': []
}
with torch.no_grad():
for i, (x, y) in enumerate(test_iterator):
model.to(device)
sm = Sample(x, y, BATCH_SIZE, device)
x, y = sm.generate_x_y()
x = x.view(-1, 3, 28, 28)
reconstructed_x, z_mu, z_var, _ = model(x, y)
blur = calc_blur(reconstructed_x)
for ii in range(z_mu.size()[-1]):
_, _, kl_per_lt_temp = calculate_loss(x, reconstructed_x,
z_mu[:, ii], z_var[:,
ii])
kl_per_lt['Latent_Dimension'].append(ii)
kl_per_lt['Latent_Mean'].append(z_mu[:, ii])
kl_per_lt['Latent_Variance'].append(z_var[:, ii])
loss, rcl, kld = calculate_loss(x, reconstructed_x, z_mu, z_var)
test_loss += loss.item()
rcl_loss += rcl.item()
kld_loss += kld.item()
return test_loss, rcl_loss, kld_loss, kl_per_lt, blur.data.item()
def load(self, filename):
"""
Load Metaphor Dataset from CSV file..
:param filename: name of the dataset file
:returns samples: list containing Sample objects
"""
samples = []
raw_samples = csv.DictReader(codecs.open(filename, 'r', 'latin-1'))
for sample in raw_samples:
if "a" in sample["sentence_id"] or len(sample["sentence_txt"].split()) <= 1:
continue
sentence = sample["sentence_txt"].replace("M_", "").replace("L_", "").lower().split()
self.discourse[sample["txt_id"]][int(sample["sentence_id"])] = sentence
sample = Sample(sentence=sample["sentence_txt"], text_id=sample["txt_id"],
sent_id=sample["sentence_id"])
samples.append(sample)
for sample in samples:
discourse, focus_position = self.get_discourse(sample.sentence, sample.text_id, sample.sent_id)
sample.update_discourse(discourse, focus_position)
if self.sort_data != 0:
samples = sorted(samples, key=lambda x: x.max_length)
return samples
def savesummary(self, filename, settings=None):
# Print amount of samples needed
fd = open(filename, "w")
# print >>fd,"Deck layout:"
# print >>fd,decklayout.REAGENTPLATE
# print >>fd,decklayout.SAMPLEPLATE
# print >>fd,decklayout.QPCRPLATE
# print >>fd,decklayout.WATERLOC
# print >>fd,decklayout.WASTE
# print >>fd,decklayout.BLEACHLOC
# print >>fd,decklayout.WASHLOC
# print >>fd
#print >>fd,"DiTi usage:",worklist.getDITIcnt()
#print >>fd
print >> fd, "Generated %s (%s-%s pyTecan-%s)" % (
datetime.now().ctime(), sys.argv[0], self.checksum, self.gitlabel)
rtime = "Run time: %d (pipetting only) + %d (thermocycling only) + %d (both) = %d minutes (%.1f hours)\n" % (
clock.pipetting / 60.0, clock.thermotime / 60,
clock.pipandthermotime / 60, clock.elapsed() / 60,
clock.elapsed() / 3600.0)
print rtime
print >> fd, rtime
reagents.printprep(fd)
Sample.printallsamples("All Samples:", fd, w=worklist)
liquidclass.LC.printalllc(fd)
if settings is not None:
pprint(settings, stream=fd)
fd.close()
def apply_action(self, action):
"""Apply the action to the grid.
If left is applied then the occupied state index will decrease by 1.
Unless the agent is already at 0, in which case the state will not
change.
If right is applied then the occupied state index will increase by 1.
Unless the agent is already at num_states-1, in which case the state
will not change.
The reward function is determined by the reward location specified when
constructing the domain.
If failure_probability is > 0 then there is the chance for the left
and right actions to fail. If the left action fails then the agent
will move right. Similarly if the right action fails then the agent
will move left.
Parameters
----------
action: int
Action index. Must be in range [0, num_actions())
Returns
-------
sample.Sample
The sample for the applied action.
Raises
------
ValueError
If the action index is outside of the range [0, num_actions())
"""
if action < 0 or action >= self.num_actions():
raise ValueError('Action index outside of bounds [0, %d)' %
self.num_actions())
new_location = self.next_location(self._state[0], action)
# in the case of failing action
if new_location == self._state[0] or random() > self.transition_probabilities[new_location]:
return Sample(self._state.copy(), action, 0., self._state.copy())
next_state = np.array([new_location])
if self.reward_location == new_location:
reward = 100.
absorb = True
sample = Sample(self._state.copy(), action, reward, next_state.copy(), absorb)
self.reset(self.initial_state)
else:
absorb = False
reward = 0.
sample = Sample(self._state.copy(), action, reward, next_state.copy(), absorb)
self._state = next_state
return sample
def initialize_warm_up_batch(args, device):
# using evenly distributed weights for warm-up stage
weights_batch = []
generate_weights_batch_dfs(0, args.obj_num, args.min_weight,
args.max_weight, args.delta_weight, [],
weights_batch)
sample_batch = []
scalarization_batch = []
temp_env = gym.make(
args.env_name) # temp_env is only used for initialization
for weights in weights_batch:
actor_critic = Policy(temp_env.observation_space.shape,
temp_env.action_space,
base_kwargs={'layernorm': args.layernorm},
obj_num=args.obj_num)
actor_critic.to(device).double()
if args.algo == 'ppo':
agent = algo.PPO(actor_critic,
args.clip_param,
args.ppo_epoch,
args.num_mini_batch,
args.value_loss_coef,
args.entropy_coef,
lr=args.lr,
eps=1e-5,
max_grad_norm=args.max_grad_norm)
else:
# NOTE: other algorithms are not supported yet
raise NotImplementedError
envs = make_vec_envs(env_name=args.env_name, seed=args.seed, num_processes=args.num_processes, \
gamma=args.gamma, log_dir=None, device=device, allow_early_resets=False, \
obj_rms = args.obj_rms, ob_rms = args.ob_rms)
env_params = {}
env_params['ob_rms'] = deepcopy(
envs.ob_rms) if envs.ob_rms is not None else None
env_params['ret_rms'] = deepcopy(
envs.ret_rms) if envs.ret_rms is not None else None
env_params['obj_rms'] = deepcopy(
envs.obj_rms) if envs.obj_rms is not None else None
envs.close()
scalarization = WeightedSumScalarization(num_objs=args.obj_num,
weights=weights)
sample = Sample(env_params, actor_critic, agent, optgraph_id=-1)
objs = evaluation(args, sample)
sample.objs = objs
sample_batch.append(sample)
scalarization_batch.append(scalarization)
temp_env.close()
return sample_batch, scalarization_batch
class Num:
def __init__(self, nums, f=lambda x: x):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = 10**32
self.hi = -10**32
self.max_val = len(nums)
self._some = Sample(self.max_val)
self.w = 1
for n in nums:
self.numInc(f(n))
def nums(self, t, f=None):
if f == None:
f = lambda x: x
n = Num(len(t))
for x in t:
n.numInc(f(x))
return n
def numInc(self, x):
if x == "?":
return x
x = float(x)
self.mu = float(self.mu)
self.n += 1
self._some.sampleInc(x)
d = x - self.mu
self.mu += (d / self.n)
self.m2 += (d * (x - self.mu))
if x > self.hi:
self.hi = x
if x < self.lo:
self.lo = x
if self.n >= 2:
self.sd = (self.m2 / (self.n - 1 + math.pow(10, -32)))**0.5
return x
def numDec(self, x):
if x == "?" or self.n == 1:
return x
self.n -= 1
d = x - self.mu
self.mu -= (d / self.n)
self.m2 -= d * (x - self.mu)
if self.n >= 2:
self.sd = (self.m2 / (self.n - 1 + math.pow(10, -32)))**0.5
return x
def numNorm(self, x):
return 0.5 if x == "?" else (x - self.lo) / (self.hi - self.lo +
math.pow(10, -32))
def numXpect(self, i, j):
n = i.n + j.n + 0.0001
return (i.n / n * i.sd + j.n / n * j.sd)
def main():
s = Sample()
print(s.call(1))
print(s.call('1'))
print(s.call(1.0))
print('DONE')
def setUp(self):
data = { 'DONOR_ID': 'A1', 'DONOR_AGE':'0 - 5', 'DONOR_HEALTH_STATUS':'NA',
'DONOR_SEX':'Female',
'DONOR_AGE_UNIT':'year', 'DONOR_LIFE_STAGE':'unknown', 'DONOR_ETHNICITY':'NA',
'SAMPLE_ONTOLOGY_URI':'http://url', 'MOLECULE':'genomic DNA', 'DISEASE':'None',
'BIOMATERIAL_TYPE':'Primary Cell', 'CELL_TYPE':'Monocyte' , 'CELL_LINE':'None'
}
self.sample = Sample( metadata=data, biomaterial_type='primary cell' )
def values(self):
sample_values = []
for task in self.tasks:
pt = self.mean().point(task)
s = Sample(pt, self.prob)
v = s.evaluate(task)
sample_values += [v]
return sample_values
def _sample_metadata(self, sample):
'''
Set samples metadata block for json hub
'''
sample = Sample(metadata=sample,
biomaterial_type=sample['BIOMATERIAL_TYPE'])
sample_dict = sample.get_samples_data()
return sample_dict
def waitpgm(self, sanitize=True):
if not self.ptcrunning:
return
#print "* Wait for PTC to finish"
if sanitize:
self.sanitize() # Sanitize tips before waiting for this to be done
worklist.comment("Wait for PTC")
while self.pgmEndTime-clock.pipetting > 120:
# Run any idle programs
oldElapsed=clock.pipetting
for ip in self.idlePgms:
if self.pgmEndTime-clock.pipetting > 120:
#print "Executing idle program with %.0f seconds"%(self.pgmEndTime-clock.pipetting)
ip(self.pgmEndTime-clock.pipetting-120)
if oldElapsed==clock.pipetting:
# Nothing was done
break
worklist.flushQueue() # Just in case
clock.pipandthermotime+=(clock.pipetting-self.pgmStartTime)
clock.thermotime+=(self.pgmEndTime-clock.pipetting)
clock.pipetting=self.pgmStartTime
#print "Waiting for PTC with %.0f seconds expected to remain"%(self.pgmEndTime-clock.pipetting)
self.lihahome()
worklist.pyrun('PTC\\ptcwait.py')
worklist.pyrun("PTC\\ptclid.py OPEN")
# worklist.pyrun('PTC\\ptcrun.py %s CALC ON'%"COOLDOWN")
# worklist.pyrun('PTC\\ptcwait.py')
worklist.vector("PTC200lid",decklayout.PTCPOS,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.CLOSE)
worklist.vector("Hotel 1 Lid",decklayout.HOTELPOS,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.OPEN)
worklist.vector("PTC200WigglePos",decklayout.PTCPOS,worklist.SAFETOEND,False,worklist.DONOTMOVE,worklist.DONOTMOVE)
worklist.vector("PTC200Wiggle",decklayout.PTCPOS,worklist.SAFETOEND,False,worklist.DONOTMOVE,worklist.CLOSE,True)
worklist.vector("PTC200Wiggle",decklayout.PTCPOS,worklist.ENDTOSAFE,False,worklist.DONOTMOVE,worklist.OPEN,True)
worklist.vector("PTC200WigglePos",decklayout.PTCPOS,worklist.ENDTOSAFE,False,worklist.DONOTMOVE,worklist.DONOTMOVE)
worklist.vector("PTC200Wiggle2Pos",decklayout.PTCPOS,worklist.SAFETOEND,False,worklist.DONOTMOVE,worklist.DONOTMOVE)
worklist.vector("PTC200Wiggle2",decklayout.PTCPOS,worklist.SAFETOEND,False,worklist.DONOTMOVE,worklist.CLOSE,True)
worklist.vector("PTC200Wiggle2",decklayout.PTCPOS,worklist.ENDTOSAFE,False,worklist.DONOTMOVE,worklist.OPEN,True)
worklist.vector("PTC200Wiggle2Pos",decklayout.PTCPOS,worklist.ENDTOSAFE,False,worklist.DONOTMOVE,worklist.DONOTMOVE)
worklist.vector("PTC200WigglePos",decklayout.PTCPOS,worklist.SAFETOEND,False,worklist.DONOTMOVE,worklist.DONOTMOVE)
worklist.vector("PTC200Wiggle",decklayout.PTCPOS,worklist.SAFETOEND,False,worklist.DONOTMOVE,worklist.CLOSE,True)
worklist.vector("PTC200Wiggle",decklayout.PTCPOS,worklist.ENDTOSAFE,False,worklist.DONOTMOVE,worklist.OPEN,True)
worklist.vector("PTC200WigglePos",decklayout.PTCPOS,worklist.ENDTOSAFE,False,worklist.DONOTMOVE,worklist.DONOTMOVE)
self.ptcrunning=False
self.moveplate(decklayout.SAMPLEPLATE,"Home")
# Mark all samples on plate as unmixed (due to condensation)
Sample.notMixed(decklayout.SAMPLEPLATE.name)
# Verify plate is in place
worklist.vector(decklayout.SAMPLEPLATE.vectorName,decklayout.SAMPLEPLATE,worklist.SAFETOEND,False,worklist.DONOTMOVE,worklist.CLOSE)
worklist.vector(decklayout.SAMPLEPLATE.vectorName,decklayout.SAMPLEPLATE,worklist.ENDTOSAFE,False,worklist.OPEN,worklist.DONOTMOVE)
worklist.romahome()
#worklist.userprompt("Plate should be back on deck. Press return to continue")
# Wash tips again to remove any drips that may have formed while waiting for PTC
worklist.wash(15,1,5,True)
def load_from_dict(self, d):
"""Load this instance from a dictionary.
"""
self.alpha = d.get('alpha', 0.05)
groups = d['groups']
for g in groups:
samp = Sample()
samp.load_from_dict(g)
self.groups.append(samp)
def __init__(self, max_val=0):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = 10**32
self.hi = -10**32
self._some = Sample(max_val)
self.w = 1
class Num:
def __init__(self, l, f=lambda x: x):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = 10**32
self.hi = -1 * (10**32)
self.max_size = 0
self.some = Sample(self.max_size)
self.w = 1
for i in l:
self.numInc(f(i))
def numInc(self, x):
if x == '?':
return x
x = float(x)
global n
self.n += 1
self.some.sampleInc(x)
d = x - self.mu
self.mu = self.mu + d / self.n
self.m2 = self.m2 + d * (x - self.mu)
if x > self.hi:
self.hi = x
if x < self.lo:
self.lo = x
if (self.n >= 2):
self.sd = math.pow((self.m2 / (self.n - 1 + 10**(-32))), .5)
return x
def numDec(self, x):
if (x == "?"):
return x
x = float(x)
if (self.n == 1):
return x
self.n = -1
d = x - self.mu
self.mu = self.mu - d / self.n
self.m2 = self.m2 - d * (x - self.mu)
if (self.n >= 2):
self.sd = math.pow(self.m2 / (self.n - 1 + 10**(-32)), .5)
return x
#Normalization
def numNorm(self, x):
return 0.5 if x == "?" else (x - self.lo) / (self.hi - self.lo +
10**(-32))
def numXpect(self, i, j):
n = i.n + i.n + 0.0001
return i.n / n * i.sd + j.n / n * j.sd
def __init__(self, max_size):
self.count = 0
self.mean = 0
self.m2 = 0
self.standard_deviation = 0
self.lo = inf
self.hi = -inf
self.sample = Sample(max_size)
def shake(self,plate,dur=60,speed=None,accel=5,returnPlate=True,samps=None,force=False):
if self.ptcrunning and plate==decklayout.SAMPLEPLATE:
self.waitpgm()
# Move the plate to the shaker, run for the given time, and bring plate back
allsamps=Sample.getAllOnPlate(plate)
if samps is None:
samps=allsamps
if all([x.isMixed() for x in samps]) and not force:
logging.notice( "No need to shake "+plate.name+", but doing so anyway.")
minspeed=0
maxspeed=2000
for x in samps:
(a,b)=x.getmixspeeds()
minspeed=max([a,minspeed])
maxspeed=min([b,maxspeed])
if speed is None:
if minspeed<maxspeed:
speed=max((maxspeed+minspeed)/2,maxspeed-50) # Mix as fast as safely possible (but always above minspeed)
else:
speed=maxspeed
if speed<minspeed-2 or speed>maxspeed+2:
others=""
for x in allsamps:
(a,b)=x.getmixspeeds()
if b<minspeed or a>maxspeed:
if a is not None and a>0:
others+=" {%s: %.1ful,G=%.2f%%,min=%.0f,max=%.0f}"%(x.name,x.volume,x.glycerolfrac()*100,a,b)
else:
others+=" {%s: %.1ful,G=%.2f%%,max=%.0f}"%(x.name,x.volume,x.glycerolfrac()*100,b)
logging.mixwarning("Mixing %s at %.0f RPM; minspeed(%.0f) > maxspeed(%.0f), limits=[%s]"%(plate.name,speed,minspeed,maxspeed,others))
else:
logging.notice("Mixing %s at %.0f RPM ( min RPM=%.0f, max RPM=%.f)"%(plate.name, speed, minspeed, maxspeed))
oldloc=plate.curloc
self.moveplate(plate,"Shaker",returnHome=False)
global __shakerActive
__shakerActive=True
worklist.pyrun("BioShake\\bioexec.py setElmLockPos")
worklist.pyrun("BioShake\\bioexec.py setShakeTargetSpeed%d"%speed)
worklist.pyrun("BioShake\\bioexec.py setShakeAcceleration%d"%accel)
worklist.pyrun("BioShake\\bioexec.py shakeOn")
self.starttimer()
Sample.shaken(plate.name,speed)
Sample.addallhistory("(S%d@%.0f)"%(dur,speed),onlyplate=plate.name)
self.waittimer(dur)
worklist.pyrun("BioShake\\bioexec.py shakeOff")
self.starttimer()
self.waittimer(accel+4)
worklist.pyrun("BioShake\\bioexec.py setElmUnlockPos")
__shakerActive=False
if returnPlate:
self.moveplate(plate,oldloc)
class Num:
def __init__(self, nums, f=lambda x: x):
self.max = len(nums)
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = 10**32
self.hi = -10**32
self.some = Sample(self.max)
self.w = 1
for n in nums:
self.num_inc(f(n))
def num_inc(self, x):
if x == "?":
return x
x = float(x)
self.mu = float(self.mu)
self.n = self.n + 1
self.some.sample_inc(x)
d = x - self.mu
self.mu = self.mu + d / self.n
self.m2 = self.m2 + d * (x - self.mu)
self.hi = max(self.hi, x)
self.lo = min(self.lo, x)
if (self.n >= 2):
self.sd = (self.m2 / (self.n - 1 + 10**-32))**0.5
return x
def num_dec(self, x):
if x == "?":
return x
if self.n == 1:
return x
x = float(x)
self.mu = float(self.mu)
self.n = self.n - 1
d = x - self.mu
self.mu = self.mu - d / self.n
self.m2 = self.m2 - d * (x - self.mu)
if self.n >= 2:
self.sd = (self.m2 / (self.n - 1 + 10**-32))**0.5
return x
def num_norm(self, x):
return x == "?" and 0.5 or (x - self.lo) / (self.hi - self.lo +
10**-32)
def num_xpect(self, i, j):
n = i.n + j.n + 0.0001
res = i.n / n * i.sd + j.n / n * j.sd
if isinstance(res, complex):
return res.real
else:
return res
def main():
song = Song(name='demo')
# It needs at least one sample. I'll just make a simple one
sample = Sample(name='sine')
# I think the samples are supposed to have a power of 2 length
sample.wave = [100 * sin(2 * pi * n / 32) for n in range(32)]
song.samples.append(sample)
# I didn't really think all the way through ex nihilo generation,
# so this is a little clunky. Make some note objects...
note_c = Note()
note_c.sample = 1 # Sample numbers start at 1, 0 is empty
note_c.pitch = 'C-2'
note_d = Note()
note_d.sample = 1
note_d.pitch = 'D-2'
note_e = Note()
note_e.sample = 1
note_e.pitch = 'E-2'
# I'm not sure if there will be side-effects of using a note
# object in multiple places. I should look into making them
# behave more like primitves. It will be ok for now.
# Make a new pattern to go in our song
pattern0 = Pattern()
# Channels are made automatically and can't be moved or replaced,
# but we'll grab one so we don't have to type pattern0[0] everywhere
channel = pattern0[0]
# Write our song
channel[0] = note_e
channel[4] = note_d
channel[8] = note_c
channel[16] = note_e
channel[20] = note_d
channel[24] = note_c
channel[32:40:2] = note_c
channel[40:48:2] = note_d
channel[48] = note_e
channel[52] = note_d
channel[56] = note_c
# Adding a pattern takes two steps
song.patterns.append(pattern0) # Store the pattern
song.positions = [0] # Give it a position
# Write out the file
song.write_file('~/Desktop/hot_cross_sines.mod')
def __init__(self, initList=[], f=same):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = Config().lo
self.hi = Config().hi
self.some = Sample()
self.w = 1
[self.numInc(x) for x in initList]
def __init__(self,inits=[],f=same):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = 10**32
self.hi = -10**32
self.some = Sample()
self.w = 1
[self.numInc(x) for x in inits]
def initWellKnownSamples():
global WATER, SSDDIL, BLEACH
WATER = Sample("Water", WATERLOC, -1, None, 50000)
SSDDIL = Sample("SSDDil", SSDDILLOC, -1, None, 50000)
BLEACH = Sample("RNase-Away",
BLEACHLOC,
-1,
None,
50000,
mixLC=LCBleachMix)
def __init__(self, no=[]):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = 10**32
self.hi = -10**32
self.some = Sample()
self.w = 1
for x in no:
self.numInc(x)
def __init__(self, l, f=lambda x: x):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = Vars().lo
self.hi = Vars().hi
self.some = Sample()
self.w = 1
for i in l:
self.numInc(f(i))
class Num:
def same(x):
return x
def __init__(self, initList=[], f=same):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = Config().lo
self.hi = Config().hi
self.some = Sample()
self.w = 1
[self.numInc(x) for x in initList]
def numInc(self, x):
if x == '?':
return
self.n += 1
self.some.sampleInc(x)
d = x - self.mu
self.mu += (d / self.n)
self.m2 += (d * (x - self.mu))
if (x > self.hi):
self.hi = x
if (x < self.lo):
self.lo = x
if (self.n >= 2):
self.sd = (self.m2 / (self.n - 1 + 10**-32))**0.5
return x
def numDec(self, x):
if x == '?':
return
if (self.n == 1):
return
self.n -= 1
d = x - self.mu
self.mu -= (d / self.n)
self.m2 -= (d * (x - self.mu))
self.m2 = abs(self.m2)
if (self.n >= 2):
self.sd = (self.m2 / (self.n - 1 + 10**-32))**0.5
return x
def numNorm(self, x):
return (x - self.lo) / (self.hi - self.lo + 10**-32)
def numXpect(i, j):
n = i.n + j.n + .0001
return i.n / n * i.sd + j.n / n * j.sd
def __init__(self, nums, f=lambda x: x):
self.max = len(nums)
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = 10**32
self.hi = -10**32
self.some = Sample(self.max)
self.w = 1
for n in nums:
self.num_inc(f(n))
def __init__(self, l, f=lambda x: x):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = 10**32
self.hi = -1 * (10**32)
self.max_size = 0
self.some = Sample(self.max_size)
self.w = 1
for i in l:
self.numInc(f(i))
def solve(self):
dim = self.prob.dim
num_samples = 5000
fp = open('sample_data_weier_v2.csv', 'w+')
for i in range(num_samples):
pt = (np.random.rand(dim) - 0.5) * 2.0
s = Sample(pt, self.prob)
values = [s.evaluate(t) for t in self.tasks]
print('sample %d at %s' % (i, pt))
row = ", ".join([str(x) for x in [i] + list(pt) + list(values)])
fp.write(row + "\n")
fp.close()
def getSample(wellx,welly,rack,grid,pos):
plate=Plate.lookup(grid,pos)
if plate==None:
plate=Plate(rack,grid,pos)
wellname="%c%d"%(ord('A')+welly-1,wellx)
well=plate.wellnumber(wellname)
sample=Sample.lookupByWell(plate,well)
if sample==None:
sample=Sample("%s.%d.%d.%d"%(rack,grid,pos,well),plate,well)
if grid==3:
sample.volume=20000 # Troughs
else:
sample.volume=0
return sample
def moveplate(self,plate,dest="Home",returnHome=True):
if self.ptcrunning and plate==decklayout.SAMPLEPLATE:
self.waitpgm()
# move to given destination (one of "Home","Magnet","Shaker","PTC" )
if plate!=decklayout.SAMPLEPLATE and plate!=decklayout.DILPLATE:
print "Only able to move %s or %s plates, not %s"%(decklayout.SAMPLEPLATE.name,decklayout.DILPLATE.name,plate.name)
assert False
if plate.curloc==dest:
#print "Plate %s is already at %s"%(plate.name,dest)
return
#print "Move plate %s from %s to %s"%(plate.name,plate.curloc,dest)
worklist.flushQueue()
self.lihahome()
cmt="moveplate %s %s"%(plate.name,dest)
worklist.comment(cmt)
if plate.curloc=="Home":
worklist.vector(plate.vectorName,plate,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.CLOSE)
elif plate.curloc=="Magnet":
worklist.vector("Magplate",decklayout.MAGPLATELOC,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.CLOSE)
elif plate.curloc=="Shaker":
worklist.vector("Shaker",decklayout.SHAKERPLATELOC,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.CLOSE)
elif plate.curloc=="PTC":
worklist.vector("PTC200",decklayout.PTCPOS,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.CLOSE)
else:
print "Plate %s is in unknown location: %s"%(plate.name,plate.curloc)
assert False
if dest=="Home":
plate.movetoloc(dest)
worklist.vector(plate.vectorName,plate,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.OPEN)
elif dest=="Magnet":
plate.movetoloc(dest,decklayout.MAGPLATELOC)
worklist.vector("Magplate",decklayout.MAGPLATELOC,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.OPEN)
elif dest=="Shaker":
plate.movetoloc(dest,decklayout.SHAKERPLATELOC)
worklist.vector("Shaker",decklayout.SHAKERPLATELOC,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.OPEN)
elif dest=="PTC":
plate.movetoloc(dest,decklayout.PTCPOS)
worklist.vector("PTC200",decklayout.PTCPOS,worklist.SAFETOEND,True,worklist.DONOTMOVE,worklist.OPEN)
else:
print "Attempt to move plate %s to unknown location: %s"%(plate.name,dest)
assert False
Sample.addallhistory("{->%s}"%dest,onlyplate=plate.name)
if returnHome:
worklist.romahome()
def addSample(self, newSampleName,newSampleRefType=None):
#
# Imports
#
import sys
import time
#
# open connection to database
#
self.getConnection()
sampleNames = []
sampleIds = []
#
# check if any of the fastqs already in database
#
data = self.c.execute('SELECT sampleId,sampleName,refType FROM samples').fetchall()
if data:
for (sampleId,sampleName,sampleRefType) in data:
#sampleName = sampleName[0]
sampleNames.append(sampleName)
sampleIds.append(sampleId)
if newSampleName in sampleNames:
msg = '#ERROR_MSG#'+time.strftime("%Y-%m-%d:%H:%M:%S",time.localtime())+'#'+str(self.analysispipe.masterPid)+'# SampleName must be uniq, there is already a sample with name '+newSampleName+' , exiting.\n'
self.analysispipe.logfile.write(msg)
sys.stderr.write(msg)
sys.exit(1)
if sampleIds: sampleId = max(sampleIds)+1
else: sampleId = 0
self.analysispipe.logfile.write('#LOGMSG#'+time.strftime("%Y-%m-%d:%H:%M:%S",time.localtime())+'#'+str(self.analysispipe.masterPid)+'# Adding sample '+newSampleName+' to database with id '+str(sampleId)+'.\n')
values = (sampleId,newSampleName,newSampleRefType)
self.c.execute('INSERT INTO samples VALUES (?,?,?)', values)
from sample import Sample
sample = Sample(sampleName=newSampleName, sampleId=sampleId,refType=newSampleRefType, analysispipe=self.analysispipe)
sample.createDirs()
self.commitAndClose()
return 0
class IRTThread(threading.Thread):
def __init__(self, parent, width, height, mainWindow):
threading.Thread.__init__(self)
self.mainWindow = mainWindow
self.glWidget = parent
self.sample = Sample(width, height)
self.sample.setRaytracer(IRT.Raytracer_Uniform)
self.screens = [numpy.zeros((3, width, height), dtype = numpy.float32),numpy.zeros((3, width, height), dtype = numpy.float32)]
self.lock = threading.Lock()
self.currentScreen = 0
self.pastFPS = []
self.need_resize = False
def resize(self, width, height):
self.screens = [numpy.zeros((3, width, height), dtype = numpy.float32),numpy.zeros((3, width, height), dtype = numpy.float32)]
self.sample.setResolution(width, height)
def get_screen(self):
return self.screens[self.currentScreen - 1]
screen = property(get_screen)
def paint(self):
print "test"
t = time.clock()
self.sample(self.screens[self.currentScreen])
t = time.clock() - t
print "test"
try:
self.lock.acquire()
self.currentScreen = 1 if self.currentScreen == 0 else 0
finally:
self.lock.release()
self.pastFPS.append(t)
if len(self.pastFPS) > 10:
self.pastFPS = self.pastFPS[-10:]
wx.CallAfter(self.mainWindow.updateStatusFPS, 10./sum(self.pastFPS))
wx.CallAfter(self.glWidget.Refresh)
def run(self):
while True:
self.paint()
def evaluate(self, _x):
self.eval_counter += 1
# Scale up the second part, to cover all the possible range
if self.mean_type == 'linear':
x = np.zeros(self.prob.dim * 2)
for i in range(self.prob.dim):
x[i] = _x[i]
for i in range(self.prob.dim, 2 * self.prob.dim):
x[i] = _x[i] - _x[i - self.prob.dim]
else:
x = np.array(_x)
self.mean().set_params(x)
sample_values = []
for task in self.tasks:
pt = self.mean().point(task)
s = Sample(pt, self.prob)
v = s.evaluate(task)
sample_values += [v]
avg_error = np.mean(sample_values)
if np.isnan(avg_error):
avg_error = 9999.9
print x, ' value: {', avg_error, '}'
self.iter_values += [sample_values] # Values for the entire iterations
self.iter_params += [x]
# If one iteration is ended
if self.eval_counter % 16 == 0:
self.iter_counter += 1
# print 'CMA Iteration', self.iter_counter, self.prob.eval_counter
# for v in self.iter_values:
# print sum(v), v
# print 'best:', self.values()
sum_values = [sum(vs) for vs in self.iter_values]
print('sum_values = %s' % sum_values)
best_index = np.nanargmin(sum_values)
print('best_index = %d' % best_index)
best_params = self.iter_params[best_index]
print('best params = %s' % best_params)
self.mean().set_params(best_params)
[o.notify_step(self, self.model) for o in self.observers]
self.iter_values = []
self.iter_params = []
return avg_error
def addDiscoverySamples(self, srList, startValList, minValList,
maxValList, colorList):
"""
Add a sample to be used for discovery fits
"""
self.hasDiscovery = True
self.parentTopLvl.hasDiscovery = True
if not self.variableName == "cuts":
raise TypeError("Discovery sample can only be added "
"to a cuts channel")
for (iSR, sr) in enumerate(srList):
sigSample = Sample("DiscoveryMode_%s" % sr, colorList[iSR])
sigSample.setNormFactor("mu_%s" % sr, startValList[iSR],
minValList[iSR], maxValList[iSR])
sigSample.setDiscovery()
sigSample.clearSystematics()
self.addSample(sigSample)
self.parentTopLvl.setSignalSample(sigSample)
histoName = "h%sNom_%s_obs_%s" % (sigSample.name, sr, self.variableName.replace("/", ""))
self.getSample("DiscoveryMode_%s" % sr).setHistoName(histoName)
configMgr.hists[histoName] = TH1F(histoName, histoName,
len(srList), 0.0,
float(len(srList)))
configMgr.hists[histoName].SetBinContent(iSR+1, startValList[iSR])
return
def solve(self):
[o.notify_init(self, self.model) for o in self.observers]
sample_values = []
for task in self.tasks:
pt = self.mean().point(task)
s = Sample(pt, self.prob)
v = s.evaluate(task)
sample_values += [v]
self.iter_values += [sample_values]
self.iter_params += [self.mean().params()]
[o.notify_step(self, self.model) for o in self.observers]
res = {'result': 'NG'}
# opt = {'verb_time': 0, 'popsize': 16, 'tolfun': 1.0}
cma.CMAOptions('tol')
opts = cma.CMAOptions()
opts.set('verb_disp', 1)
# opts.set('tolfun', 0.001)
# opts.set('tolx', 0.0000001)
# opts.set('tolx', 1.0)
opts.set('ftarget', 0.001)
num_offsprings = 16
opts.set('popsize', num_offsprings)
max_iter = int(20000 / self.n / num_offsprings)
print('maxiter: %d' % max_iter)
opts.set('maxiter', max_iter)
for key, value in opts.iteritems():
print '[', key, ']\n', value
x0 = np.random.rand(self.mean().paramdim) - 0.5
# print cma.CMAOptions()
# exit(0)
print()
print('------- CMA-ES --------')
res = cma.fmin(self.evaluate, x0, 1.0, opts)
print('-----------------------')
print()
# np.set_printoptions(precision=6, suppress=True)
print('the answer: %s' % res[0])
[o.notify_solve(self, self.model) for o in self.observers]
return res
def __init__(self):
'Create a new experiment with given sample locations for water and WASTE; totalTime is expected run time in seconds, if known'
worklist.comment("Generated %s"%(datetime.now().ctime()))
worklist.userprompt("The following reagent tubes should be present: %s"%Sample.getAllLocOnPlate(decklayout.REAGENTPLATE))
worklist.userprompt("The following eppendorf tubes should be present: %s"%Sample.getAllLocOnPlate(decklayout.EPPENDORFS))
worklist.email(dest='*****@*****.**',subject='Run started (Generate: %s)'%(datetime.now().ctime()))
worklist.email(dest='*****@*****.**',subject='Tecan error',onerror=1)
self.cleanTips=0
# self.sanitize() # Not needed, TRP does it, also first use of tips will do this
self.useDiTis=False
self.ptcrunning=False
self.overrideSanitize=False
self.pgmStartTime=None
self.pgmEndTime=None
# Access PTC and RIC early to be sure they are working
worklist.pyrun("PTC\\ptctest.py")
# worklist.periodicWash(15,4)
worklist.userprompt("Verify that PTC thermocycler lid pressure is set to '2'.")
self.idlePgms=[]
self.timerStartTime=[None]*8
def readAsVector(cls, filename):
reader = cls(filename)
header = reader.readHeader()
statusPacket = reader.readStatusPacket()
seq = []
while reader.isFinished() is False:
rawSample = reader.readSample(header)
sample = Sample.fromRaw(rawSample, statusPacket)
yield (header, statusPacket, sample)
reader.close()
def savesummary(self,filename):
# Print amount of samples needed
fd=open(filename,"w")
# print >>fd,"Deck layout:"
# print >>fd,decklayout.REAGENTPLATE
# print >>fd,decklayout.SAMPLEPLATE
# print >>fd,decklayout.QPCRPLATE
# print >>fd,decklayout.WATERLOC
# print >>fd,decklayout.WASTE
# print >>fd,decklayout.BLEACHLOC
# print >>fd,decklayout.WASHLOC
# print >>fd
#print >>fd,"DiTi usage:",worklist.getDITIcnt()
#print >>fd
rtime="Run time: %d (pipetting only) + %d (thermocycling only) + %d (both) = %d minutes\n"%(clock.pipetting/60.0,clock.thermotime/60, clock.pipandthermotime/60, clock.elapsed()/60)
print rtime
print >>fd,rtime
reagents.printprep(fd)
Sample.printallsamples("All Samples:",fd,w=worklist)
liquidclass.LC.printalllc("All LC:",fd)
fd.close()
def savesummary(self,filename):
# Print amount of samples needed
fd=open(filename,"w")
# print >>fd,"Deck layout:"
# print >>fd,decklayout.REAGENTPLATE
# print >>fd,decklayout.SAMPLEPLATE
# print >>fd,decklayout.QPCRPLATE
# print >>fd,decklayout.WATERLOC
# print >>fd,decklayout.WASTE
# print >>fd,decklayout.BLEACHLOC
# print >>fd,decklayout.WASHLOC
# print >>fd
#print >>fd,"DiTi usage:",worklist.getDITIcnt()
#print >>fd
print >>fd,"Generated %s (%s-%s pyTecan-%s)"%(datetime.now().ctime(),sys.argv[0],self.checksum,self.gitlabel)
rtime="Run time: %d (pipetting only) + %d (thermocycling only) + %d (both) = %d minutes (%.1f hours)\n"%(clock.pipetting/60.0,clock.thermotime/60, clock.pipandthermotime/60, clock.elapsed()/60, clock.elapsed()/3600.0)
print rtime
print >>fd,rtime
reagents.printprep(fd)
Sample.printallsamples("All Samples:",fd,w=worklist)
liquidclass.LC.printalllc(fd)
fd.close()
def __init__(self, parent, width, height, mainWindow):
threading.Thread.__init__(self)
self.mainWindow = mainWindow
self.glWidget = parent
self.sample = Sample(width, height)
self.sample.setRaytracer(IRT.Raytracer_Uniform)
self.screens = [numpy.zeros((3, width, height), dtype = numpy.float32),numpy.zeros((3, width, height), dtype = numpy.float32)]
self.lock = threading.Lock()
self.currentScreen = 0
self.pastFPS = []
self.need_resize = False
def get_corpus(dirs):
persons = defaultdict(Person)
for d in dirs:
print ("processing {} ...".format(d))
for fname in sorted(glob.glob(os.path.join(d, "*.wav"))):
basename = os.path.basename(fname)
gender, name, _ = basename.split("_")
p = persons[name]
p.name, p.gender = name, gender
try:
orig_sample = Sample.from_wavfile(fname)
p.add_sample(orig_sample)
except Exception as e:
print ("Exception occured while reading {}: {} ".format(fname, e))
print ("======= traceback =======")
print (traceback.format_exc())
print ("=========================")
return persons
def __init__(self, parent, width, height, mainWindow):
super(IRTThread, self).__init__(parent)
self.mainWindow = mainWindow
self.glWidget = parent
self.sample = Sample(width, height)
self.sample.setRaytracer(IRT.Raytracer_Halton_2_3)
self.screens = [numpy.zeros((3, width, height), dtype = numpy.float32),numpy.zeros((3, width, height), dtype = numpy.float32)]
self.lock = QReadWriteLock()
self.currentScreen = 0
self.pastFPS = []
self.commands = []
self.origin = self.sample.origin
self.direction = self.sample.direction
self.orientation = self.sample.orientation
self.sample.raytracer.setViewer(self.origin, self.direction)
self.sample.raytracer.setOrientation(self.orientation)
def get_corpus():
persons = defaultdict(Person)
dirs = [
'../test-data/corpus/Style_Reading',
'../test-data/corpus/Style_Spontaneous',
'../test-data/corpus/Style_Whisper',
]
for d in dirs:
print("processing {} ..." . format(d))
for fname in glob.glob(os.path.join(d, "*.wav")):
basename = os.path.basename(fname)
gender, name, _ = basename.split('_')
p = persons[name]
p.name, p.gender = name, gender
try:
p.add_sample(Sample.from_wavfile(fname))
except Exception as e:
print("Exception occured while reading {}: {} " . format(
fname, e))
print("======= traceback =======")
print(traceback.format_exc())
print("=========================")
return persons
def test_amazon_resume(self):
print datetime.now()
print "Instantiating Sample"
sample = Sample("/Users/cptullio/Predicao-de-Links/PredLig/src/data/amazon_resume.txt", 20, (0.5, 0.5))
print datetime.now()
print "Configuring Attributes or Features"
sample.set_attributes_list({"preferential_attachment":{}, "common_neighbors":{}, "sum_of_neighbors":{}})
print datetime.now()
print "Rescue the Sample"
sample.get_sample()
print datetime.now()
print "Classifying the data"
table = sample.set_classification_dataset()
print datetime.now()
print "Making Prediction Link"
predictor = LinkPrediction(dataset = table, folds_number = 2)
print datetime.now()
print "Applying Classifier"
print predictor.apply_classifier()
pass
class IRTThread(QThread):
def __init__(self, parent, width, height, mainWindow):
super(IRTThread, self).__init__(parent)
self.mainWindow = mainWindow
self.glWidget = parent
self.sample = Sample(width, height)
self.sample.setRaytracer(IRT.Raytracer_Halton_2_3)
self.screens = [numpy.zeros((3, width, height), dtype = numpy.float32),numpy.zeros((3, width, height), dtype = numpy.float32)]
self.lock = QReadWriteLock()
self.currentScreen = 0
self.pastFPS = []
self.commands = []
self.origin = self.sample.origin
self.direction = self.sample.direction
self.orientation = self.sample.orientation
self.sample.raytracer.setViewer(self.origin, self.direction)
self.sample.raytracer.setOrientation(self.orientation)
def resize(self, width, height):
self.screens = [numpy.zeros((3, width, height), dtype = numpy.float32),numpy.zeros((3, width, height), dtype = numpy.float32)]
self.sample.setResolution(width, height)
def get_screen(self):
return self.screens[self.currentScreen - 1]
screen = property(get_screen)
def paint(self):
t = time.time()
self.sample(self.screens[self.currentScreen])
t = time.time() - t
try:
self.lock.lockForWrite()
self.currentScreen = 1 if self.currentScreen == 0 else 0
finally:
self.lock.unlock()
self.pastFPS.append(t)
if len(self.pastFPS) > 10:
self.pastFPS = self.pastFPS[-10:]
self.mainWindow.emit(SIGNAL("updateFPS(double)"), 10./sum(self.pastFPS))
self.glWidget.emit(SIGNAL("updateView()"))
def run(self):
while True:
self.paint()
while self.commands:
command = self.commands.pop()
command()
def rotateUp(self):
"""
Rotates the raytracer in the up direction
"""
self.orientation = rotate(self.orientation, numpy.cross(self.orientation, self.direction), -math.pi/180)
self.direction = rotate(self.direction, numpy.cross(self.orientation, self.direction), -math.pi/180)
self.sample.raytracer.setViewer(self.origin, self.direction)
self.sample.raytracer.setOrientation(self.orientation)
def rotateDown(self):
"""
Rotates the raytracer in the down direction
"""
self.orientation = rotate(self.orientation, numpy.cross(self.orientation, self.direction), math.pi/180)
self.direction = rotate(self.direction, numpy.cross(self.orientation, self.direction), math.pi/180)
self.sample.raytracer.setViewer(self.origin, self.direction)
self.sample.raytracer.setOrientation(self.orientation)
def rotateLeft(self):
"""
Rotates the raytracer in the left direction
"""
self.direction = rotate(self.direction, self.orientation, -math.pi/180)
self.sample.raytracer.setViewer(self.origin, self.direction)
def rotateRight(self):
"""
Rotates the raytracer in the right direction
"""
self.direction = rotate(self.direction, self.orientation, math.pi/180)
self.sample.raytracer.setViewer(self.origin, self.direction)
def goIn(self):
"""
Goes inside the screen
"""
self.origin += self.direction * .1
self.sample.raytracer.setViewer(self.origin, self.direction)
def goBack(self):
"""
Goes back outside the screen
"""
self.origin -= self.direction * .1
self.sample.raytracer.setViewer(self.origin, self.direction)
def inclinateLeft(self):
"""
Rotates the raytracer in the left direction
"""
self.orientation = rotate(self.orientation, self.direction, -math.pi/180)
self.sample.raytracer.setOrientation(self.orientation)
def inclinateRight(self):
"""
Rotates the raytracer in the right direction
"""
self.orientation = rotate(self.orientation, self.direction, math.pi/180)
self.sample.raytracer.setOrientation(self.orientation)
def setOversampling(self, oversampling):
self.sample.raytracer.setOversampling(oversampling)
def setLevels(self, levels):
self.sample.raytracer.setLevels(levels)
def loadData(matrixFile, outDir):
#open the file and parse the contents
logging.info('Parsing datamatrix')
file = open(matrixFile, 'r')
#store a few things for later
headers = []
sampleSummary = Sample('all')
samples = {}
#loop over the file
for line in file.readlines():
#remove the return
line = line.rstrip('\n')
#tokeninze the line
tokens = line.split('\t')
#if it is the first line we can just store the headers
if line.startswith('GeneName'):
headers = tokens
#else we need to process the data
else:
sampleName = tokens[0]
#print sampleName
#create my sample
sample = Sample(sampleName)
#now lets start processing everything
for num, variantInfo in enumerate(tokens[1:]):
#grab the header info (we shouldn't be doing this for each sample, need to fix)
gene, refpos = headers[num + 1].split('_')
reference, position = re.findall(r"[^\W\d_]+|\d+", refpos)
#print '%i %s %s %s' % (num, gene, reference, position)
#now lets parse the variant info
#if LOW_DEPTH, then skip
if not variantInfo.startswith('LOW_DEPTH'):
variantInfoTokens = re.findall(r"[-+]?\d*\.\d+|\d+|\w+", variantInfo) #variantInfo.split('/')
#simple case is where there is just one alternate allele
if len(variantInfoTokens) == 3:
#base case
#reference
sample.addVariant(gene, reference, reference, position, 1.0 - float(variantInfoTokens[2]))
sampleSummary.addVariant(gene, reference, reference, position, 1.0 - float(variantInfoTokens[2]))
#variant
sample.addVariant(gene, reference, variantInfoTokens[1], position, float(variantInfoTokens[2]))
sampleSummary.addVariant(gene, reference, variantInfoTokens[1], position, float(variantInfoTokens[2]))
elif len(variantInfoTokens) == 6:
#need to handle when there are two alternate alleles
#reference
sample.addVariant(gene, reference, reference, position, 1.0 - float(variantInfoTokens[4]) - float(variantInfoTokens[5]))
sampleSummary.addVariant(gene, reference, reference, position, 1.0 - float(variantInfoTokens[4]) - float(variantInfoTokens[5]))
#variant 1
sample.addVariant(gene, reference, variantInfoTokens[1], position, float(variantInfoTokens[4]))
sampleSummary.addVariant(gene, reference, variantInfoTokens[1], position, float(variantInfoTokens[4]))
#variant 2
sample.addVariant(gene, reference, variantInfoTokens[2], position, float(variantInfoTokens[5]))
sampleSummary.addVariant(gene, reference, variantInfoTokens[2], position, float(variantInfoTokens[5]))
elif len(variantInfoTokens) == 2:
#this means there was a deletion
#reference
sample.addVariant(gene, reference, reference, position, 1.0 - float(variantInfoTokens[1]))
sampleSummary.addVariant(gene, reference, reference, position, 1.0 - float(variantInfoTokens[1]))
#deletion
sample.addVariant(gene, reference, '-', position, float(variantInfoTokens[1]))
sampleSummary.addVariant(gene, reference, '-', position, float(variantInfoTokens[1]))
else:
#this really shoudln't happen
print variantInfoTokens
logging.warning('Sample with more than two alternate alleles - %s %s' % (sampleName, variantInfo))
#now add the sample
samples[sampleName] = sample
#close the file
file.close()
#now lets build our feature vectors, for now it will be all the variants and all the genes
logging.info('Building feature vectors')
#dump the array to a spreadsheet for debugging
spreadsheet = xlsxwriter.Workbook('%s/features.xlsx' % outDir)
featureWorksheet = spreadsheet.add_worksheet('Features')
#set up our huge feature array and class label vectors. features is
features = np.zeros((len(samples), len(sampleSummary.getVariants()) + len(sampleSummary.getGenes())))
labels = np.zeros((len(samples)))
#now let's populate our matrix
for sampleNumber, sample in enumerate(sorted(samples)):
sample = samples[sample]
labels[sampleNumber] = sample.getClassLabel()
featureWorksheet.write(sampleNumber+1, 0, sample.getSampleName())
featureWorksheet.write(sampleNumber+1, 1, sample.getClassLabel())
#first start with the variants
for featureNumber, variant in enumerate(sorted(sampleSummary.getVariants())):
geneName, position, referenceBase, variantBase = variant.split('_')
featureWorksheet.write(0, featureNumber+2, variant)
if(sample.hasVariant(geneName, referenceBase, variantBase, position)):
#add frequency to array
features[sampleNumber][featureNumber] = sample.getVariantFrequency(geneName, referenceBase, variantBase, position)
featureWorksheet.write(sampleNumber+1, featureNumber+2, sample.getVariantFrequency(geneName, referenceBase, variantBase, position))
else:
featureWorksheet.write(sampleNumber+1, featureNumber+2, 0)
#now at the gene level
for featureNumber, gene in enumerate(sorted(sampleSummary.getGenes())):
featureWorksheet.write(0, featureNumber+len(sampleSummary.getVariants())+1, gene)
if(sample.hasGene(gene)):
features[sampleNumber][featureNumber + len(sampleSummary.getVariants())] = 1
featureWorksheet.write(sampleNumber+1, featureNumber+len(sampleSummary.getVariants())+2, 1)
else:
featureWorksheet.write(sampleNumber+1, featureNumber+len(sampleSummary.getVariants())+2, 0)
#close the spreadsheet
spreadsheet.close()
#return the matrix
print features
print labels
return(features, labels)
def evaluate(self, x):
self.eval_counter += 1
s = Sample(x, self.prob)
v = s.evaluate(self.current_task)
print('%.6f <--- %s' % (v, x))
return v
db = client['data']
chemicals = db['chemical']
# Initialize available transforms. Ignore ones having more than one retron.
transforms = []
for rec in db['retro'].find():
try:
t = Transform(rec['reaction_smarts'].encode('ascii'),
dbid=rec['_id'])
except ValueError:
continue
if len(t.retrons) == 1:
transforms.append(t)
# Get a random sample of chemical compounds.
sample = Sample(chemicals, size=args.size, rng_seed=args.seed)
# Initialize data structures for stats gathering.
keys = ['accepted', 'valid', 'total']
new_stats = dict(zip(keys, len(keys) * [0]))
old_stats = dict(zip(keys, len(keys) * [0]))
# For each chemical in the sample perform a single retrosynthetic step.
reactions = {}
disconnected_count = 0
for chem_rec in sample.get():
try:
smi = chem_rec['smiles'].encode('ascii')
except (AttributeError, KeyError):
sys.stderr.write('Chemical {0} has no SMILES'.format(chem_rec['_id']))
continue
def shake(self,plate,dur=60,speed=None,accel=5,returnPlate=True,samps=None):
if self.ptcrunning and plate==decklayout.SAMPLEPLATE:
self.waitpgm()
# Move the plate to the shaker, run for the given time, and bring plate back
allsamps=Sample.getAllOnPlate(plate)
if samps is None:
samps=allsamps
if all([x.isMixed() for x in samps]):
print "No need to shake ",plate
maxvol=max([x.volume for x in allsamps])
minvol=min([x.volume for x in samps if not x.isMixed() ]+[200])
(minspeed,maxspeed)=plate.getmixspeeds(minvol*0.95,maxvol+5) # Assume volumes could be off
if minspeed>maxspeed:
print "minspeed(",minspeed,") > maxspeed (",maxspeed,")"
if speed is None:
if minspeed<maxspeed:
speed=(maxspeed+minspeed)/2
else:
speed=maxspeed
warned=False
if speed>maxspeed:
print "WARNING: %s plate contains wells with up to %.2f ul, which may spill at %d RPM: "%(plate.name, maxvol, speed),
for x in samps:
tmp=plate.getmixspeeds(x.volume,x.volume)
if tmp[1]<speed:
print "%s[%.1ful, max=%.0f RPM] "%(x.name,x.volume,tmp[1]),
print
warned=True
if globals.verbose and speed<minspeed:
print "NOTICE: %s plate contains unmixed wells that may not be mixed at %d RPM: "%(plate.name, speed),
for x in samps:
if x.isMixed():
continue
tmp=plate.getmixspeeds(x.volume*0.95,x.volume+5)
if speed<tmp[0]:
print "%s[%.1ful, min=%.0f RPM, max=%.0f RPM] "%(x.name,x.volume,tmp[0],tmp[1]),
print
warned=True
if warned or globals.verbose:
print " Mixing %s at %.0f RPM (min unmixed vol=%.0ful -> min RPM=%.0f; max vol=%.0ful -> max RPM=%.f)"%(plate.name, speed, minvol, minspeed, maxvol, maxspeed)
oldloc=plate.curloc
self.moveplate(plate,"Shaker",returnHome=False)
global __shakerActive
__shakerActive=True
worklist.pyrun("BioShake\\bioexec.py setElmLockPos")
worklist.pyrun("BioShake\\bioexec.py setShakeTargetSpeed%d"%speed)
worklist.pyrun("BioShake\\bioexec.py setShakeAcceleration%d"%accel)
worklist.pyrun("BioShake\\bioexec.py shakeOn")
self.starttimer()
Sample.shaken(plate.name,speed)
Sample.addallhistory("(S%d@%.0f)"%(dur,speed),onlyplate=plate.name)
self.waittimer(dur)
worklist.pyrun("BioShake\\bioexec.py shakeOff")
self.starttimer()
self.waittimer(accel+4)
worklist.pyrun("BioShake\\bioexec.py setElmUnlockPos")
__shakerActive=False
if returnPlate:
self.moveplate(plate,oldloc)
"'{}'").format(sample_id))
family_count += 1
info_cell = ws["M" + str(rownum)]
info = info_cell.value
if info is None:
info = "NA"
info.rstrip()
tissue_cell = ws["C" + str(rownum)]
tissue = tissue_cell.value
if tissue is None:
tissue = "NA"
tissue.rstrip()
sample = Sample(sample_id, info, tissue)
in_family = True
family.append(sample)
if sample.info.find("singleton") != -1:
# Found a singleton!
sample.affected = True
update_family(family)
print >> out, family_ped(family, family_count).rstrip()
# This ends the current family.
if config.debug:
print(sys.stderr, "Found a singleton. Family complete.")
family = []
in_family = False
# Note that the next row may be a None or a new family member..
def pause(self,duration):
self.starttimer()
self.waittimer(duration)
Sample.addallhistory("(%ds)"%duration)
