def run_phasing_chain(phaser, problem, params=None):
'''The main call that runs the phasing, stats saving , and post-processing as one long pipeline.
Returns the populated request object.'''
request = util.Struct(problem=problem,
params=params if params else PhaseParam(),
g_orig=problem.genotype.data.copy(),
stats=util.Struct())
# Run phasing processing chain
start = time.time()
phaser.handle(request)
t = time.time() - start
request.stats.time = t
return request
def run(self, problem, params=None):
'''Run the phasing processing chain. Adapts the generic Filter interface to include
both a Problem and PhasingParam inputs. If params=None, using default PhaseParam values.'''
'''A template method that delegates to runner(), which accepts two input parameters.'''
return self.handle(
util.Struct(problem=problem,
params=params if params else PhaseParam()))
def override_imputed_by_plink(imputed_file,
plink_prefix,
process_line_factory=process_line_factory,
**kwargs):
'''Main program for overriding imputing genotype file by a PLINK data file. Accepts an argument dictionary.'''
# Default options
options = util.Struct(num_header_lines=0,
num_metadata_cols=8,
override_tag=im.constants.UNPHASED,
max_flipped_hom=10,
max_mismatch_partial=5,
delimiter='\t',
index_file=os.environ['OBER_DATA'] +
'/cgi/README.assembly_sample_subject.csv',
pedigree_file=im.itu.HUTT_PED,
genotype_id_file=im.examples.CHR22 + '/hutt.tfam',
warnings=False,
key='bp')
# Override with passed arguments
options.update(**kwargs)
return __override_imputed_by_plink(
imputed_file,
plink_prefix,
options,
process_line_factory=process_line_factory)
def loadTopology(ifname):
topo = util.Struct(nodes={}, edges=[])
for vid1, vid2 in util.itertokens(ifname, sep=" ", func=int):
topo.edges.append((vid1, vid2))
topo.nodes[vid1] = True
topo.nodes[vid2] = True
return topo
def main():
apollocaffe.set_device(0)
#apollocaffe.set_cpp_loglevel(0)
apollocaffe.set_random_seed(0)
np.random.seed(0)
job = sys.argv[1]
corpus_name = sys.argv[2]
config = util.Struct(**yaml.load(CONFIG))
if corpus_name == "abstract":
train_scenes, dev_scenes, test_scenes = corpus.load_abstract()
else:
assert corpus_name == "birds"
train_scenes, dev_scenes, test_scenes = corpus.load_birds()
apollo_net = ApolloNet()
print "loaded data"
print "%d training examples" % len(train_scenes)
listener0_model = Listener0Model(apollo_net, config.model)
speaker0_model = Speaker0Model(apollo_net, config.model)
sampling_speaker1_model = SamplingSpeaker1Model(apollo_net, config.model)
compiled_speaker1_model = CompiledSpeaker1Model(apollo_net, config.model)
if job == "train.base":
train(train_scenes, dev_scenes, listener0_model, apollo_net, config.opt)
train(train_scenes, dev_scenes, speaker0_model, apollo_net, config.opt)
apollo_net.save("models/%s.base.caffemodel" % corpus_name)
exit()
if job == "train.compiled":
apollo_net.load("models/%s.base.caffemodel" % corpus_name)
print "loaded model"
train(train_scenes, dev_scenes, compiled_speaker1_model, apollo_net,
config.opt)
apollo_net.save("models/%s.compiled.caffemodel" % corpus_name)
exit()
if job in ("sample.base", "sample.compiled"):
if job == "sample.base":
apollo_net.load("models/%s.base.caffemodel" % corpus_name)
else:
apollo_net.load("models/%s.compiled.caffemodel" % corpus_name)
print "loaded model"
if job == "sample.base":
models = {
"sampling_speaker1": sampling_speaker1_model,
}
elif job == "sample.compiled":
models = {
"compiled_speaker1": compiled_speaker1_model,
}
name = job.split(".")[1]
run_experiment("one_different", corpus_name, name, models, dev_scenes)
run_experiment("by_similarity", corpus_name, name, models, dev_scenes)
run_experiment("all_same", corpus_name, name, models, dev_scenes)
def opt(name, type=None, default=None):
o = util.Struct(name=name, type=type, default=default)
if type is None:
if default is not None:
o.type = type_builtin(default)
else:
o.type = str #raise Exception("need type for %s" % name)
#if o.type==bool: o.type=int
return o
def main(**kwargs):
'''Main program - accepts argument dictionary.'''
# Default options
options = util.Struct(data_file=None, variant_id=False)
# Override with passed arguments
options.update(**kwargs)
# (valid, options, error_msg) = __validate_options(options)
# if not valid:
# raise ValueError('Bad options: %s' % (error_msg,))
return __main(options)
def expect_get_posts(self):
post = data.BlogPost()
post.id = util.Struct(text='tag:blogger.com,1999:blog-111.post-222')
feed = data.BlogFeed()
feed.entry = [post]
def check_path(query):
return query.custom_parameters['path'] == '/path/to/post'
self.client.get_posts('111', query=mox.Func(check_path)).AndReturn(feed)
def main(**kwargs):
'''Main program - accepts argument dictionary.'''
# Default options
options = util.Struct(num_header_lines=0, num_metadata_cols=8, buf_size=10000,
pedigree_file=im.itu.HUTT_PED, genotype_file=im.examples.CHR22)
# Override with passed arguments
options.update(**kwargs)
# (valid, options, error_msg) = __validate_options(options)
# if not valid:
# raise ValueError('Bad options: %s' % (error_msg,))
return __main(options)
def setUp(self):
super(BloggerTest, self).setUp()
self.auth_entity = BloggerV2Auth(name='name',
blog_ids=['111'],
blog_hostnames=['my.blawg'],
picture_url='http://pic')
self.client = self.mox.CreateMock(BloggerClient)
self.comment = data.Comment()
self.comment.id = util.Struct(
text='tag:blogger.com,1999:blog-111.post-222.comment-333')
self.comment.to_string = lambda: '<foo></foo>'
def test_update_mastodon_pictures_get_actor_404(self):
self.expect_requests_get(
'https://foo.com' + test_mastodon.API_ACCOUNT % 123,
headers={'Authorization': 'Bearer towkin'},
).AndRaise(
requests.exceptions.HTTPError(
response=util.Struct(status_code='404', text='foo')))
self.mox.ReplayAll()
mastodon = self._setup_mastodon()
resp = self.client.get('/cron/update_mastodon_pictures')
self.assertEqual(200, resp.status_code)
self.assertEqual('http://before', mastodon.key.get().picture)
def prepare_figure_vs_snp(info, title=None, xaxis='snp', snp_index=None):
'''Prepare a template figure vs. SNP # or location in base pairs for a ProblemInfo object.
Return a plot data struct.'''
snp_index = snp_index if snp_index is None else info.snp_range
data = util.Struct()
if xaxis == 'snp':
data.x = info.snp_range[snp_index]
data.xlabel = 'SNP #'
else:
data.x = info.snp['base_pair'][snp_index] / im.constants.MEGA_BASE_PAIR
data.xlabel = 'SNP Position [Mbp]'
data.xmin, data.xmax = np.min(data.x), np.max(data.x)
if title: P.title(title)
P.xlabel(data.xlabel)
P.xlim((data.xmin, data.xmax))
return data
def main(**kwargs):
'''Main program - accepts argument dictionary.'''
# Default options
options = util.Struct(pedigree=None,
prefix=None,
tped=None,
tfam=None,
haplotype=None,
out=None,
debug=False,
target=None,
pedigree_genotyped=None)
# Override with passed arguments
options.update(**kwargs)
valid, options, error_msg = __validate_options(options)
if not valid: raise ValueError('Bad options: %s' % (error_msg, ))
return __main(options)
def main(**kwargs):
'''Main program - accepts argument dictionary.'''
# Default options
options = util.Struct(pedigree=None,
prefix=None,
tped=None,
tfam=None,
input=None,
output=None,
debug=False,
stage=0,
print_times=True,
impute=IMPUTE_OPTION.NONE,
min_output=False,
selected_samples=None)
# Override with passed arguments
options.update(**kwargs)
valid, options = __validate_options(options)
if not valid: raise ValueError('Bad options')
return __main(options)
def impute2_concordance_chrom_vs_window_size(output_dir_prefix, chrom, runs):
'''Return the data of all windows along an entire chromosome, for each of several window sizes specified
via the number of nodes. ''runs'' is an array of tuples (node#s, instances_per_nodes) of each run.'''
stats = [None] * len(runs)
for k, (nodes, instances_per_node) in enumerate(runs):
windows = nodes * instances_per_node
stats[k] = (nodes, windows, np.array([(s,) + impute2_stats(impute2_concordance('%s.windows_%d/chr%d/run_impute2/node-%04d/run_impute2-%04d.stats.haps' % \
(output_dir_prefix, windows, chrom, s / instances_per_node, s)))
for s in xrange(windows)],
dtype=[
('window', 'i4'),
('concordance', 'f4'),
('het_concordance', 'f4'),
('bp_center', 'i12')
]))
result = util.Struct()
result.chrom = chrom
result.instances_per_node = instances_per_node
result.stats = stats
return result
def main(in_file, info, segment_file, out_dir, **kwargs):
'''Main program - reads options from a method parameter dictionary.'''
# Default options
options = util.Struct(num_processes=1,
region_size=10,
regions=None,
snp_index=None,
profile=False,
min_degree=4,
min_len=2.0,
margin=0.8,
debug=0,
algorithm='naive',
threshold=0.995)
# Override with passed arguments
options.update(**kwargs)
args = in_file, info, segment_file, out_dir
args, options = __validate_options(args, options)
return __main(args, options)
def read_call_rates(count_file, num_samples):
count_called = np.loadtxt(count_file, usecols=range(1, 27, 3), dtype=np.long)
chrom = range(1, count_called.shape[0] + 1) # Chromosome number
count_full = np.sum(count_called[:, np.array([0, 1, 3, 4])], axis=1)
count_all = np.sum(count_called, axis=1)
count_partial = count_all - count_called[:, 8]
call_rate_full = (1.0 * count_full) / count_all
call_rate_partial = (1.0 * count_partial) / count_all
total_full = np.sum(count_full, axis=0)
total_partial = np.sum(count_partial, axis=0)
total_all = np.sum(count_all)
return util.Struct(chrom=chrom,
call_rate_full=call_rate_full,
call_rate_partial=call_rate_partial,
num_snps=(1.0 * total_all) / num_samples, \
total_call_rate=(100. * total_full) / total_all, \
total_call_rate_partial=(100. * total_partial) / total_all)
def test_entire_pipeline(self):
'''Run the entire pipeline. Using a small # of surrogate parents, for speed.'''
g = self.problem.genotype
# Inject a mock DAO so that we don't need the real ID coef file, which is huge here
self.problem.pedigree._idcoef_dao = mock_dao.IdCoefDao(
self.problem.pedigree.num_genotyped)
phaser = im.phase.build_phasing_pipeline(
util.Struct(impute=im.phase.IMPUTE_OPTION.IMPUTE_AND_FILL,
debug=False,
print_times=False,
stage=0))
im.phase.run_phasing_chain(
phaser, self.problem,
im.PhaseParam(distant_phasing_params=[(0.9, 2, 0.95)]))
im.itu.assert_problem_stats(self.problem, 22640, 20225, 144)
assert_equal(g.num_filled, 22640,
'Incorrect number of imputed genotypes')
assert_equal(
g.num_missing, 0,
'Incorrect number of missing genotypes; there should not be any after imputation'
)
def runExperiment(ifname):
setLogLevel('info')
print "*** Loading viro topology"
topo = loadTopology(ifname)
net = Mininet(controller=POX, switch=OVSKernelSwitch, host=ViroHost)
print "*** Creating controllers"
remoteModules = [
'viro.core', 'viro.remote.controller', 'viro.remote.dhcpd',
'viro.remote.arpd'
]
localModules = ['viro.core', 'viro.local.controller']
port = it.count(6633)
controllers = util.Struct(remote=None, local={})
name = remoteControllerName()
controllers.remote = net.addController(name,
modules=remoteModules,
port=port.next())
for vid in topo.nodes:
name = localControllerName(vid)
controllers.local[vid] = net.addController(name,
modules=localModules,
port=port.next())
print "*** Creating switches"
switches = {}
for vid in topo.nodes:
switches[vid] = net.addSwitch(switchName(vid),
dpid=util.vid2dpid(vid, 16))
print "*** Creating hosts"
hosts = {}
for vid in topo.nodes:
hosts[vid] = net.addHost(hostName(vid),
ip=None,
detachScreen=True,
disableIPv6=True,
useDHCP=True)
print "*** Creating links"
print "*** Adding switch-to-host links"
for vid, sw in switches.iteritems():
sw.linkTo(hosts[vid])
print "*** Adding switch-to-switch links"
for vid1, vid2 in topo.edges:
switches[vid1].linkTo(switches[vid2])
print "*** Starting network"
net.build()
controllers.remote.start()
for ctrl in controllers.local.itervalues():
ctrl.start()
for vid, sw in switches.iteritems():
sw.start([controllers.remote, controllers.local[vid]])
print "*** Initializing hosts"
for h in hosts.itervalues():
h.initViroHost()
print "*** Running CLI"
CLI(net)
print "*** Stopping network"
net.stop()
except pytc.Error, e:
print repr(k), repr(v)
raise e #print e
def __contains__(self, k):
return repr(k) in self.tc
def __getattr__(self, name):
return getattr(self.tc, name)
# confused over what tc really wants
#import threading
#tc_threadlocal = threading.local()
import util
tc_threadlocal = util.Struct()
tc_threadlocal.dbs = {}
def has_db(filename):
global tc_threadlocal
if not hasattr(tc_threadlocal, 'dbs'):
tc_threadlocal.dbs = {}
return filename in tc_threadlocal.dbs
def get_db(filename):
global tc_threadlocal
assert has_db(filename)
return tc_threadlocal.dbs[filename]
