def run_phasing_chain(phaser, problem, params=None): '''The main call that runs the phasing, stats saving , and post-processing as one long pipeline. Returns the populated request object.''' request = util.Struct(problem=problem, params=params if params else PhaseParam(), g_orig=problem.genotype.data.copy(), stats=util.Struct()) # Run phasing processing chain start = time.time() phaser.handle(request) t = time.time() - start request.stats.time = t return request
def run(self, problem, params=None): '''Run the phasing processing chain. Adapts the generic Filter interface to include both a Problem and PhasingParam inputs. If params=None, using default PhaseParam values.''' '''A template method that delegates to runner(), which accepts two input parameters.''' return self.handle( util.Struct(problem=problem, params=params if params else PhaseParam()))
def override_imputed_by_plink(imputed_file, plink_prefix, process_line_factory=process_line_factory, **kwargs): '''Main program for overriding imputing genotype file by a PLINK data file. Accepts an argument dictionary.''' # Default options options = util.Struct(num_header_lines=0, num_metadata_cols=8, override_tag=im.constants.UNPHASED, max_flipped_hom=10, max_mismatch_partial=5, delimiter='\t', index_file=os.environ['OBER_DATA'] + '/cgi/README.assembly_sample_subject.csv', pedigree_file=im.itu.HUTT_PED, genotype_id_file=im.examples.CHR22 + '/hutt.tfam', warnings=False, key='bp') # Override with passed arguments options.update(**kwargs) return __override_imputed_by_plink( imputed_file, plink_prefix, options, process_line_factory=process_line_factory)
def loadTopology(ifname): topo = util.Struct(nodes={}, edges=[]) for vid1, vid2 in util.itertokens(ifname, sep=" ", func=int): topo.edges.append((vid1, vid2)) topo.nodes[vid1] = True topo.nodes[vid2] = True return topo
def main(): apollocaffe.set_device(0) #apollocaffe.set_cpp_loglevel(0) apollocaffe.set_random_seed(0) np.random.seed(0) job = sys.argv[1] corpus_name = sys.argv[2] config = util.Struct(**yaml.load(CONFIG)) if corpus_name == "abstract": train_scenes, dev_scenes, test_scenes = corpus.load_abstract() else: assert corpus_name == "birds" train_scenes, dev_scenes, test_scenes = corpus.load_birds() apollo_net = ApolloNet() print "loaded data" print "%d training examples" % len(train_scenes) listener0_model = Listener0Model(apollo_net, config.model) speaker0_model = Speaker0Model(apollo_net, config.model) sampling_speaker1_model = SamplingSpeaker1Model(apollo_net, config.model) compiled_speaker1_model = CompiledSpeaker1Model(apollo_net, config.model) if job == "train.base": train(train_scenes, dev_scenes, listener0_model, apollo_net, config.opt) train(train_scenes, dev_scenes, speaker0_model, apollo_net, config.opt) apollo_net.save("models/%s.base.caffemodel" % corpus_name) exit() if job == "train.compiled": apollo_net.load("models/%s.base.caffemodel" % corpus_name) print "loaded model" train(train_scenes, dev_scenes, compiled_speaker1_model, apollo_net, config.opt) apollo_net.save("models/%s.compiled.caffemodel" % corpus_name) exit() if job in ("sample.base", "sample.compiled"): if job == "sample.base": apollo_net.load("models/%s.base.caffemodel" % corpus_name) else: apollo_net.load("models/%s.compiled.caffemodel" % corpus_name) print "loaded model" if job == "sample.base": models = { "sampling_speaker1": sampling_speaker1_model, } elif job == "sample.compiled": models = { "compiled_speaker1": compiled_speaker1_model, } name = job.split(".")[1] run_experiment("one_different", corpus_name, name, models, dev_scenes) run_experiment("by_similarity", corpus_name, name, models, dev_scenes) run_experiment("all_same", corpus_name, name, models, dev_scenes)
def opt(name, type=None, default=None): o = util.Struct(name=name, type=type, default=default) if type is None: if default is not None: o.type = type_builtin(default) else: o.type = str #raise Exception("need type for %s" % name) #if o.type==bool: o.type=int return o
def main(**kwargs): '''Main program - accepts argument dictionary.''' # Default options options = util.Struct(data_file=None, variant_id=False) # Override with passed arguments options.update(**kwargs) # (valid, options, error_msg) = __validate_options(options) # if not valid: # raise ValueError('Bad options: %s' % (error_msg,)) return __main(options)
def expect_get_posts(self): post = data.BlogPost() post.id = util.Struct(text='tag:blogger.com,1999:blog-111.post-222') feed = data.BlogFeed() feed.entry = [post] def check_path(query): return query.custom_parameters['path'] == '/path/to/post' self.client.get_posts('111', query=mox.Func(check_path)).AndReturn(feed)
def main(**kwargs): '''Main program - accepts argument dictionary.''' # Default options options = util.Struct(num_header_lines=0, num_metadata_cols=8, buf_size=10000, pedigree_file=im.itu.HUTT_PED, genotype_file=im.examples.CHR22) # Override with passed arguments options.update(**kwargs) # (valid, options, error_msg) = __validate_options(options) # if not valid: # raise ValueError('Bad options: %s' % (error_msg,)) return __main(options)
def setUp(self): super(BloggerTest, self).setUp() self.auth_entity = BloggerV2Auth(name='name', blog_ids=['111'], blog_hostnames=['my.blawg'], picture_url='http://pic') self.client = self.mox.CreateMock(BloggerClient) self.comment = data.Comment() self.comment.id = util.Struct( text='tag:blogger.com,1999:blog-111.post-222.comment-333') self.comment.to_string = lambda: '<foo></foo>'
def test_update_mastodon_pictures_get_actor_404(self): self.expect_requests_get( 'https://foo.com' + test_mastodon.API_ACCOUNT % 123, headers={'Authorization': 'Bearer towkin'}, ).AndRaise( requests.exceptions.HTTPError( response=util.Struct(status_code='404', text='foo'))) self.mox.ReplayAll() mastodon = self._setup_mastodon() resp = self.client.get('/cron/update_mastodon_pictures') self.assertEqual(200, resp.status_code) self.assertEqual('http://before', mastodon.key.get().picture)
def prepare_figure_vs_snp(info, title=None, xaxis='snp', snp_index=None): '''Prepare a template figure vs. SNP # or location in base pairs for a ProblemInfo object. Return a plot data struct.''' snp_index = snp_index if snp_index is None else info.snp_range data = util.Struct() if xaxis == 'snp': data.x = info.snp_range[snp_index] data.xlabel = 'SNP #' else: data.x = info.snp['base_pair'][snp_index] / im.constants.MEGA_BASE_PAIR data.xlabel = 'SNP Position [Mbp]' data.xmin, data.xmax = np.min(data.x), np.max(data.x) if title: P.title(title) P.xlabel(data.xlabel) P.xlim((data.xmin, data.xmax)) return data
def main(**kwargs): '''Main program - accepts argument dictionary.''' # Default options options = util.Struct(pedigree=None, prefix=None, tped=None, tfam=None, haplotype=None, out=None, debug=False, target=None, pedigree_genotyped=None) # Override with passed arguments options.update(**kwargs) valid, options, error_msg = __validate_options(options) if not valid: raise ValueError('Bad options: %s' % (error_msg, )) return __main(options)
def main(**kwargs): '''Main program - accepts argument dictionary.''' # Default options options = util.Struct(pedigree=None, prefix=None, tped=None, tfam=None, input=None, output=None, debug=False, stage=0, print_times=True, impute=IMPUTE_OPTION.NONE, min_output=False, selected_samples=None) # Override with passed arguments options.update(**kwargs) valid, options = __validate_options(options) if not valid: raise ValueError('Bad options') return __main(options)
def impute2_concordance_chrom_vs_window_size(output_dir_prefix, chrom, runs): '''Return the data of all windows along an entire chromosome, for each of several window sizes specified via the number of nodes. ''runs'' is an array of tuples (node#s, instances_per_nodes) of each run.''' stats = [None] * len(runs) for k, (nodes, instances_per_node) in enumerate(runs): windows = nodes * instances_per_node stats[k] = (nodes, windows, np.array([(s,) + impute2_stats(impute2_concordance('%s.windows_%d/chr%d/run_impute2/node-%04d/run_impute2-%04d.stats.haps' % \ (output_dir_prefix, windows, chrom, s / instances_per_node, s))) for s in xrange(windows)], dtype=[ ('window', 'i4'), ('concordance', 'f4'), ('het_concordance', 'f4'), ('bp_center', 'i12') ])) result = util.Struct() result.chrom = chrom result.instances_per_node = instances_per_node result.stats = stats return result
def main(in_file, info, segment_file, out_dir, **kwargs): '''Main program - reads options from a method parameter dictionary.''' # Default options options = util.Struct(num_processes=1, region_size=10, regions=None, snp_index=None, profile=False, min_degree=4, min_len=2.0, margin=0.8, debug=0, algorithm='naive', threshold=0.995) # Override with passed arguments options.update(**kwargs) args = in_file, info, segment_file, out_dir args, options = __validate_options(args, options) return __main(args, options)
def read_call_rates(count_file, num_samples): count_called = np.loadtxt(count_file, usecols=range(1, 27, 3), dtype=np.long) chrom = range(1, count_called.shape[0] + 1) # Chromosome number count_full = np.sum(count_called[:, np.array([0, 1, 3, 4])], axis=1) count_all = np.sum(count_called, axis=1) count_partial = count_all - count_called[:, 8] call_rate_full = (1.0 * count_full) / count_all call_rate_partial = (1.0 * count_partial) / count_all total_full = np.sum(count_full, axis=0) total_partial = np.sum(count_partial, axis=0) total_all = np.sum(count_all) return util.Struct(chrom=chrom, call_rate_full=call_rate_full, call_rate_partial=call_rate_partial, num_snps=(1.0 * total_all) / num_samples, \ total_call_rate=(100. * total_full) / total_all, \ total_call_rate_partial=(100. * total_partial) / total_all)
def test_entire_pipeline(self): '''Run the entire pipeline. Using a small # of surrogate parents, for speed.''' g = self.problem.genotype # Inject a mock DAO so that we don't need the real ID coef file, which is huge here self.problem.pedigree._idcoef_dao = mock_dao.IdCoefDao( self.problem.pedigree.num_genotyped) phaser = im.phase.build_phasing_pipeline( util.Struct(impute=im.phase.IMPUTE_OPTION.IMPUTE_AND_FILL, debug=False, print_times=False, stage=0)) im.phase.run_phasing_chain( phaser, self.problem, im.PhaseParam(distant_phasing_params=[(0.9, 2, 0.95)])) im.itu.assert_problem_stats(self.problem, 22640, 20225, 144) assert_equal(g.num_filled, 22640, 'Incorrect number of imputed genotypes') assert_equal( g.num_missing, 0, 'Incorrect number of missing genotypes; there should not be any after imputation' )
def runExperiment(ifname): setLogLevel('info') print "*** Loading viro topology" topo = loadTopology(ifname) net = Mininet(controller=POX, switch=OVSKernelSwitch, host=ViroHost) print "*** Creating controllers" remoteModules = [ 'viro.core', 'viro.remote.controller', 'viro.remote.dhcpd', 'viro.remote.arpd' ] localModules = ['viro.core', 'viro.local.controller'] port = it.count(6633) controllers = util.Struct(remote=None, local={}) name = remoteControllerName() controllers.remote = net.addController(name, modules=remoteModules, port=port.next()) for vid in topo.nodes: name = localControllerName(vid) controllers.local[vid] = net.addController(name, modules=localModules, port=port.next()) print "*** Creating switches" switches = {} for vid in topo.nodes: switches[vid] = net.addSwitch(switchName(vid), dpid=util.vid2dpid(vid, 16)) print "*** Creating hosts" hosts = {} for vid in topo.nodes: hosts[vid] = net.addHost(hostName(vid), ip=None, detachScreen=True, disableIPv6=True, useDHCP=True) print "*** Creating links" print "*** Adding switch-to-host links" for vid, sw in switches.iteritems(): sw.linkTo(hosts[vid]) print "*** Adding switch-to-switch links" for vid1, vid2 in topo.edges: switches[vid1].linkTo(switches[vid2]) print "*** Starting network" net.build() controllers.remote.start() for ctrl in controllers.local.itervalues(): ctrl.start() for vid, sw in switches.iteritems(): sw.start([controllers.remote, controllers.local[vid]]) print "*** Initializing hosts" for h in hosts.itervalues(): h.initViroHost() print "*** Running CLI" CLI(net) print "*** Stopping network" net.stop()
except pytc.Error, e: print repr(k), repr(v) raise e #print e def __contains__(self, k): return repr(k) in self.tc def __getattr__(self, name): return getattr(self.tc, name) # confused over what tc really wants #import threading #tc_threadlocal = threading.local() import util tc_threadlocal = util.Struct() tc_threadlocal.dbs = {} def has_db(filename): global tc_threadlocal if not hasattr(tc_threadlocal, 'dbs'): tc_threadlocal.dbs = {} return filename in tc_threadlocal.dbs def get_db(filename): global tc_threadlocal assert has_db(filename) return tc_threadlocal.dbs[filename]