def build_module(module, required): import os.path tools.mkdir_safe('product/modules/%s' % module) out = 'product/modules/%s/%s.o' % (module, module) srcs = [] metafile = 'modules/%s/%s.meta' % (module, module) if os.path.exists(metafile): tools.copy_file('product/modules/%s/%s.meta' % (module, module), metafile) srcs.append(metafile) meta = tools.load_meta(metafile) extra_objs = '' if 'objs' in meta: extra_objs = meta['objs'] src = 'modules/%s/%s.rip' % (module, module) if tools.depends(out, module_deps + [src]): tools.pprint('MOD', src, out) args = [ 'product/ripe', conf["RFLAGS"], '-n', module, '-c', srcs, src, extra_objs, '-o', out ] # Required (default) packages have already been typed, and are # loaded by default. Hence, they do not need to be typed. if required: args.append('--omit-typing') if conf["VERBOSITY"] > 1: args.append('-v') if required: tools.call(args) else: if not tools.try_call(args): failed_modules.append(module)
def build_module(module, required): import os.path tools.mkdir_safe('product/modules/%s' % module) out = 'product/modules/%s/%s.o' % (module, module) srcs = [] metafile = 'modules/%s/%s.meta' % (module, module) if os.path.exists(metafile): tools.copy_file('product/modules/%s/%s.meta' % (module, module), metafile) srcs.append(metafile) meta = tools.load_meta(metafile) extra_objs = '' if 'objs' in meta: extra_objs = meta['objs'] src = 'modules/%s/%s.rip' % (module, module) if tools.depends(out, module_deps + [src]): tools.pprint('MOD', src, out) args = ['product/ripe', conf["RFLAGS"], '-n', module, '-c', srcs, src, extra_objs, '-o', out] # Required (default) packages have already been typed, and are # loaded by default. Hence, they do not need to be typed. if required: args.append('--omit-typing') if conf["VERBOSITY"] > 1: args.append('-v') if required: tools.call(args) else: if not tools.try_call(args): failed_modules.append(module)
def cons_flex(target, src, depends): tdepends = [src] + depends # Check timestamps if tools.depends(target, tdepends): tools.pprint("LEX", src, target) tools.call(['flex', src])
def cons_yacc(target, src, depends): tdepends = [src] + depends # Check timestamps if tools.depends(target, tdepends): tools.pprint("YAC", src, target) tools.call(['bison', src])
def cons_obj(target, src, depends): tdepends = [src] + depends # Check timestamps if tools.depends(target, tdepends): # Build object tools.pprint('CC', src, target) tools.call([CC, CFLAGS, '-c', src, '-o', target])
def cons_module(src, dest, module, extra_CFLAGS=''): if tools.depends(dest, module_deps + [src]): tools.pprint('MOD', src, dest) args = ['product/ripe', '-n', module, '-c', src, '-o', dest, '-f', '"%s"' % extra_CFLAGS] if tools.is_verbose(): args.append('-v') tools.call(args)
def build_ranker(): dtrain = xgb.DMatrix(FLAGS.data_dir + 'train_letor.txt') dvali = xgb.DMatrix(FLAGS.data_dir + 'vali_letor.txt') dtest = xgb.DMatrix(FLAGS.data_dir + 'test_letor.txt') group_train = get_groups(FLAGS.data_dir + '/train_letor.group') group_vali = get_groups(FLAGS.data_dir + '/vali_letor.group') group_test = get_groups(FLAGS.data_dir + '/test_letor.group') recall_scale = scoring.get_recall_test_scale() params = { 'objective': 'rank:ndcg', 'max_depth': 10, 'eta': 0.1, 'gamma': 1.0, 'min_child_weight': 0.1, 'eval_metric': ['ndcg@2', 'map@2', 'ndcg@1'] } xgb_ranker = xgb.train(params, dtrain, num_boost_round=5000, early_stopping_rounds=20, evals=[(dvali, 'vali')]) ranks = xgb_ranker.predict(dtest) scores, positions = scoring.ranker_predict(ranks, dtest, group_test) tools.pprint(scores) # print(xgb_ranker.eval(dvali)) joblib.dump(xgb_ranker, FLAGS.data_dir + '/xgb_ranker.model') # xgb_ranker = joblib.load('../data/dedup/xgb_ranker.model') itest = pd.read_csv('../data/dedup/test_letor.ix', header=None, sep='\t') itest.columns = ['qid', 'synid', 'fid', 'target'] client = MongoClient(tools.c_HOST) db = client['cache'] test_qids = itest['qid'].unique().tolist() positions_solr = db['solr_positions'].find( { 'et_id': { '$in': test_qids }, 'i': { '$lte': max(group_test) - 1 } }, projection=['i']) positions_solr = pd.Series([p['i'] for p in positions_solr if p['i'] >= 0]) scoring.plot_topn_curves([positions, positions_solr], '../data/dedup/cumsum_test.pdf', scale=recall_scale, labels=['reranking', 'SOLR'], title='Test: found in top N')
def clr_predict(probs, dmtx, threshold=0.4): y = dmtx.get_label() c = Counter(y) y_pred = (probs >= threshold).astype(int) rep = classification_report(y, y_pred, labels=[1], output_dict=True) rep = rep['1'] rep['base_accuracy'] = c[0] / sum(c.values()) rep['accuracy'] = accuracy_score(y, y_pred) rep = {k: round(v, 4) for k, v in rep.items()} tools.pprint(rep) print('\n') return y_pred
def constructLayerObjects(og): ''' Reads TiledMap.ObjectGroup data and returns a list of game objects. ''' layerObjects = [] tools.pprint( og, 'ObjectGroup', GameObjectCreator.objectGroupAttrs) for o in og.objects: print ' GameObject :', o.props newGameObj = GameObjectCreator.constructGameObject(o.props) assert newGameObj layerObjects.append(newGameObj) return layerObjects
def self_test(self): if self.debug: print "launching unite self_test..." tests = self.get_tests("../tests") tools.pprint(tests) #self.run_tests(map(lambda x:"../tests/"+x,tests)) self.run_tests(tests)
def bootstrap(newripe, oldripe, depends): if tools.depends(newripe, [oldripe] + depends): tools.pprint('RIP', 'riperipe/*.rip', newripe) args = [conf["VALGRIND"], oldripe, '-s', '-o', newripe, bootstrap_srcs] tools.call(args)
def copy_file(dest, src): import shutil if tools.depends(dest, [src]): tools.pprint('CP', src, dest) shutil.copy(src, dest)
def cons_gen(gen_program, target, t): if tools.depends(target, [gen_program]): tools.pprint('GEN', gen_program, target) tools.call([gen_program, t, '>', target])
def cons_bin(target, objs, depends): tdepends = objs + depends if tools.depends(target, tdepends): tools.pprint('BIN', target) tools.call([CC, CFLAGS, LFLAGS, objs, '-o', target])
def link_objs(objs, output): # Link objects into the output program if tools.depends(output, objs): arr = tools.flatten([LD, '-r', objs, '-o', output]) tools.pprint('LD', output) tools.call(arr)