def build_module(module, required):
import os.path
tools.mkdir_safe('product/modules/%s' % module)
out = 'product/modules/%s/%s.o' % (module, module)
srcs = []
metafile = 'modules/%s/%s.meta' % (module, module)
if os.path.exists(metafile):
tools.copy_file('product/modules/%s/%s.meta' % (module, module),
metafile)
srcs.append(metafile)
meta = tools.load_meta(metafile)
extra_objs = ''
if 'objs' in meta:
extra_objs = meta['objs']
src = 'modules/%s/%s.rip' % (module, module)
if tools.depends(out, module_deps + [src]):
tools.pprint('MOD', src, out)
args = [
'product/ripe', conf["RFLAGS"], '-n', module, '-c', srcs, src,
extra_objs, '-o', out
]
# Required (default) packages have already been typed, and are
# loaded by default. Hence, they do not need to be typed.
if required:
args.append('--omit-typing')
if conf["VERBOSITY"] > 1:
args.append('-v')
if required:
tools.call(args)
else:
if not tools.try_call(args):
failed_modules.append(module)
def build_module(module, required):
import os.path
tools.mkdir_safe('product/modules/%s' % module)
out = 'product/modules/%s/%s.o' % (module, module)
srcs = []
metafile = 'modules/%s/%s.meta' % (module, module)
if os.path.exists(metafile):
tools.copy_file('product/modules/%s/%s.meta' % (module, module), metafile)
srcs.append(metafile)
meta = tools.load_meta(metafile)
extra_objs = ''
if 'objs' in meta:
extra_objs = meta['objs']
src = 'modules/%s/%s.rip' % (module, module)
if tools.depends(out, module_deps + [src]):
tools.pprint('MOD', src, out)
args = ['product/ripe', conf["RFLAGS"], '-n', module,
'-c', srcs, src, extra_objs, '-o', out]
# Required (default) packages have already been typed, and are
# loaded by default. Hence, they do not need to be typed.
if required:
args.append('--omit-typing')
if conf["VERBOSITY"] > 1:
args.append('-v')
if required:
tools.call(args)
else:
if not tools.try_call(args):
failed_modules.append(module)
def cons_flex(target, src, depends):
tdepends = [src] + depends
# Check timestamps
if tools.depends(target, tdepends):
tools.pprint("LEX", src, target)
tools.call(['flex', src])
def cons_yacc(target, src, depends):
tdepends = [src] + depends
# Check timestamps
if tools.depends(target, tdepends):
tools.pprint("YAC", src, target)
tools.call(['bison', src])
def cons_obj(target, src, depends):
tdepends = [src] + depends
# Check timestamps
if tools.depends(target, tdepends):
# Build object
tools.pprint('CC', src, target)
tools.call([CC, CFLAGS, '-c', src, '-o', target])
def cons_module(src, dest, module, extra_CFLAGS=''):
if tools.depends(dest, module_deps + [src]):
tools.pprint('MOD', src, dest)
args = ['product/ripe', '-n', module, '-c', src, '-o', dest,
'-f', '"%s"' % extra_CFLAGS]
if tools.is_verbose():
args.append('-v')
tools.call(args)
def build_ranker():
dtrain = xgb.DMatrix(FLAGS.data_dir + 'train_letor.txt')
dvali = xgb.DMatrix(FLAGS.data_dir + 'vali_letor.txt')
dtest = xgb.DMatrix(FLAGS.data_dir + 'test_letor.txt')
group_train = get_groups(FLAGS.data_dir + '/train_letor.group')
group_vali = get_groups(FLAGS.data_dir + '/vali_letor.group')
group_test = get_groups(FLAGS.data_dir + '/test_letor.group')
recall_scale = scoring.get_recall_test_scale()
params = {
'objective': 'rank:ndcg',
'max_depth': 10,
'eta': 0.1,
'gamma': 1.0,
'min_child_weight': 0.1,
'eval_metric': ['ndcg@2', 'map@2', 'ndcg@1']
}
xgb_ranker = xgb.train(params,
dtrain,
num_boost_round=5000,
early_stopping_rounds=20,
evals=[(dvali, 'vali')])
ranks = xgb_ranker.predict(dtest)
scores, positions = scoring.ranker_predict(ranks, dtest, group_test)
tools.pprint(scores)
# print(xgb_ranker.eval(dvali))
joblib.dump(xgb_ranker, FLAGS.data_dir + '/xgb_ranker.model')
# xgb_ranker = joblib.load('../data/dedup/xgb_ranker.model')
itest = pd.read_csv('../data/dedup/test_letor.ix', header=None, sep='\t')
itest.columns = ['qid', 'synid', 'fid', 'target']
client = MongoClient(tools.c_HOST)
db = client['cache']
test_qids = itest['qid'].unique().tolist()
positions_solr = db['solr_positions'].find(
{
'et_id': {
'$in': test_qids
},
'i': {
'$lte': max(group_test) - 1
}
},
projection=['i'])
positions_solr = pd.Series([p['i'] for p in positions_solr if p['i'] >= 0])
scoring.plot_topn_curves([positions, positions_solr],
'../data/dedup/cumsum_test.pdf',
scale=recall_scale,
labels=['reranking', 'SOLR'],
title='Test: found in top N')
def clr_predict(probs, dmtx, threshold=0.4):
y = dmtx.get_label()
c = Counter(y)
y_pred = (probs >= threshold).astype(int)
rep = classification_report(y, y_pred, labels=[1], output_dict=True)
rep = rep['1']
rep['base_accuracy'] = c[0] / sum(c.values())
rep['accuracy'] = accuracy_score(y, y_pred)
rep = {k: round(v, 4) for k, v in rep.items()}
tools.pprint(rep)
print('\n')
return y_pred
def constructLayerObjects(og):
'''
Reads TiledMap.ObjectGroup data and returns a list of
game objects.
'''
layerObjects = []
tools.pprint(
og,
'ObjectGroup',
GameObjectCreator.objectGroupAttrs)
for o in og.objects:
print ' GameObject :', o.props
newGameObj = GameObjectCreator.constructGameObject(o.props)
assert newGameObj
layerObjects.append(newGameObj)
return layerObjects
def self_test(self):
if self.debug: print "launching unite self_test..."
tests = self.get_tests("../tests")
tools.pprint(tests)
#self.run_tests(map(lambda x:"../tests/"+x,tests))
self.run_tests(tests)
def bootstrap(newripe, oldripe, depends):
if tools.depends(newripe, [oldripe] + depends):
tools.pprint('RIP', 'riperipe/*.rip', newripe)
args = [conf["VALGRIND"], oldripe,
'-s', '-o', newripe, bootstrap_srcs]
tools.call(args)
def bootstrap(newripe, oldripe, depends):
if tools.depends(newripe, [oldripe] + depends):
tools.pprint('RIP', 'riperipe/*.rip', newripe)
args = [conf["VALGRIND"], oldripe, '-s', '-o', newripe, bootstrap_srcs]
tools.call(args)
def copy_file(dest, src):
import shutil
if tools.depends(dest, [src]):
tools.pprint('CP', src, dest)
shutil.copy(src, dest)
def cons_gen(gen_program, target, t):
if tools.depends(target, [gen_program]):
tools.pprint('GEN', gen_program, target)
tools.call([gen_program, t, '>', target])
def cons_bin(target, objs, depends):
tdepends = objs + depends
if tools.depends(target, tdepends):
tools.pprint('BIN', target)
tools.call([CC, CFLAGS, LFLAGS, objs, '-o', target])
def link_objs(objs, output):
# Link objects into the output program
if tools.depends(output, objs):
arr = tools.flatten([LD, '-r', objs, '-o', output])
tools.pprint('LD', output)
tools.call(arr)
