def read_2d_variable_as_map(filename, var_name):
"""
Reads variables from the solution file of a GLPK-solved problem in the
following form:
No. Column name St Activity Lower bound Upper bound Marginal
------ ------------ -- ------------- ------------- ------------- -------------
1 avb[0,0] B 103 0
2 avb[0,1] B 0 0
The variable must be precisely 2-d.
:param filename: name of the file containing the GLPK solution
:param var_name: name of the variable to be read
:return: a dictionary of dictionaries containing the value of the 2-d variable
"""
result = {}
regex = "{0}\[(\d+),(\d+)\]\s*(\*)?\s*(\d+(\.\d+)?)".format(var_name)
def my_proc(m):
value = float(m.group(4))
if value == 0:
return
row = int(m.group(1))
col = int(m.group(2))
if row not in result:
result[row] = {}
result[row][col] = value
util.parse(filename, regex, my_proc)
return result
def __init__(self, *args, **kwargs):
check_args(*args, **kwargs)
self._fmap = {}
self._bmap = {}
self._rawmap = args[0]
self._rflag = False
self._fmap = parse(args, flag=self._rflag)
self._bmap = parse(args, flag=not self._rflag)
def read_status(file_name):
result = []
def status_shredder(matcher):
result.append(matcher.group(1))
result.append(matcher.group(2))
util.parse(file_name, "Status:\s*(\w+)\s*(\w+)", status_shredder)
return result
def build_replica_networks(node_count,
replica_count,
slave_factor,
previous=None,
working_dir='.',
use_existing_solution=False):
"""
Returns a 3-d structure (map of map of maps) describing the replica networks for
a cluster of node_count size with replica_count replicas.
:param node_count: the number of nodes
:param replica_count: the number of replicas
:param slave_factor: the max slaves of each node
:param previous: the previous replica connections
:param working_dir: the working directory
:param use_existing_solution: whether to use an existing solution if found
:return:
"""
if working_dir is None:
working_dir = '.'
if previous is None:
previous = [[[0 for _ in range(node_count)]
for _ in range(node_count)]
for _ in range(replica_count)]
prev_connections_string = make_prev_connections_string(previous)
params = dict(n=node_count,
r=replica_count,
s=slave_factor,
prev_connections=prev_connections_string)
m = get_replica_gen_model()
m.working_dir = working_dir
m.set_use_existing_solution(use_existing_solution)
result = m.solve(params)
if result != 0:
print "No solution to problem: return code is: ", result
exit(result)
result_file = m.get_result_file()
# next read replication matrix
rep = {}
regex = "x\[(\d+),(\d+)\,(\d+)]\s*\*\s*(\d+)"
def my_processor(m):
val = int(m.group(4))
if val == 0:
return
rk = int(m.group(3))
if rk not in rep:
rep[rk] = {}
ni = int(m.group(1))
if ni not in rep[rk]:
rep[rk][ni] = {}
nj = int(m.group(2))
rep[rk][ni][nj] = val
util.parse(result_file, regex, my_processor)
return rep
def delitem(self, key):
count = 0
raw = deepcopy(self._rawmap)
for (k, v) in raw:
if k == key or v == key:
self._rawmap.remove((k, v))
count += 1
if count > 0:
self._bmap = parse([self._rawmap], True)
self._fmap = parse([self._rawmap], False)
else:
raise TypeError('the key isn`t exist.')
def test_parse(self):
contents = util.parse(open('test_input.txt'))
self.assertEquals(4, len(contents))
self.assertEquals('nhms', contents[0][0][3])
self.assertEquals('dairy', contents[0][1][0])
self.assertEquals('fish', contents[0][1][1])
self.assertEquals('fish', contents[3][1][0])
contents = util.parse(open('input.txt'))
self.assertEquals(38, len(contents))
self.assertEquals('nchjqv', contents[37][0][0])
self.assertEquals('eggs', contents[37][1][0])
pass
def add(self, key, value):
if isinstance(value, set):
if not self._rflag:
for i in value:
self._rawmap.append((key, i))
else:
for i in value:
self._rawmap.append((i, key))
self._bmap = parse([self._rawmap], True)
self._fmap = parse([self._rawmap], False)
else:
raise TypeError('the args isn`t right.')
def poll(self, timestamp, filters):
logging.info('Polling feed "%s"' % self.url)
result = []
self.last_poll = timestamp
username = util.decode_password(self.username)
password = util.decode_password(self.password)
d = util.parse(self.url, username, password, self.etag, self.modified)
self.etag = util.get(d, 'etag', None)
self.modified = util.get(d, 'modified', None)
feed = util.get(d, 'feed', None)
if feed:
self.title = self.title or util.get(feed, 'title', '')
self.link = self.link or util.get(feed, 'link', self.url)
entries = util.get(d, 'entries', [])
for entry in reversed(entries):
id = create_id(entry)
if id in self.id_set:
continue
self.item_count += 1
self.id_list.append(id)
self.id_set.add(id)
item = Item(self, id)
item.timestamp = calendar.timegm(
util.get(entry, 'date_parsed', time.gmtime()))
item.title = util.format(util.get(entry, 'title', ''),
settings.POPUP_TITLE_LENGTH)
item.description = util.format(util.get(entry, 'description', ''),
settings.POPUP_BODY_LENGTH)
item.link = util.get(entry, 'link', '')
item.author = util.format(util.get(entry, 'author',
'')) # TODO: max length
if all(filter.filter(item) for filter in filters):
result.append(item)
self.clean_cache(settings.FEED_CACHE_SIZE)
return result
def q1(testcase):
n, s, u = util.parse(testcase)
game = Game(n, s, u)
print('\n---------------- strong dominant strategies ----------------')
for i in range(1, n + 1):
sds_i = game.strongly_dominant_strategy(i)
print(f'player {i}: sds = {sds_i}')
print('\n----------------- weak dominant strategies -----------------')
for i in range(1, n + 1):
wds_i = game.weakly_dominant_strategy(i)
print(f'player {i}: wds = {wds_i}')
print('\n----------- strong dominant strategy equilibrium -----------')
print(f'sdse: {game.sdse()}')
print('\n------------ weak dominant strategy equilibrium ------------')
print(f'wdse: {game.wdse()}')
print('\n-------------- pure strategy nash equilibrium --------------')
psne = game.psne()
if psne is None:
psne = 'does not exist'
print(f'psne: {psne}')
print('\n-------------------------- maxmin --------------------------')
for i in range(1, n + 1):
value, strategy_set = game.maxmin(i)
print(f'player {i}: value = {value}, strategies = {strategy_set}')
print('\n-------------------------- minmax --------------------------')
for i in range(1, n + 1):
value, strategy_set = game.minmax(i)
print(f'player {i}: value = {value}, strategies = {strategy_set}')
def main():
try:
q = unicodedata.normalize('NFC', unicode(sys.argv[1].strip()))
except:
q = ""
(title, area, tag, day, note) = util.parse(q[1:])
display = title.strip() + ' '
if area != 'Inbox':
display += '@' + area + ' '
if day:
display += '> ' + day.strftime('%m/%d %a')
subtitle = "title #tag @t|n|s ::note >duedate (ex. fri, 3d, 2w, 12/31)"
help = {"valid": False, "subtitle": subtitle}
output = []
output.append({
"title": display,
"subtitle": subtitle,
"arg": q,
"valid": True,
"mods": {
"alt": help,
"ctrl": help,
"cmd": help,
"shift": help
}
})
print json.dumps({"items": output})
def work(self, body):
data = parse(body, DataSet)
print "Tokenizing " + data.id
for s in data.sentences:
s.tokens = word_tokenize(s.text)
self.write(as_json(data))
def poll(self, timestamp, filters):
logging.info('Polling feed "%s"' % self.url)
result = []
self.last_poll = timestamp
username = util.decode_password(self.username)
password = util.decode_password(self.password)
d = util.parse(self.url, username, password, self.etag, self.modified)
self.etag = util.get(d, 'etag', None)
self.modified = util.get(d, 'modified', None)
feed = util.get(d, 'feed', None)
if feed:
self.title = self.title or util.get(feed, 'title', '')
self.link = self.link or util.get(feed, 'link', self.url)
entries = util.get(d, 'entries', [])
for entry in reversed(entries):
id = create_id(entry)
if id in self.id_set:
continue
self.item_count += 1
self.id_list.append(id)
self.id_set.add(id)
item = Item(self, id)
item.timestamp = calendar.timegm(util.get(entry, 'date_parsed', time.gmtime()))
item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH)
item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH)
item.link = util.get(entry, 'link', '')
item.author = util.format(util.get(entry, 'author', '')) # TODO: max length
if all(filter.filter(item) for filter in filters):
result.append(item)
self.clean_cache(settings.FEED_CACHE_SIZE)
return result
def solve(input_path):
f = open(input_path)
tiles = util.parse(f)
grid = construct_grid(tiles)
# Now the values of tiles have been flipped and/or rotated
# and grid is a 2-dimensional array indexed as [x][y].
for k, v in tiles.iteritems():
tiles[k] = util.strip_border(v)
# Next make 8 versions of string lists.
images = [None] * 8
images[0] = util.create_image(tiles, grid)
images[1] = util.text_rotate(images[0])
images[2] = util.text_rotate(images[1])
images[3] = util.text_rotate(images[2])
images[4] = util.text_flip(images[3])
images[5] = util.text_rotate(images[4])
images[6] = util.text_rotate(images[5])
images[7] = util.text_rotate(images[6])
for i in range(len(images)):
if util.highlight_monsters(images[i]):
return sum(
[sum([1 for c in row if c == '#']) for row in images[i]])
def main():
q = sys.argv[1]
(title, area, tag, day, note) = util.parse(q[1:])
if not title: return;
script = """tell application id "com.culturedcode.ThingsMac"
activate
set newToDo to make new to do
set name of newToDo to "%s"
""" % title
if q[0] == 'm' and not note:
note = getClipboardText()
if note:
note = note.replace('"', '\\\"')
note = note.replace('\n', '\\\n')
script += ' set notes of newToDo to "%s"\n'%(note)
if day:
today = date.today()
delta = day - today
script += ' set due date of newToDo to (current date)+%d*days\n'% (delta.days)
if tag:
script += 'set tag names of newToDo to "%s"\n'%(tag)
script += 'show newTodo\nmove newTodo to list id "%s"\nend tell' % area
# print script
applescript.AppleScript(script).run()
def work(self, body):
data = parse(body, DataSet)
print "Tagging " + data.id
for sent in data.sentences:
tokens = [t.word for t in sent.tokens]
sent.tokens = [Token(t[0], t[1]) for t in pos_tag(tokens)]
self.write(as_json(data))
def solve(input_path):
f = open(input_path)
player1, player2 = util.parse(f)
winner_number = util.play_part2_game(player1, player2, 0)
winner = player1
if winner_number == 2:
winner = player2
return util.score(winner)
def work(self, body):
data = parse(body, DataSet)
print "Splitting " + data.id
sentences = sent_tokenize(data.text)
for s in sentences:
data.sentences.append(Sentence(s))
self.write(as_json(data))
def mongo_count(self, line, cell=''):
if not self._conn:
return "[ERROR] connect mongodb before %count"
parsed = parse('%s\n%s' % (line, cell), self)
try:
return parsed['collection'].count()
except Exception as e:
return "[ERROR] fail to query ", e
def mongo_count(self, line, cell=''):
if not self._conn:
return "[ERROR] connect mongodb before %count"
parsed = parse('%s\n%s' % (line, cell), self)
try:
return parsed['collection'].count()
except Exception as e:
return "[ERROR] fail to query ", e
def mongo_find_one(self, line, cell=''):
if not self._conn:
return "[ERROR] connect mongodb before %find"
parsed = parse('%s\n%s' % (line, cell), self)
try:
query = find_query_pymongo(parsed['data'])
return parsed['collection'].find_one(query)
except Exception as e:
return "[ERROR] fail to query ", e
def solve(input_path):
f = open(input_path)
player1, player2 = util.parse(f)
while (player1 and player2):
util.play_round(player1, player2)
winner = player1
if player2:
winner = player2
return util.score(winner)
def insert(self, line, cell='', local_ns={}):
if not self._conn:
return "[ERROR] connect mongodb before %insert"
parsed = parse('%s\n%s' % (line, cell), self)
try:
data = insert_query_pymongo(parsed['data'])
parsed['collection'].insert(data)
except Exception as e:
return "[ERROR] fail to insert data", e
def mongo_print(self, line, cell=''):
if not self._conn:
return "[ERROR] connect mongodb before %print"
parsed = parse('%s\n%s' % (line, cell), self)
# print db.collection.find()
try:
result = print_json(parsed['data'])
except Exception:
return "[ERROR] fail to print about", parsed['data']
def insert(self, line, cell='', local_ns={}):
if not self._conn:
return "[ERROR] connect mongodb before %insert"
parsed = parse('%s\n%s' % (line, cell), self)
try:
data = insert_query_pymongo(parsed['data'])
parsed['collection'].insert(data)
except Exception as e:
return "[ERROR] fail to insert data", e
def mongo_print(self, line, cell=''):
if not self._conn:
return "[ERROR] connect mongodb before %print"
parsed = parse('%s\n%s' % (line, cell), self)
# print db.collection.find()
try:
result = print_json(parsed['data'])
except Exception:
return "[ERROR] fail to print about", parsed['data']
def mongo_find_one(self, line, cell=''):
if not self._conn:
return "[ERROR] connect mongodb before %find"
parsed = parse('%s\n%s' % (line, cell), self)
try:
query = find_query_pymongo(parsed['data'])
return parsed['collection'].find_one(query)
except Exception as e:
return "[ERROR] fail to query ", e
def q2(testcase):
n, s, u = util.parse(testcase)
game = TwoPlayer(n, s, u)
msne = game.msne()
if msne is None:
msne = 'not able to find'
print(f'\nmsne: (P1, P2) = {msne}\n')
def extract_text_to_arr(self, files, parseType, parseRule, columnTypes=None, filter=None):
textRDD = self.textFiles(files)
wordRDD = textRDD.map(lambda line: util.parse(line, parseType, parseRule, columnTypes)).filter(
lambda words: words != None)
if filter:
wordRDD = wordRDD.filter(filter)
return util.ETLRDD(wordRDD._jrdd, self, wordRDD._jrdd_deserializer)
def poll(self, timestamp, filters):
try:
logging.info('Polling feed "%s"' % self.url)
result = []
self.last_poll = timestamp
username = util.decode_password(self.username)
password = util.decode_password(self.password)
d = util.parse(self.url, username, password, self.etag, self.modified)
self.etag = util.get(d, 'etag', None)
self.modified = util.get(d, 'modified', None)
feed = util.get(d, 'feed', None)
if feed:
self.title = self.title or util.get(feed, 'title', '')
self.link = self.link or util.get(feed, 'link', self.url)
entries = util.get(d, 'entries', [])
for entry in reversed(entries):
found = False
id = create_id(entry)
for i in self.id_set:
# be cheap, compare the links - they should be unique.
# Can't compare objects anymore because the time is present which will always be different
if id[1] == i[1]:
found = True
break
if found:
continue # The entry is not new, move on to the next one.
self.id_set.add(id)
self.item_count += 1
item = Item(self, id)
item.timestamp = calendar.timegm(util.get(entry, 'date_parsed', time.gmtime()))
item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH)
item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH)
item.link = util.get(entry, 'link', '')
item.author = util.format(util.get(entry, 'author', '')) # TODO: max length
if all(filter.filter(item) for filter in filters):
result.append(item)
# determine if there are any entries in self.id_set that are older than CACHE_AGE_LIMIT
# if there are, remove them from the list (they aged out of the cache)
idsToRemove = set()
now = datetime.fromtimestamp(time.time())
for tempId in self.id_set:
diff = now - datetime.fromtimestamp(tempId[3])
if abs(diff.days) > settings.CACHE_AGE_LIMIT:
idsToRemove.add(tempId) # can't modify seld.id_set right here.
for i in idsToRemove:
logging.info('Removing %s because its too old' % i[1])
self.id_set.remove(i)
return result
except Exception, e:
logging.info('Error durring poll: %s' % e)
raise
def solve(input_path):
f = open(input_path)
parsed = util.parse(f)
num_foods = len(parsed)
allergens = set()
ingredients = set()
for pair in parsed:
for ingredient in pair[0]:
ingredients.add(ingredient)
for allergen in pair[1]:
allergens.add(allergen)
print('Number of foods and allergens:', num_foods, len(allergens))
# Maps allergen string to a set of foods (as ints) containing the allergen
allergen_to_food = dict()
for allergen in allergens:
allergen_to_food[allergen] = set([
index for index in range(num_foods) if allergen in parsed[index][1]
])
print('allergen_to_food', allergen_to_food)
# Maps allergen to a set (which will be modified) of ingredients.
# In truth exactly one ingredient contains the allergen.
allergen_to_possible_ingredients = dict()
for allergen in allergens:
for food in allergen_to_food[allergen]:
new_set = set(parsed[food][0])
if allergen not in allergen_to_possible_ingredients:
allergen_to_possible_ingredients[allergen] = new_set
else:
allergen_to_possible_ingredients[allergen] = (
allergen_to_possible_ingredients[allergen].intersection(
new_set))
print('allergen_to_possible_ingredients', allergen_to_possible_ingredients)
suspicious_ingredients = set(
itertools.chain.from_iterable([
allergen_to_possible_ingredients[allergen]
for allergen in allergens
]))
clear_ingredients = [
ingredient for ingredient in ingredients
if ingredient not in suspicious_ingredients
]
print('sus', list(suspicious_ingredients))
print('clear', list(clear_ingredients))
print(parsed)
result = 0
for pair in parsed:
result += sum(
[1 for ingredient in pair[0] if ingredient in clear_ingredients])
return result
def read_variable(filename, var_name):
"""
Reads variables from the solution of a GLPK-solved problem in the
following form:
No. Column name St Activity Lower bound Upper bound Marginal
------ ------------ -- ------------- ------------- ------------- -------------
1 avb[0,0] B 103 0
2 avb[0,1] B 0 0
The variables must be a 3-d variable max.
:param filename: name of the file containing the GLPK solution
:param var_name: name of the variable
:return: a multi-dimensional array holding the variable result
"""
result = []
# groups for the following regular expression
# (g1) (g2(g3)(g4(g5)) (g6) (g7 (g8))
regex = "{0}\[(\d+)(,(\d+)(,(\d+))?)?\]\s*([A-Z\*]+)?\s*(\d+(\.\d+)?)".format(var_name)
def my_proc(m):
i = int(m.group(1))
j = int(m.group(3)) if m.group(3) is not None else None
k = int(m.group(5)) if m.group(5) is not None else None
val = float(m.group(7))
if j is not None:
util.ensure_has_capacity(result, i + 1, lambda: [])
if k is not None:
util.ensure_has_capacity(result[i], j + 1, lambda: [])
util.ensure_has_capacity(result[i][j], k + 1)
result[i][j][k] = val
else:
util.ensure_has_capacity(result[i], j + 1)
result[i][j] = val
else:
util.ensure_has_capacity(result, i + 1)
result[i] = val
util.parse(filename, regex, my_proc)
return result
def __delitem__(self, key):
if not self._rflag:
count = 0
raw = deepcopy(self._rawmap)
for (k, v) in raw:
if k == key:
self._rawmap.remove((k, v))
count += 1
if count > 0:
self._bmap = parse([self._rawmap], True)
self._fmap = parse([self._rawmap], False)
else:
count = 0
raw = deepcopy(self._rawmap)
for (k, v) in raw:
if v == key:
self._rawmap.remove((k, v))
count += 1
if count > 0:
self._bmap = parse([self._rawmap], True)
self._fmap = parse([self._rawmap], False)
def combine_result(feature_list): combo_feature = dict() for file_name in feature_list: for e in util.parse(file_name): user = e['user_id'] merchant = e['merchant_id'] prob = e['prob'] um = (user,merchant) if um not in combo_feature: combo_feature[um] = [] combo_feature[um].append(prob) return combo_feature
def __init__(self, data_path, C, epsilon, show_prog):
_, y, X = parse(data_path)
self._show_prog = show_prog
b, alpha = self.__optimize(y, X, C, epsilon)
alpha_i = []
for i in range(len(alpha)):
if alpha.item(i,0) != 0:
alpha_i.append(i)
self._b = b
self._y = y
self._X = X
self._alpha = alpha
self._alpha_i = alpha_i
def q3(testcase):
n, s, u = util.parse(testcase)
game = TwoPlayerZeroSum(n, s, u)
saddle_point = game.saddle_point()
msne = game.msne()
if saddle_point is None:
saddle_point = 'does not exist'
if msne is None:
msne = 'not able to find'
print(f'\nsaddle point: {saddle_point}')
print(f'msne: (P1, P2) = {msne}\n')
def work(self, body):
terminators = ('.', '!', '?')
question = parse(body)
print "Combining snippets for " + question['id']
snippets = list()
for snippet in question['snippets']:
text = snippet['text'].strip()
if not text.endswith(terminators):
text = text + '.'
snippets.append(text)
data = DataSet()
data.id = question['id']
data.text = ' '.join(snippets)
self.write(as_json(data))
def lr_aggregation(self,train_label,prob_threshold): train_result_set = set() for e in util.parse(train_label): train_result_set.add((int(e['user_id']),int(e['item_id']))) f1_ay = np.zeros(self.n_models) for i in xrange(self.n_models): pred = self.train_data[self.model_list[i].predict_proba(self.train_x)[:,1]>=prob_threshold] result = set() for k in xrange(len(pred)): result.add((int(pred[k][0]),int(pred[k][1]))) user_item_non_geo = set() for uik in result: user = str(uik[0]) item = str(uik[1]) if self.user_geo[user] != set() and self.item_geo[item] !=set(): if self.user_geo[user]&self.item_geo[item] == set(): user_item_non_geo.add(uik) for ui in user_item_non_geo: result.remove(ui) f1_ay[i] = util.offline_f1(result,train_result_set) # f1_ay = preprocessing.scale(f1_ay) self.lr_agg_coef = np.zeros(self.n_features) self.lr_agg_intercept = 0 for i in xrange(self.n_models): coef = self.model_list[i].coef_.reshape(self.n_features) intercept = self.model_list[i].intercept_.reshape(1) self.lr_agg_coef += f1_ay[i] * coef self.lr_agg_intercept += f1_ay[i] * intercept self.lr_agg_coef /= np.sum(f1_ay) self.lr_agg_intercept /= np.sum(f1_ay) def sigmoid_decision(x): return 1.0 / (1+np.exp(-(np.dot(self.lr_agg_coef,x.T)+self.lr_agg_intercept))) lr_agg_pred = self.val_data[sigmoid_decision(self.val_x)>=prob_threshold] print result = set() for k in xrange(len(pred)): result.add((int(pred[k][0]),int(pred[k][1]))) util.offline_f1(result,self.val_set) user_item_non_geo = set() for uik in result: user = str(uik[0]) item = str(uik[1]) if self.user_geo[user] != set() and self.item_geo[item] !=set(): if self.user_geo[user]&self.item_geo[item] == set(): user_item_non_geo.add(uik) for ui in user_item_non_geo: result.remove(ui) f1 = util.offline_f1(result,self.val_set) return f1
def __init__(self, data_path, C, epsilon, show_prog):
_, y, X = parse(data_path)
self._show_prog = show_prog
b, alpha = self.__optimize(y, X, C, epsilon)
alpha_i = []
for i in range(len(alpha)):
if alpha.item(i, 0) != 0:
alpha_i.append(i)
self._b = b
self._y = y
self._X = X
self._alpha = alpha
self._alpha_i = alpha_i
def test_minmax():
"""test minmax value of a game"""
testcase = 'testdir/test.game/test.1'
n, s, u = util.parse(testcase)
game = Game(n, s, u)
minmax_value = [1, 2]
minmax_strategy = [set(['c']), set(['q', 'r'])]
for i in range(1, n + 1):
value, strategy = game.minmax(i)
assert minmax_value[i - 1] == value, 'minmax value failed'
assert minmax_strategy[i - 1] == strategy, 'minmax strategy failed'
def test_maxmin():
"""test maxmin value of a game"""
testcase = 'testdir/test.game/test.2'
n, s, u = util.parse(testcase)
game = Game(n, s, u)
maxmin_value = [2, 2]
maxmin_strategy = [set(['a', 'b']), set(['p'])]
for i in range(1, n + 1):
value, strategy = game.maxmin(i)
assert maxmin_value[i - 1] == value, 'maxmin value failed'
assert maxmin_strategy[i - 1] == strategy, 'maxmin strategy failed'
def test_parse():
testcase = 'testdir/test.util'
n, s, u = parse(testcase)
assert n == 2, f'{testcase} [n] got {n} expected 2'
assert len(s) == 2, f'{testcase} [len(s)] got {len(s)} expected 2'
res_s = [['0', '1'], ['0', '1']]
assert s == res_s, f'{testcase} [strategy_profile] got {s}, expected {res_s}'
strategy_vectors = [['0', '0'], ['0', '1'], ['1', '0'], ['1', '1'],
['1', 'a']]
results = [[-2, 2], [5, -5], [3, -3], [0, 0], None]
for sv, res in zip(strategy_vectors, results):
assert u(sv) == res, f'{testcase} [utility_function] failed'
def connect(self):
self.handshake = encode(chr(19) + 'BitTorrent protocol' + (8 * chr(0))) + self.info_hash + self.peer_id
self.sock = socket.create_connection((self.ip, self.port), 3)
self.sock.sendall(self.handshake)
self.response = self.sock.recv(4096)
self.mesg = bytearray(self.response)
try:
self.response2 = self.sock.recv(4096)
self.mesg += bytearray(self.response2)
except:
pass
self.check()
self.mesglist = parse(self.mesg)
self.pieces = piecelist(self.mesglist)
def out_similar_merchant(user_beh_log,similar_file):
# Compute the top 5 similar partners for each merchant.
# The similarity is simply defined by normalized number of overlapping customers
print "Compute the top 5 similar partners for each merchant"
merchant_customer = dict()
for e in util.parse(user_beh_log):
user = e['user_id']
merchant = e['seller_id']
act = int(e['action_type'])
if merchant not in merchant_customer:
merchant_customer[merchant] = set()
if act == 2:
merchant_customer[merchant].add(user)
num_merchant = len(merchant_customer)
merchant_list = np.array(merchant_customer.keys())
print "merchant num: %d"%(len(merchant_list))
m2m_sim = np.zeros((num_merchant,num_merchant))
for i in xrange(num_merchant):
for j in xrange(num_merchant):
if i != j:
m2m_sim[i][j] = len(merchant_customer[merchant_list[i]]&merchant_customer[merchant_list[j]])
# m2m_sim = preprocessing.StandardScaler().fit_transform(m2m_sim)
m2m_sim = preprocessing.normalize(m2m_sim,norm='l2')
merchant_sim_m = dict()
for i in xrange(num_merchant):
top_k = merchant_list[np.argsort(m2m_sim[i,:])[-5:]]
merchant_sim_m[merchant_list[i]] = top_k
# print merchant_sim_m
fo = open(similar_file, "w")
fo.write('merchant_id,sim_m1,sim_m2,sim_m3,sim_m4,sim_m5\n')
for i in xrange(num_merchant):
fo.write(merchant_list[i]+',')
for k in xrange(5):
if k != 4:
fo.write(merchant_sim_m[merchant_list[i]][k]+',')
else:
fo.write(merchant_sim_m[merchant_list[i]][k]+'\n')
fo.close()
return m2m_sim
def test_weak_dominant():
"""test weak dominant strategy and equilibrium"""
testcase = 'testdir/test.game/test.2'
n, s, u = util.parse(testcase)
game = Game(n, s, u)
real_weak_strategies = ['a', 'p']
weak_strategies = list()
for i in range(1, n + 1):
weak_strategies.append(game.weakly_dominant_strategy(i))
assert real_weak_strategies == weak_strategies, 'weakly dominant strategy failed'
real_wdse = ['a', 'p']
wdse = game.wdse()
assert real_wdse == wdse, 'wdse failed'
def test_strong_dominant():
"""test strong dominant strategy and equilibrium"""
# create game from testcase
testcase = 'testdir/test.game/test.1'
n, s, u = util.parse(testcase)
game = Game(n, s, u)
real_strong_strategies = ['a', 'p']
strong_strategies = list()
for i in range(1, n + 1):
strong_strategies.append(game.strongly_dominant_strategy(i))
assert real_strong_strategies == strong_strategies, 'strongly dominant strategy failed'
real_sdse = ['a', 'p']
sdse = game.sdse()
assert real_sdse == sdse, 'sdse failed'
def test_zero_sum():
testcase = 'testdir/test.msne/test.0'
n, s, u = util.parse(testcase)
game = TwoPlayerZeroSum(n, s, u)
game = game.iterative_elimination()
result = game.msne()
# if linear program is not able to find the MSNE then fail
assert result is not None, 'Non zero sum game MSNE failed'
result_1, result_2 = result
logging.info(
f'[iterative elimination] S1 = {game.s[0]}, S2 = {game.s[1]}')
logging.info(f'Results P1: {result_1} P2: {result_2}')
condition = equal(result_1[0], result_1[1]) and equal(
result_1[1], result_1[2])
condition = condition and equal(result_2[0], result_2[1]) and equal(
result_2[1], result_2[2])
assert condition, 'Non zero sum game MSNE failed'
def main():
try: q = unicodedata.normalize('NFC', unicode(sys.argv[1].strip()))
except: q = ""
(title, area, tag, day, note) = util.parse(q[1:])
display = title.strip()+' '
if area != 'Inbox':
display += '@'+area+' '
if day:
display += '> ' + day.strftime('%m/%d %a')
subtitle = "title #tag @t|n|s ::note >duedate (ex. fri, 3d, 2w, 12/31)"
help = {"valid": False, "subtitle": subtitle}
output = []
output.append({"title": display, "subtitle": subtitle, "arg":q, "valid": True,
"mods": { "alt": help, "ctrl":help, "cmd":help, "shift":help}})
print json.dumps({"items": output})
def __init__(self,train_data,val_label,test_data):
self.val_set = set()
for e in util.parse(val_label):
self.val_set.add((int(e['user_id']),int(e['item_id'])))
self.train_data = sp.genfromtxt(train_data,delimiter=',',skip_header=1)
self.val_data = sp.genfromtxt(test_data,delimiter=',',skip_header=1)
feature_should_scale = [2,3,4,5,8,11,19,20,21,22,24,27,28,29,30]
for i in feature_should_scale:
self.train_data[:,i] = preprocessing.scale(self.train_data[:,i])
self.val_data[:,i] = preprocessing.scale(self.val_data[:,i])
self.train_x = self.train_data[:,2:-2]
self.train_y = self.train_data[:,-1]
self.val_x = self.val_data[:,2:-2]
self.val_y = self.val_data[:,-1]
self.n_features = self.train_x.shape[1]
self.neg_train_data = self.train_data[self.train_y==0]
self.pos_train_data = self.train_data[self.train_y==1]
self.user_geo,self.item_geo = util.ex_user_item_geohash('tianchi_mobile_recommend_train_user.csv','tianchi_mobile_recommend_train_item.csv')
print "Total <user,item> pair: train = %d test = %d"%(len(self.train_data),len(self.val_data))
def _insert_loop(self):
curses.noecho()
curses.cbreak()
line = ""
self.stdscr.move(self.scr_y-1,0)
while self.mode == 0:
ch = self.stdscr.getch()
# KEY_Enter
if ch == 10 :
self.stdscr.clear()
self.stdscr.refresh()
self.content.append(line)
line = ""
self.draw_screen()
self._draw_line()
self.stdscr.move(self.scr_y-1,0)
# KEY_Esc
elif ch == 27:
self.mode = 1
line = ''
self.stdscr.move(self.scr_y-1,0)
self.stdscr.clrtoeol()
self.stdscr.refresh()
# KEY_Backspace
elif ch == 127:
y,x = curses.getsyx()
line = line[:-1]
self.draw_inputArea(line, y, x-1)
else:
if ch < 256:
string = chr(ch)
else:
string = parse(ch)
self.stdscr.addstr(string)
line += string
self._normal_loop()
As I mentioned yesterday in Chapter, we are founding the Undergraduate Endowment Committee (UEC) this semester. With a strong foundation in place, the DSP Endowment will benefit not only the active Brotherhood, but also our experience as alumni down the road. If you are interesting in taking part in our Fraternity's first steps towards an Endowment, please take several minutes to fill out the attached UEC Application.
As co-chairs, Jed and I are looking for 5-7 committee members who are as excited as we are about the Endowment. The deadline for the application is Tuesday October 8th at 6:00pm.
I'm looking forward to hearing your ideas within the application.
Fraternally,
Don
""")
s.append("""
Hey ctas!
It appears that a number of us are busy with prelims and other obligations so we have decided to cancel the CTAS BBQ/Picnic.
Instead, please meet us in RPCC lobby at 11am on Sunday, September 29th for Brunch. We hope to see you there!
Best,
Rick Jean
Cornell University | Class of 2014
B.Sc. Applied Economics and Management
Mobile: 646-648-2503
Email: [email protected]
""")
for string in s:
print util.parse(string)
# import pdb; pdb.set_trace()
print util.parse(s[6])
#-*- coding:utf-8 -*-
import numpy as np
import scipy as sp
import util
import feature
from sklearn import preprocessing
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from datetime import *
item_subset = set()
for e in util.parse('tianchi_mobile_recommend_train_item.csv'):
item_subset.add(e['item_id'])
# oneday feature offline test
train_file = 'oneday_feature1217.csv'
val_file = 'oneday_feature1218.csv'
train_data = sp.genfromtxt(train_file,delimiter=',',skip_header=1)
val_data = sp.genfromtxt(val_file,delimiter=',',skip_header=1)
fs = [2,3,4,5]
for i in fs:
train_data[:,i] = preprocessing.scale(train_data[:,i])
val_data[:,i] = preprocessing.scale(val_data[:,i])
train_x = train_data[:,2:-1]
def setup(path):
v, e = parse(path)
g = Graph(v, e)
f = FlowNetwork(g)
return f
def execute(args):
print 'Starting the artificial neural network'
if len(args) < 2:
usage()
sys.exit()
###############################################################################
# Data
# names feature labels
# y shuffled names
# x features that correspond to shuffled names
names, y, x = parse(args[1])
x = clean(names, x)
usePowerset = args[0]
# Build features to include in test
features = args[2:]
if len(features) == 0:
features = names
# print 'Selected features:', features
# Build all subsets of features, if requested
if usePowerset.lower() == 'true':
combos = powerset(features)
else:
combos = [features]
# map from feature set, to map of correct counts for each person
feature_performance = {}
highest_correct = 0
best_combo = {}
for c in combos:
if len(c) == 0:
continue
print 'Attempting feature set:', c
x_selected = selectFeatures(copy.copy(names), c, x)
# Split into testing and traiing data
x_train, x_test, y_train, y_test = train_test_split(x_selected, y,
test_size=0.2,
random_state=0)
###############################################################################
# Models
logistic = linear_model.LogisticRegression(C=L_REGULARIZATION)
rbm = BernoulliRBM(random_state=0, verbose=True, learning_rate=N_LEARNING_RATE, n_iter=N_ITER, n_components=N_COMPONENTS)
# Note: attempted StandardScaler, MinMaxScaler, MaxAbsScaler, without strong results
# Not needed, since data is scaled to the [0-1] range by clean()
classifier = Pipeline(steps=[('rbm', rbm),('logistic', logistic)])
# ###############################################################################
# Training
print 'Training the classifier...'
# Training RBM-Logistic Pipeline
classifier.fit(x_train,y_train)
correct = 0
label_counts = defaultdict(int)
for i in range(len(x_test)):
test = x_test[i]
if len(test) == 1:
test = test.reshape(-1, 1)
else:
test = [test]
predicted = classifier.predict(test)
if predicted == y_test[i]:
correct += 1
label_counts[predicted[0]] += 1
if correct >= highest_correct:
highest_correct = correct
best_combo = c
feature_performance[str(c)] = {'predictions':label_counts,'expected':Counter(y_test)}
###############################################################################
# Evaluation
# evaluate(classifier, x_test, y_test)
summary = feature_performance[str(best_combo)]
print 'Accuracy:\t\t\t', highest_correct, 'correct gives', (highest_correct * 1.0/len(y_test)), 'compared to guessing', (1.0/len(summary['expected']))
print 'Best feature set:\t\t', best_combo
print 'Identified %d out of %d labels'%(len(summary['predictions']),len(summary['expected']))
for p in summary['predictions']:
pred = summary['predictions'][p]
tot = summary['expected'][p]
print '\t %s \t\t %d\t of %d \t (%f)'%(p, pred, tot, pred * 1.0/tot)
def execute(args):
print 'Starting the gridsearch on the artificial neural network'
if len(args) < 1:
usage()
sys.exit()
###############################################################################
# Data
# names feature labels
# y shuffled names
# x features that correspond to shuffled names
names, y, x = parse(args[0])
x = clean(names, x)
# Build features to include in test
features = args[1:]
if len(features) == 0:
features = names
print 'Selected features:', features
x = selectFeatures(names, features, x)
# Split into testing and traiing data
x_train, x_test, y_train, y_test = train_test_split(x, y,
test_size=0.2,
random_state=0)
###############################################################################
# Models
logistic = LogisticRegression()
rbm = BernoulliRBM(random_state=0, verbose=True)
classifier = Pipeline(steps=[('rbm', rbm),('logistic', logistic)])
# ###############################################################################
# Training
print 'Training the classifier...'
# Training RBM-Logistic Pipeline
# classifier.fit(x_train, y_train)
# Training Logistic regression
# logistic_classifier = LogisticRegression(C=100.0)
# logistic_classifier.fit(x_train, y_train)
# evaluate(classifier, logistic_classifier, x_test, y_test)
###############################################################################
# Evaluation
scores = ['precision', 'recall']
for score in scores:
print("# Tuning hyper-parameters for %s" % score)
print()
clf = GridSearchCV(classifier, param_grid=param_grid, cv=3, scoring='%s_weighted' % score)
clf.fit(x_train, y_train)
print("The best parameters are %s with a score of %0.2f"
% (clf.best_params_, clf.best_score_))
print()
print("Grid scores on development set:")
print()
for params, mean_score, scores in clf.grid_scores_:
print("%0.3f (+/-%0.03f) for %r"
% (mean_score, scores.std() * 2, params))
print()
print("Detailed classification report:")
print()
print("The model is trained on the full development set.")
print("The scores are computed on the full evaluation set.")
print()
y_true, y_pred = y_test, clf.predict(x_test)
print(classification_report(y_true, y_pred))
print()
def write(self, bytes):
self.trace("raw", "RECV: %r", bytes)
self.input.write(bytes)
self.state = parse(self.state)
