def iou_calc(self, boxes1, boxes2): #x, y, w, h
boxes1 = torch.stack([
boxes1[:, :, :, :, 0] - boxes1[:, :, :, :, 2] / 2.0,
boxes1[:, :, :, :, 1] - boxes1[:, :, :, :, 3] / 2.0,
boxes1[:, :, :, :, 0] + boxes1[:, :, :, :, 2] / 2.0,
boxes1[:, :, :, :, 1] + boxes1[:, :, :, :, 3] / 2.0
])
boxes1 = boxes1.permute(1, 2, 3, 4, 0)
boxes2 = torch.stack([
boxes2[:, :, :, :, 0] - boxes2[:, :, :, :, 2] / 2.0,
boxes2[:, :, :, :, 1] - boxes2[:, :, :, :, 3] / 2.0,
boxes2[:, :, :, :, 0] + boxes2[:, :, :, :, 2] / 2.0,
boxes2[:, :, :, :, 1] + boxes2[:, :, :, :, 3] / 2.0
])
boxes2 = boxes2.permute(1, 2, 3, 4, 0)
# print('boxes1')
# print(boxes1)
# print('boxes2')
# print (boxes2)
# calculate the left up point & right down point
lu = torch.max(boxes1[:, :, :, :, :2], boxes2[:, :, :, :, :2])
rb = torch.min(boxes1[:, :, :, :, 2:], boxes2[:, :, :, :, 2:])
# print('lu')
# print(lu)
# print('rb')
# print(rb)
# intersection
intersection = torch.max((rb - lu), Variable(torch.zeros(rb.size())))
inter_square = intersection[:, :, :, :, 0] * intersection[:, :, :, :,
1]
# print('intersection')
# print(intersection)
# print('inter_square')
# print(inter_square)
square1 = (boxes1[:, :, :, :, 2] - boxes1[:, :, :, :, 0]) * \
(boxes1[:, :, :, :, 3] - boxes1[:, :, :, :, 1])
square2 = (boxes2[:, :, :, :, 2] - boxes2[:, :, :, :, 0]) * \
(boxes2[:, :, :, :, 3] - boxes2[:, :, :, :, 1])
square1 = torch.clamp(square1, 0.00001, 112 * 112)
square2 = torch.clamp(square2, 0.00001, 112 * 112)
# print('square1')
# print(square1)
# print('square2')
# print(square2)
union_square = square1 + square2 - inter_square
# print('uniou_square')
# print(union_square)
# print('iou')
# print(inter_square / union_square)
result = inter_square / union_square
# for i0 in range(100):
# for i1 in range(7):
# for i2 in range(7):
# for i3 in range(2):
# if math.isnan(result[i0,i1,i2,i3].data.cpu().numpy()):
# Tracer()()
if math.isnan(torch.sum(result).data.cpu().numpy()):
Tracer()()
return result #shape = (batch_size, 7, 7, 2)
import cv2
import numpy as np
from IPython.core.debugger import Tracer
keyboard = Tracer()
from scipy.interpolate import UnivariateSpline
def create_LUT_8UC1(x, y):
spl = UnivariateSpline(x, y, k=2)
return spl(xrange(256))
def _get_images_from_batches(batch):
batch_size = batch.shape[0]
img_width = batch.shape[1]
img_height = batch.shape[2]
img_channel = batch.shape[3]
imgs = np.split(batch, batch_size)
reshaped_imgs = []
for img in imgs:
img = img.reshape(img_width, img_height, img_channel)
reshaped_imgs.append(img)
return reshaped_imgs, img_width, img_height, img_channel
def trans2uint(batch):
batch = np.interp(batch, [0, 1], [0, 255])
batch = np.ndarray.astype(batch, 'uint8')
return batch
from arsenal.debug import saverr # registers hook
from IPython import embed as ip
#from IPython.frontend.terminal.embed import InteractiveShellEmbed
#_ip = InteractiveShellEmbed(banner1='')
def enable_ultratb(mode='Context', **kwargs):
from IPython.core import ultratb
sys.excepthook = ultratb.FormattedTB(mode=mode, **kwargs)
# TODO: look IPython's debugging stuff..
# http://ipython.org/ipython-doc/dev/api/generated/IPython.core.debugger.html
from IPython.core.debugger import Tracer
set_trace = lambda: Tracer()()
#from IPython.Debugger import Pdb
from pdb import set_trace, pm, Pdb
def enable_debug_hook():
"Register pdb's post-mortem debugger as the handler for uncaught exceptions."
def debug_hook(*args):
sys.__excepthook__(*args)
pm()
sys.excepthook = debug_hook
import pandas as pd
from exceptions import WrongDataFrameSize, NoScanAvailable
from IPython.core.debugger import Tracer
debughere = Tracer()
import json
def extract_scan(df, ip="", protocol='tcp'):
no_scan = []
if ip:
scan = df[df["ip"] == ip][protocol].iloc[0]
else:
if not isinstance(df, pd.core.series.Series):
raise WrongDataFrameSize
pass
else:
scan = df[protocol]
scan_df = pd.DataFrame(columns=[
'conf', 'cpe', 'extrainfo', 'name', 'port', 'product', 'reason',
'state', 'version'
])
if isinstance(scan, str):
scan = json.loads(scan)
scan = [dict(scan[x], **{"port": x}) for x in list(scan.keys())]
scan_df = scan_df.append(scan[0], ignore_index=True)
if len(scan) > 1:
scan_df = scan_df.append(scan[1:], ignore_index=True)
scan_df.insert(0, "ip", ip, allow_duplicates=True)
return scan_df
else:
#!/usr/bin/python
#-------------------------------------------------------------------------------
#License GPL v3.0
#Author: Alexandre Manhaes Savio <*****@*****.**>
#Grupo de Inteligencia Computational <www.ehu.es/ccwintco>
#Universidad del Pais Vasco UPV/EHU
#Use this at your own risk!
#-------------------------------------------------------------------------------
from IPython.core.debugger import Tracer; debug_here = Tracer()
import os, sys, argparse
import numpy as np
import nibabel as nib
import aizkolari_utils as au
def set_parser():
parser = argparse.ArgumentParser(description='Save thresholded NifTi volumes computed from the given volume.')
parser.add_argument('-i', '--input', dest='input', required=True,
help='list of files to be thresholded. If it is a list, it must go within quotes.')
parser.add_argument('-o', '--outdir', dest='outdir', required=False, default = '',
help='name of the output directory where the results will be saved. If not given, will put aside the correspondent input file.')
parser.add_argument('-t', '--thresholds', dest='threshs', required=False, default='95'
help='list of floats within [0,100] separated by blank space. If it is a list, it must go within quotes')
parser.add_argument('-m', '--mask', default='', dest='mask', required=False,
help='Mask file.')
parser.add_argument('-e', '--extension', default='.nii.gz', dest='ext', required=False,
help='Output files extension.')
parser.add_argument('-a', '--absolute', default=False, action='append', dest='abs', required=False,
def create_db():
global success
global conn
global db_path
global MRSTY_TABLE_FILE
global MRCON_TABLE_FILE
global MRREL_TABLE_FILE
global LRABR_TABLE_FILE
print("\ncreating umls.db")
# connect to the .db file we are creating.
db_path = os.path.join(umls_tables, 'umls.db')
conn = sqlite3.connect(db_path)
conn.text_factory = str
print("opening files")
# load data in files.
try:
mrsty_path = os.path.join(umls_tables, 'MRSTY.RRF')
MRSTY_TABLE_FILE = open(mrsty_path, "r")
except IOError:
print("\nNo file to use for creating MRSTY.RRF table\n")
sys.exit()
try:
mrcon_path = os.path.join(umls_tables, 'MRCONSO.RRF')
MRCON_TABLE_FILE = open(mrcon_path, "r")
except IOError:
print("\nNo file to use for creating MRCONSO.RRF table\n")
sys.exit()
try:
mrrel_path = os.path.join(umls_tables, 'MRREL.RRF')
MRREL_TABLE_FILE = open(mrrel_path, "r")
except IOError:
print("\nNo file to use for creating MRREL.RRF table\n")
sys.exit()
try:
lrabr_path = os.path.join(umls_tables, "..", "LEX", 'LRABR')
from IPython.core.debugger import Tracer # NOQA
Tracer()()
LRABR_TABLE_FILE = open(lrabr_path, "r")
except IOError:
print("\nNo file to use for creating LRABR table\n")
sys.exit()
print("creating tables")
c = conn.cursor()
# create tables.
c.execute("CREATE TABLE MRSTY( CUI, TUI, STN, STY, ATUI, CVF ) ;")
c.execute("CREATE TABLE MRCON( CUI, LAT, TS, LUI, STT, SUI, ISPREF, AUI, SAUI, SCUI, SDUI, SAB, TTY, CODE, STR, SRL, SUPPRESS, CVF ) ;")
c.execute("CREATE TABLE MRREL( CUI1, AUI1, STYPE1, REL, CUI2, AUI2, STYPE2, RELA, RUI, SRUI, SAB, SL, RG, DIR, SUPPRESS, CVF );")
c.execute("CREATE TABLE LRABR( EUI1, ABR, TYPE, EUI2, STR);")
print("inserting data into MRSTY table")
for line in MRSTY_TABLE_FILE:
line = line.strip('\n')
assert line[-1] == '|', "str: {}, char: ".format(line, line[-1])
line = line.split('|')
# end will always be empty str
line.pop()
assert len(line) == 6
c.execute(
"INSERT INTO MRSTY( CUI, TUI, STN, STY, ATUI, CVF ) values( ?, ?, ?, ?, ?, ?)", tuple(line))
print("inserting data into MRCON table")
for line in MRCON_TABLE_FILE:
line = line.strip('\n')
assert line[-1] == '|', "str: {}, char: ".format(line, line[-1])
line = line.split('|')
# end will always be empty str
line.pop()
assert len(line) == 18
c.execute("INSERT INTO MRCON( CUI, LAT, TS, LUI, STT, SUI, ISPREF, AUI, SAUI, SCUI, SDUI, SAB, TTY, CODE, STR, SRL, SUPPRESS, CVF ) values ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);", tuple(line))
print("inserting data into MRREL table")
for line in MRREL_TABLE_FILE:
line = line.strip('\n')
assert line[-1] == '|', "str: {}, char: ".format(line, line[-1])
line = line.split('|')
# end will always be empty str
line.pop()
assert len(line) == 16
c.execute("INSERT INTO MRREL( CUI1, AUI1, STYPE1, REL, CUI2, AUI2, STYPE2, RELA, RUI, SRUI, SAB, SL, RG, DIR, SUPPRESS, CVF ) values( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )", tuple(line))
print("inserting into LRABR table")
for line in LRABR_TABLE_FILE:
line = line.strip('\n')
assert line[-1] == '|', "str: {}, char: ".format(line, line[-1])
line = line.split('|')
line.pop()
assert len(line) == 5
c.execute(
"INSERT INTO LRABR( EUI1, ABR, TYPE, EUI2, STR) values( ?, ?, ?, ?,?)", tuple(line))
print("creating indices")
# create indices for faster queries
c.execute("CREATE INDEX mrsty_cui_map ON MRSTY(CUI)")
c.execute("CREATE INDEX mrcon_str_map ON MRCON(STR)")
c.execute("CREATE INDEX mrcon_cui_map ON MRCON(CUI)")
c.execute("CREATE INDEX mrrel_cui2_map ON MRREL( CUI2 )")
c.execute("CREATE INDEX mrrel_cui1_map on MRREL( CUI1 ) ")
c.execute("CREATE INDEX mrrel_rel_map on MRREL( REL )")
c.execute("CREATE INDEX lrabr_abr_map on LRABR(ABR)")
c.execute("CREATE INDEX lrabr_str_map on LRABR(STR)")
# save changes to .db
conn.commit()
success = True
print("\nsqlite database created")
def backtest(filename, data, schedule_data):
f = open(filename)
f.readline()
player_data = {}
time_data = []
for i in xrange(50):
line = f.readline()
if line is None or len(line) == 0:
break
date = int(line[:3])
print date
jsonvalue = "{"+f.readline()+"}"
value = json.loads(jsonvalue)
time_data.insert(0,(date,value))
for p in value:
if not p in player_data:
player_data[p] = [0]
player_data[p].insert(0,value[p])
time_data2 = convertToPlayersTimeData(time_data, data)
teams = set([i.team for i in data])
for i in xrange(len(time_data2)):
stamp_data = time_data2[i][1]
Tracer()()
portfolio = ["rohit sharma", "ajinkya rahane", "david warner", "glenn maxwell", "robin uthappa", "shane watson", "sandeep sharma", "sunil narine", "pravin tambe", "yuzvendra chahal", "bhuvneshwar kumar"]
# portfolio = ["yuzvendra chahal", "shakib al hasan", "shane watson", "rohit sharma", "sandeep sharma", "sunil narine", "ajinkya rahane", "jacques kallis", "robin uthappa", "jayant yadav","bhuvneshwar kumar"]
# portfolio = ["manish pandey", "rohit sharma","jacques kallis","robin uthappa", "aditya tare", "ambati rayudu", "morne morkel","piyush chawla","sunil narine","lasith malinga","pragyan ojha"]
power_player = "glenn maxwell"
# power_player = "bhuvneshwar kumar"
portfolio_p = set([getPlayer(data, p)[0] for p in portfolio])
power_player_p = getPlayer(data, power_player)[0]
points = 0
subs = 75
mlab = Matlab(matlab='/Applications/MATLAB_R2013a.app/bin/matlab')
mlab.start()
for i in xrange(4,len(time_data2)):
# START = str(time_data2[i][0])
# CURRENT_TEAM = set(portfolio)
# SUBSTITUTIONS = subs
# PAST_STATS = time_data[i][1]
print "\n\n\n\n\n\n"
print (subs, str(time_data2[i][0]))
print set(portfolio_p)
print points
print "\n\n\n\n\n\n"
# print time_data[i-1][1]
# Tracer()()
inp = (subs, str(time_data2[i][0]), set(portfolio), time_data[i-1][1])
backtest_pickteam.pickTeam(data, schedule_data, inp)
res = mlab.run_func('/Users/deedy/Dev/FantasyIPL-Moneyball/python2matlab.m', {}, maxtime = 500)
changes = backtest_results.getResults(data, schedule_data, res['result'])
subs -= changes[2]
portfolio_p = changes[0]
power_player_p = changes[1]
# Tracer()()
# update portfolio
# update subs
# update power player
# Tracer()()
teams = [(p,time_data2[i][1][p] - time_data2[i-1][1][p] ) for p in time_data2[i][1] if p in portfolio_p]
print teams
pthis = 0
for i in teams:
if power_player_p == i[0]:
pthis += 2*i[1]
else:
pthis += i[1]
points+= pthis
print "{0}\t{1}\t{2}\n\n".format(points, pthis, subs)
# print "{0}\t{1}".format(time_data2[i][0] , teams)
mlab.stop()
Tracer()()
f.close()
def setUpAndSaveVars(players, scores, schedule):
numplayers = len(players)
# A and B
team_total_constraint = [1.0] * numplayers
wicketkeeper_constraint = [float(player.is_keeper) for player in players]
A = matrix([team_total_constraint, wicketkeeper_constraint])
A = A.trans()
B = matrix([11.0, 1.0])
# G and H
all_unstrict_constraints = []
all_unstrict_constraints_limits = []
teams = set([player.team for player in players])
teams = list(teams)
for team in teams:
all_unstrict_constraints.append(
[float(player.team == team) for player in players])
all_unstrict_constraints_limits.append(6.0)
batsman_constraint = [-float(player.is_batsman) for player in players]
allrounder_constraint = [
-float(player.is_allrounder) for player in players
]
uncapped_constraint = [-float(player.is_uncapped) for player in players]
bowler_constraint = [-float(player.is_bowler) for player in players]
bowling_constraint = [
-float(player.is_bowler or player.is_allrounder) for player in players
]
overseas_constraint = [float(player.is_overseas) for player in players]
price_constraint = [float(player.price) / 1000000.0 for player in players]
all_unstrict_constraints.extend([
batsman_constraint, allrounder_constraint, uncapped_constraint,
bowler_constraint, bowling_constraint, overseas_constraint,
price_constraint
])
all_unstrict_constraints_limits.extend(
[-4.0, -1.0, -1.0, -2.0, -5.0, 4.0, 10.0])
G = matrix(all_unstrict_constraints)
G = G.trans()
H = matrix(all_unstrict_constraints_limits)
# P and Q
Q = [-score for score in scores]
# find_naive_one_time_best_team(Q,A,B,G,H)
# Tracer()()
Q_new = []
days = len(schedule["schedule"])
Tracer()()
start_index = schedule["schedule"].index(
[x for x in schedule["schedule"] if x[0] == START][0])
past_games = {}
for t in teams:
past_games[t] = 0
for day in schedule["schedule"][:start_index]:
for t in day[1]:
past_games[t] += 1
## PAST SQUADS
curr_team_indices = []
for player_name in PAST_STATS:
hscore = PAST_STATS[player_name]
player_arr = getPlayer(players, player_name)
if len(player_arr) > 1:
print "Too many people with name {0}".format(player_name)
Tracer()()
elif len(player_arr) == 0:
print "Name not found - {0}".format(player_name)
Tracer()()
else:
player = player_arr[0]
player_ind = players.index(player)
if past_games[player.team] > 0:
# print player
# print Q[player_ind]
weight_to_ipl = 0.2 + 0.8 * (past_games[player.team] / 14.0)
weight_to_ipl = 1
Q[player_ind] = Q[player_ind] * (
1 - weight_to_ipl) + weight_to_ipl * (
-hscore) / past_games[player.team]
if hscore == 0:
Q[player_ind] *= 0.5
# print Q[player_ind]
# print "\n\n"
# Tracer()()
scores = [-x for x in Q]
##
print sorted(zip(players, Q), key=lambda x: -x[1])
Tracer()()
days = len(schedule["schedule"][start_index:])
for day in schedule["schedule"][start_index:]:
match_mask = [0] * len(all_unstrict_constraints[0])
for playing_team in day[1]:
match_mask = [
sum(x) for x in zip(
match_mask, all_unstrict_constraints[teams.index(
playing_team)])
]
# Tracer()()
##### Kevin Pietersen ruled out for 417
if day[0] == '417':
ind = players.index(getPlayer(players, "kevin pietersen")[0])
match_mask[ind] = 0
####
Q_new.extend([a * b for a, b in zip(Q, match_mask)])
Q_new = matrix(Q_new).trans()
#### Current Team
curr_team_indices = []
for player_name in CURRENT_TEAM:
player_arr = getPlayer(players, player_name)
if len(player_arr) > 1:
print "Too many people with name {0}".format(player_name)
Tracer()()
else:
curr_team_indices.append(players.index(player_arr[0]))
curr_team = [0] * numplayers
for x in curr_team_indices:
curr_team[x] = 1
####
Tracer()()
save_filename = 'player-optimization-data.mat'
print "Saving variables as {0}".format('player-optimization-data.mat')
sio.savemat(
save_filename, {
'A': A,
'B': B,
'G': G,
'H': H,
'Q': Q,
'Q_new': Q_new,
'days': days,
'curr_team': curr_team,
'numplayers': numplayers,
'substitutions': SUBSTITUTIONS,
'scores': Q,
'start_index': start_index
})
def scorePlayerAlgo2(player):
if not hasattr(player, "tournamentStats"):
return -1, -1, -1, -1
if len(player.tournamentStats) == 0:
# Tracer()()
return -1, -1, -1, -1
allfieldingstats = [t[1]['fieldingStats'] for t in player.tournamentStats]
compiled_fielding_stats = {}
for afs in allfieldingstats[0].viewkeys():
compiled_fielding_stats[afs] = []
for fs in allfieldingstats:
for afs in fs.viewkeys():
compiled_fielding_stats[afs].append(float(fs[afs]))
allbowlingstats = [t[1]['bowlingStats'] for t in player.tournamentStats]
compiled_bowling_stats = {}
for afs in allbowlingstats[0].viewkeys():
compiled_bowling_stats[afs] = []
for fs in allbowlingstats:
for afs in fs.viewkeys():
try:
if not fs[afs] == '-':
compiled_bowling_stats[afs].append(float(fs[afs]))
else:
compiled_bowling_stats[afs].append(0.0)
except:
Tracer()()
allbattingstats = [t[1]['battingStats'] for t in player.tournamentStats]
compiled_batting_stats = {
'a': [],
'b': [],
'inns': [],
'no': [],
'sr': [],
'm': [],
'4s': [],
'6s': [],
'hs': [],
'r': [],
'100s': [],
'50s': []
}
for afs in allbattingstats[0].viewkeys():
compiled_batting_stats[afs] = []
for fs in allbattingstats:
for afs in fs.viewkeys():
try:
if not afs in compiled_batting_stats:
compiled_batting_stats[afs].append(0.0)
if type(fs[afs]) is int:
compiled_batting_stats[afs].append(float(fs[afs]))
elif fs[afs][-1] == '*':
compiled_batting_stats[afs].append(float(fs[afs][:-1]))
elif not fs[afs] == '-':
compiled_batting_stats[afs].append(float(fs[afs]))
except Exception as e:
Tracer()()
stats = {}
stats['bowlingStats'] = {}
stats['bowlingStats']['r'] = sum(compiled_bowling_stats['r'])
stats['bowlingStats']['inns'] = sum(compiled_bowling_stats['inns'])
stats['bowlingStats']['10w'] = sum(compiled_bowling_stats['10w'])
stats['bowlingStats']['nb'] = sum(compiled_bowling_stats['nb'])
stats['bowlingStats']['wb'] = sum(compiled_bowling_stats['wb'])
stats['bowlingStats']['maid'] = sum(compiled_bowling_stats['maid'])
stats['bowlingStats']['4w'] = sum(compiled_bowling_stats['4w'])
stats['bowlingStats']['5w'] = sum(compiled_bowling_stats['5w'])
stats['bowlingStats']['b'] = sum(compiled_bowling_stats['b'])
stats['bowlingStats']['wmaid'] = sum(compiled_bowling_stats['wmaid'])
stats['bowlingStats']['m'] = sum(compiled_bowling_stats['m'])
stats['bowlingStats']['6s'] = sum(compiled_bowling_stats['6s'])
stats['bowlingStats']['4s'] = sum(compiled_bowling_stats['4s'])
stats['bowlingStats']['ov'] = sum(compiled_bowling_stats['ov'])
stats['bowlingStats']['w'] = sum(compiled_bowling_stats['w'])
stats['bowlingStats']['d'] = sum(compiled_bowling_stats['d'])
stats['bowlingStats'][
'e'] = 6 * stats['bowlingStats']['r'] / stats['bowlingStats'][
'b'] if stats['bowlingStats']['b'] != 0 else sys.float_info.max
stats['bowlingStats'][
'sr'] = 100 * stats['bowlingStats']['w'] / stats['bowlingStats'][
'b'] if stats['bowlingStats']['b'] != 0 else sys.float_info.max
stats['bowlingStats'][
'a'] = stats['bowlingStats']['r'] / stats['bowlingStats'][
'w'] if stats['bowlingStats']['w'] != 0 else sys.float_info.max
stats['bowlingStats']['bbmw'] = max(compiled_bowling_stats['bbmw'])
stats['bowlingStats']['bbiw'] = max(compiled_bowling_stats['bbiw'])
tmpind = compiled_bowling_stats['bbmr'].index(
min([
compiled_bowling_stats['bbmr'][j] for j in [
i for i, val in enumerate(compiled_bowling_stats['bbmw'])
if val == stats['bowlingStats']['bbmw']
]
]))
stats['bowlingStats']['bbmr'] = compiled_bowling_stats['bbmr'][tmpind]
tmpind = compiled_bowling_stats['bbir'].index(
min([
compiled_bowling_stats['bbir'][j] for j in [
i for i, val in enumerate(compiled_bowling_stats['bbiw'])
if val == stats['bowlingStats']['bbiw']
]
]))
stats['bowlingStats']['bbir'] = compiled_bowling_stats['bbir'][tmpind]
stats['battingStats'] = {}
stats['battingStats']['r'] = sum(compiled_batting_stats['r'])
stats['battingStats']['inns'] = sum(compiled_batting_stats['inns'])
stats['battingStats']['no'] = sum(compiled_batting_stats['no'])
stats['battingStats']['b'] = sum(compiled_batting_stats['b'])
stats['battingStats']['m'] = sum(compiled_batting_stats['m'])
stats['battingStats']['100s'] = sum(compiled_batting_stats['100s'])
stats['battingStats']['50s'] = sum(compiled_batting_stats['50s'])
stats['battingStats']['4s'] = sum(compiled_batting_stats['4s'])
stats['battingStats']['6s'] = sum(compiled_batting_stats['6s'])
stats['battingStats']['hs'] = max(compiled_batting_stats['hs'])
stats['battingStats']['a'] = stats['battingStats']['r'] / (
stats['battingStats']['inns'] - stats['battingStats']['no']) if (
stats['battingStats']['inns'] -
stats['battingStats']['no']) != 0 else sys.float_info.max
stats['battingStats'][
'sr'] = 100 * stats['battingStats']['r'] / stats['battingStats'][
'b'] if stats['battingStats']['b'] else sys.float_info.max
stats['fieldingStats'] = {}
stats['fieldingStats']['m'] = sum(compiled_fielding_stats['m'])
stats['fieldingStats']['inns'] = sum(compiled_fielding_stats['inns'])
stats['fieldingStats']['c'] = sum(compiled_fielding_stats['c'])
stats['fieldingStats']['ro'] = sum(compiled_fielding_stats['ro'])
stats['fieldingStats']['s'] = sum(compiled_fielding_stats['s'])
# Tracer()()
battingpoints = 0
bowlingpoints = 0
fieldingpoints = 0
# Batting
battingstats = stats['battingStats']
innings = float(battingstats['inns'])
matches = float(battingstats['m'])
if not innings == 0:
expected_runs = int(battingstats['r']) / innings
try:
strike_rate = float(battingstats['sr'])
except:
strike_rate = 0
sixes = int(battingstats['6s'])
battingpoints = expected_runs + expected_runs * (
strike_rate / 100) + 10 * math.floor(
expected_runs / 25) + 2 * (sixes / innings)
battingpoints = battingpoints
#Bowling
bowlingstats = stats['bowlingStats']
innings = float(bowlingstats['inns'])
matches = float(bowlingstats['m'])
if not innings == 0:
wickets = int(bowlingstats['w'])
balls = int(bowlingstats['b'])
maidens = int(bowlingstats['maid'])
dots = int(bowlingstats['d'])
runs_given = int(bowlingstats['r'])
econscore = (1.5 * balls - runs_given) / innings
if econscore > 0:
econscore *= 2
bowlingpoints = econscore + 20 * (wickets / innings) + 10 * math.floor(
(wickets / innings) / 2) + 20 * (maidens / innings) + (dots /
innings)
bowlingpoints = bowlingpoints
#Fielding
fieldingstats = stats['fieldingStats']
matches = float(fieldingstats['m'])
catches = int(fieldingstats['c'])
stumpings = int(fieldingstats['s'])
runouts = int(fieldingstats['ro'])
fieldingpoints = 10 * (catches / matches) + 15 * (
stumpings / matches) + 12.5 * (runouts / matches)
# if "South" in player.name:
# Tracer()()
if matches < 5:
battingpoints /= (5 - matches)
bowlingpoints /= (5 - matches)
fieldingpoints /= (5 - matches)
totalpoints = battingpoints + bowlingpoints + fieldingpoints
# print "{0}\t\tBat: {1}\tBowl: {2}\tField: {3}\tTotal:{4}".format(player, battingpoints, bowlingpoints, fieldingpoints, totalpoints)
return totalpoints, battingpoints, bowlingpoints, fieldingpoints
def parse(self, response):
# Random string is used to construct the XHR sent to twitter.com
random_str = "BD1UO2FFu9QAAAAAAAAETAAAAAcAAAASAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
data = json.loads(response.body_as_unicode())
#default rate delay is 12s
# rate_delay = self.settings['DOWNLOAD_DELAY']
# rate_delay = 2
# delay_choices = [(1,30), (2,25), (3,20),(4,15),(5,10)]
# delay_choices = [(1,50), (2,30), (3,10),(4,8),(5,2)]
# delay_choices = [(0,1),(1,89), (2,4), (3,3),(4,2),(5,1)]
# delay_choices = [(1,60), (2,20), (3,10),(4,8),(5,2)]
# delay_choices = [(0,33),(1,56), (2,5), (3,3),(4,2),(5,1)]
# if data["max_position"] is not None:
try:
if data['items_html'] is not None:
tweets = self.extract_tweets(data['items_html'])
# If we have no tweets, then we can break the loop early
if len(tweets) == 0 and data['has_more_items'] is False:
Tracer()()
pprint(data)
logging.log(logging.DEBUG, data)
logging.log(
logging.INFO, "Reach the end of search results( " +
self.query + " )")
return
# yield Request(url=next_url, callback=self.parse)
for tweet in tweets:
# push parsed item to mongoDB pipline
yield self.parse_tweet(tweet, response)
# If we haven't set our min tweet yet, set it now
if self.min_tweet is None:
self.is_first_query = True
self.min_tweet = tweets[0]
elif self.min_tweet is not tweets[0]:
self.min_tweet = tweets[0]
# continue_search = self.save_tweets(tweets)
# The max tweet is the last tweet in the list
self.max_tweet = tweets[-1]
if self.min_tweet['tweet_id'] is not self.max_tweet[
'tweet_id']:
self.max_position = "TWEET-%s-%s-%s" % (
self.max_tweet['tweet_id'], self.min_tweet['tweet_id'],
random_str)
# '''
# is_first_query is a indicator used to identify the intial query. With the intial query
# the crwaler can simulate the hand-shake request while the delay time is greater than a
# predefined time period, for instance, 22 seconds
# '''
# if self.is_first_query:
# self.data_max_position = self.max_position
# self.is_first_query = False
# Construct next url to crawl
next_url = self.construct_url(
self.query,
max_position=self.max_position,
operater="max_position")
# Sleep for rate_delay
# Tracer()()
# delay_multiple = self.weighted_choice(delay_choices)
# if delay_multiple is not 0:
# delay_time = random.uniform(rate_delay*(delay_multiple-1), rate_delay*delay_multiple)
# logging.log(logging.DEBUG,"Sleep for "+ str(delay_time) +" seconds")
# time.sleep(delay_time)
# # if delay_time > 22:
# # next_url = self.construct_url(
# # self.query,
# # max_position=self.data_max_position,
# # operater="min_position")
# # yield Request(url=next_url, callback=self.parse,dont_filter=True)
# else:
# logging.log(logging.DEBUG,"Sleep for 0 seconds")
print
print "Next Request:" + "TWEET-%s-%s" % (
self.max_tweet['tweet_id'], self.min_tweet['tweet_id'])
print
# Tracer()()
yield Request(url=next_url,
callback=self.parse,
dont_filter=True)
except Exception, e:
pass
def process(index, col):
global err
inst, major, degree, season, decision, status, date_add, date_add_ts, comment = None, None, None, None, None, None, None, None, None
if len(col) != 6:
Tracer()()
try:
inst = col[0].text.strip().encode('ascii', 'ignore')
except:
Tracer()()
try:
major = None
progtext = col[1].text.strip().encode('ascii', 'ignore')
for (p, nam) in PROGS:
if p.lower() in progtext.lower():
major = nam
break
if not major:
major = 'Other'
errlog['major'].append((index, col))
degree = None
for (d, deg) in DEGREE:
if d in progtext:
degree = deg
break
if not degree:
degree = 'Other'
season = None
mat = re.search('\([SF][01][0-9]\)', progtext)
if mat:
season = mat.group()[1:-1]
else:
mat = re.search('\(\?\)', progtext)
if mat:
season = None
except NameError as e:
print e
Tracer()()
except:
print "Unexpected error:", sys.exc_info()[0]
Tracer()()
try:
extra = col[2].find(class_='extinfo')
gpafin, grev, grem, grew, new_gre, sub = None, None, None, None, None, None
if extra:
gre_text = extra.text.strip().encode('ascii', 'ignore')
gpa = re.search('Undergrad GPA: ((?:[0-9]\.[0-9]{1,2})|(?:n/a))',
gre_text)
general = re.search(
'GRE General \(V/Q/W\): ((?:1[0-9]{2}/1[0-9]{2}/(?:(?:[0-6]\.[0-9]{2})|(?:99\.99)|(?:56\.00)))|(?:n/a))',
gre_text)
new_gref = True
subject = re.search('GRE Subject: ((?:[2-9][0-9]0)|(?:n/a))',
gre_text)
if gpa:
gpa = gpa.groups(1)[0]
if not gpa == 'n/a':
try:
gpafin = float(gpa)
except:
Tracer()()
else:
errlog['gpa'].append((index, gre_text))
if not general:
general = re.search(
'GRE General \(V/Q/W\): ((?:[2-8][0-9]0/[2-8][0-9]0/(?:(?:[0-6]\.[0-9]{2})|(?:99\.99)|(?:56\.00)))|(?:n/a))',
gre_text)
new_gref = False
if general:
general = general.groups(1)[0]
if not general == 'n/a':
try:
greparts = general.split('/')
if greparts[2] == '99.99' or greparts[
2] == '0.00' or greparts[2] == '56.00':
grew = None
else:
grew = float(greparts[2])
grev = int(greparts[0])
grem = int(greparts[1])
new_gre = new_gref
if new_gref and (grev > 170 or grev < 130 or grem > 170
or grem < 130 or
(grew and (grew < 0 or grew > 6))):
errlog['general'].append((index, gre_text))
grew, grem, grev, new_gre = None, None, None, None
elif not new_gref and (grev > 800 or grev < 200
or grem > 800 or grem < 200 or
(grew and
(grew < 0 or grew > 6))):
errlog['general'].append((index, gre_text))
grew, grem, grev, new_gre = None, None, None, None
except Exception as e:
Tracer()()
else:
errlog['general'].append((index, gre_text))
if subject:
subject = subject.groups(1)[0]
if not subject == 'n/a':
sub = int(subject)
else:
errlog['subject'].append((index, gre_text))
extra.extract()
decision = col[2].text.strip().encode('ascii', 'ignore')
try:
decisionfin, method, decdate, decdate_ts = None, None, None, None
(decisionfin, method, decdate) = re.search(
'((?:Accepted)|(?:Rejected)|(?:Wait listed)|(?:Other)|(?:Interview))? ?via ?((?:E-[mM]ail)|(?:Website)|(?:Phone)|(?:Other)|(?:Postal Service)|(?:Unknown))? ?on ?([0-9]{1,2} [A-Z][a-z]{2} [0-9]{4})?',
decision).groups()
if method and method == 'E-Mail':
method = 'E-mail'
if method and method == 'Unknown':
method = 'Other'
if decdate:
try:
decdate_date = datetime.datetime.strptime(
decdate, '%d %b %Y')
decdate_ts = decdate_date.strftime('%s')
decdate = decdate_date.strftime('%d-%m-%Y')
except Exception as e:
decdate_date, decdate_ts, decdate = None, None, None
except Exception as e:
Tracer()()
except Exception as e:
Tracer()()
try:
statustxt = col[3].text.strip().encode('ascii', 'ignore')
if statustxt in STATUS:
status = STATUS[statustxt]
else:
status = None
except:
Tracer()()
try:
date_addtxt = col[4].text.strip().encode('ascii', 'ignore')
date_add_date = datetime.datetime.strptime(date_addtxt, '%d %b %Y')
date_add_ts = date_add_date.strftime('%s')
date_add = date_add_date.strftime('%d-%m-%Y')
except:
# print(col[4].text.strip().encode('ascii', 'ignore'))
Tracer()()
try:
comment = col[5].text.strip().encode('ascii', 'ignore')
except:
Tracer()()
res = [
inst, major, degree, season, decisionfin, method, decdate, decdate_ts,
gpafin, grev, grem, grew, new_gre, sub, status, date_add, date_add_ts,
comment
]
print res
return res
def extract_tweets(self, items_html):
"""
Parses Tweets from the given HTML
:param items_html: The HTML block with tweets
:return: A JSON list of tweets
"""
try:
soup = BeautifulSoup(items_html, "lxml")
tweets = []
twitter_username_re = re.compile(
r'(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z_]+[A-Za-z0-9_]+[A-Za-z]+[A-Za-z0-9])'
# r'(?<=@)\w+'
)
for li in soup.find_all("li", class_='js-stream-item'):
# Tracer()()
# If our li doesn't have a tweet-id, we skip it as it's not going
# to be a tweet.
if 'data-item-id' not in li.attrs:
continue
tweet = {
'tweet_id': li['data-item-id'],
'text': None,
# 'is_retweet':false,
'user_id': None,
'user_screen_name': None,
'user_name': None,
'created_at_ts': None,
'created_at_iso': None,
# 'convo_url': None,
# 'image_url': [],
'num_retweets': 0,
'num_favorites': 0,
'keyword': [],
'quote_tweet_id': None,
'quote_tweet_userid': None,
'quote_tweet_username': None,
'quote_tweet_screenname': None,
'quote_tweet_text': None,
'html': None
}
try:
tweet['html'] = str(li)
except Exception, e:
Tracer()()
pass
'''
Extract tweet text
'''
try:
text_p = li.find("p", class_="tweet-text")
if text_p is not None:
tweet['text'] = text_p.get_text()
except Exception, e:
Tracer()()
logging.log(logging.DEBUG,
"ERROR(extract_text_p): %s" % (str(e), ))
traceback.print_exc()
'''
Extract quote tweet content if exists
'''
try:
quote_tweet = li.find("div",
class_="QuoteTweet-innerContainer")
if quote_tweet is not None:
# Tracer()()
tweet['quote_tweet_id'] = quote_tweet['data-item-id']
tweet['quote_tweet_userid'] = quote_tweet[
'data-user-id']
tweet['quote_tweet_screenname'] = quote_tweet[
'data-screen-name']
tweet['quote_tweet_username'] = quote_tweet.find(
"b", class_="QuoteTweet-fullname").get_text()
tweet['quote_tweet_text'] = quote_tweet.find(
"div", class_="QuoteTweet-text").get_text()
except Exception, e:
Tracer()()
logging.log(logging.DEBUG,
"ERROR(extract_quote_tweet): %s" % (str(e), ))
traceback.print_exc()
eval(cmd)
%prun [opts] statement # profiler; profile.help(); -r # return pstats.Stats obj; / -l # <limit> string (only some fn_names),int(#ln),float(.25 - quarter of output),e.g. '-l __init__ -l 5' top 5 ln of constructors; / -s <key> # sort by key
%prun -l 7 -s cumulative fn1(); %run -p -s cumulative fnm.py; %lprun -f fn1 -f fn2 statemt_to_profile # %prun (cProfile) for macro_profiling and %lprun (line_profiler) for micro_profiling
$python -m cProfile cprof_example.py; execfile(fnm); %run [-n -i -t [-N<N>] -d [-b<N>] -p [profile options]] fnm [args] # like $python fnm args; with ipy traceback; %run in empty nsp, %run -i in current nsp; leaves behind vars
%timeit range(1000) # Line magics (1-liner,args on line1)
%%timeit x=numpy.random.randn((100,100)) / numpy.linalg.svd(x) # setup not timed
import profile,pstats; from timeit import Timer; Timer('t=a;a=b;b=t','a=1; b=2').timeit(); Timer('a,b=b,a','a=1; b=2').timeit() # profile,pstats; traditional swapping args vs tuple arg pack/unpack
timer=time.clock if sys.platform[:3]=='win' else time.time; min(Timer.total(1000,str.upper,'spam') for i in range(50)) # profile -wall_clock,cpu_clock
# c.TerminalIPythonApp.extensions=['line_profiler']
# %load_ext line_profiler # works for iPython.exe, not for pyDev (%lprun unavail)
import unittest,doctest ## unit/system testing, white/black box testing
#--- 07 dbg/traceback(~er_msg,fnm/line#/fn/fnm_txt + fnm_caller/..),ipdb,inspect --------------------------------------
# tb_obj~err_log, it contains "stack_trace" (fnm,line#,fn_nm,er_type,er_msg),has link to frm (possibly frm_dead as well); ipdb.pm(tb_obj) (no frame stack walk here, just look at obj before gc deletes them?);
o=sys.exc_info()[2]; o=sys.last_traceback; insp_stack2(o)
%tb 'prints tb, mode set up in %xmode'; traceback.print_exc(file=open('fnm.txt','w')); threading.settrace(fn) 'tb_multithr'
from IPython.core.debugger import Tracer; Tracer()()
#--- ipdb h; init in .pdbrc,pdb~ipdb;
# kb(),insp_stack2(),whos2(); from numpy.lib.utils import source; source(fn1) 'fnm+src,good enough';
# help,h; alias,unalias;
# run,restart; where,w,bt; up,u; down,d; next,n; step,s; jump,j;continue,cont,c; return,r; until,unt; commands[cmd to continue]; debug[dbg2];
# whatis;args,a; pinfo,pdef,pdoc; list,l; pp,p # pp get_ipython().magic('who_ls'); src_code,print
# break,b,tbreak,enable,disable,condition,ignore,clear,cl; quit,q,exit; EOF; #bpts
# if stuck on input - do empty line; stuck on output - "q"
# b=123; whos2() 'mixed statemt fail in ipdb'; a?? fails in ipdb,eval(), use numpy.lib.utils.source(fn), insp_stack2,whos2(),
# frm_stack~stack_trace~stack_snapshot, for interactive and post-mortem debugging (pm uses traceback)
%pdb on; %debug~ipdb.pm() # open dbg at latest exception (it clobbers/kills prev ones), init in `InteractiveShell.pdb`;
#--- ipdb test
get_ipython().magic('%reset -f'); from vm0 import *; import ipdb; ipdb.launch_ipdb_on_exception(); # ipdb.set_trace()
ipdb.run('wh()'); ipdb.pm() # dbg/traceback printout; ipdb.post_mortem, vars in "old fn frames" are preserved until gc
def set_trace(): from IPython.core.debugger import Pdb; Pdb(color_scheme='Linux').set_trace(sys._getframe().f_back) # set bpt
def debug(f,*args,**kargs): from IPython.core.debugger import Pdb; pdb=Pdb(color_scheme='Linux'); return pdb.runcall(f,*args,**kargs) # dbg_run_fn
def loss_function_vec(
self,
predicts,
labels,
threshold,
cal_accuracy=False
): #labels: [batch_size, cell_size_x, cell_size_y, 2+5] (x, y, w, h, C, p(c0), p(c1))
predict_class = predicts[:, :self.cell_size * self.cell_size *
self.num_classes].contiguous().view(
self.batch_size, self.cell_size,
self.cell_size, self.num_classes
) #batch_size, cell_size, cell_size, num of class (class score)
predict_confidence = predicts[:, self.cell_size * self.cell_size *
self.num_classes:(
self.cell_size * self.cell_size *
self.num_classes +
self.cell_size * self.cell_size * 2
)].contiguous().view(
self.batch_size, self.cell_size,
self.cell_size, 2
) #batch_size, cell_size, cell_size, num of boxes (box confidence)
predict_boxes = predicts[:, (
self.cell_size * self.cell_size * self.num_classes +
self.cell_size * self.cell_size * 2
):].contiguous().view(
self.batch_size, self.cell_size, self.cell_size, 2, 4
) # batch_size, cell_size, cell_size, boxes_num, 4 (box coordinate)
gt_object = labels[:, :, :,
4].contiguous().view(self.batch_size,
self.cell_size, self.cell_size,
1)
gt_boxes = labels[:, :, :,
0:4].contiguous().view(self.batch_size,
self.cell_size,
self.cell_size, 1, 4)
gt_boxes = gt_boxes.expand(self.batch_size, self.cell_size,
self.cell_size, 2, 4)
gt_classes = labels[:, :, :, 5:]
predict_boxes_tran = torch.stack([
(predict_boxes[:, :, :, :, 0] + self.offset) * 16,
(predict_boxes[:, :, :, :, 1] + self.offset.permute(0, 2, 1, 3)) *
16, predict_boxes[:, :, :, :, 2] * 112,
predict_boxes[:, :, :, :, 3] * 112
])
predict_boxes_tran = predict_boxes_tran.permute(1, 2, 3, 4, 0)
gt_boxes_tran = torch.stack([
(gt_boxes[:, :, :, :, 0] + self.offset) * 16,
(gt_boxes[:, :, :, :, 1] + self.offset.permute(0, 2, 1, 3)) * 16,
gt_boxes[:, :, :, :, 2] * 112, gt_boxes[:, :, :, :, 3] * 112
])
gt_boxes_tran = gt_boxes_tran.permute(1, 2, 3, 4, 0)
gt_iou = self.iou_calc(predict_boxes_tran, gt_boxes_tran)
max_iou = torch.max(gt_iou, 3, keepdim=True)
max_iou = max_iou[0]
object_mask = torch.mul(
torch.ge(gt_iou, max_iou).float(), gt_object.float())
noob_mask = Variable(torch.ones(object_mask.size())) - object_mask
#class loss
delta_p = gt_classes - predict_class
delta_p_obj = delta_p * gt_object
class_loss = self.lambda_class * torch.sum(delta_p_obj**
2) / self.batch_size
#coord loss
coord_mask = object_mask.contiguous().view(self.batch_size,
self.cell_size,
self.cell_size, 2, 1)
coord_delta = predict_boxes[:, :, :, :, :2] - gt_boxes[:, :, :, :, :2]
coord_delta_mask = coord_delta * coord_mask
size_delta = torch.sqrt(predict_boxes[:, :, :, :, 2:]) - torch.sqrt(
gt_boxes[:, :, :, :, 2:])
size_delta_mask = size_delta * coord_mask
coord_loss = (torch.sum(coord_delta_mask**2) + \
torch.sum(size_delta_mask**2))/self.batch_size * self.lambda_coord
#iou loss
confidence_delta = predict_confidence - gt_iou
iou_loss = (torch.sum((confidence_delta*object_mask)**2)+ \
self.lambda_noobj * torch.sum((confidence_delta*noob_mask)**2))/self.batch_size
# print(class_loss)
# print(coord_loss)
# print(iou_loss)
total_loss = class_loss + coord_loss + iou_loss
# print (total_loss)
# pdb.set_trace()
accuracy = []
if cal_accuracy is False:
accuracy = [0, 0, 0]
else:
#############detection accuracy###############
gt_noob = Variable(torch.ones(gt_object.size())) - gt_object
threshold = threshold
max_confidence = torch.max(predict_confidence, 3, keepdim=True)
detect_iou = torch.ge(max_confidence[0], threshold).float()
detect_iou_tp = detect_iou * gt_object
detect_iou_fp = detect_iou * gt_noob
detect_tp_accu = torch.sum(detect_iou_tp) / self.batch_size
detect_fp_accu = torch.sum(detect_iou_fp) / self.batch_size
detect_gt = torch.sum(gt_object) / self.batch_size
detect_tp_accu = detect_tp_accu / detect_gt
detect_fp_accu = detect_fp_accu / detect_gt
accuracy.append(detect_tp_accu)
accuracy.append(detect_fp_accu)
#############iou accuracy#####################
iou_accu = torch.sum(
max_confidence[0] * detect_iou_tp) / self.batch_size
iou_accu = iou_accu / detect_gt
accuracy.append(iou_accu)
#############class accuracy###################
predicted_class = torch.max(predict_class, 3, keepdim=True)[1]
groundtruth_classes = torch.max(gt_classes, 3, keepdim=True)[1]
class_eq = (predicted_class == groundtruth_classes)
class_hit = class_eq.float() * detect_iou * gt_object
class_accu = torch.sum(
class_hit.data.float()) / torch.sum(gt_object)
accuracy.append(class_accu)
if math.isnan(total_loss.data.cpu().numpy()):
for i0 in range(self.batch_size):
for i1 in range(self.cell_size):
for i2 in range(self.cell_size):
for i3 in range(2):
if math.isnan(
confidence_delta[i0, i1, i2,
i3].data.cpu().numpy()):
print([i0, i1, i2, i3])
Tracer()()
return total_loss, accuracy
Tracer()()
from IPython.core.debugger import Tracer; keyboard = Tracer()
import tensorflow as tf
import abc
class BaseModel(object):
"""
Abstracted class for neural network models using tensorflow.
"""
def __init__(self, name, seed):
tf.set_random_seed(seed)
self.seed = seed
self.name = name
def getAllVariables(self):
# returns dictionary wihch contrains all model's variable
# e.x: modelname := 'CNN' -> 'CNN/foo/bar', 'foo/CNN/bar' are OK, 'CTCNN/foo/bar' will be ignored
r_dict = {}
for v in tf.all_variables():
strNum = v.name.find(self.name+'/')
if strNum >= 0:
if strNum == 0 or v.name[strNum-1] == '/':
r_dict[v.name] = v
return r_dict
def checkWeightSum(self):
# retuens sum of all variables in the model
s = 0.0
for name, v in self.getAllVariables().iteritems():
s += tf.reduce_sum(v)
CreateLib().
TODO: Add facility to generate our own custom defined stub files. In particular
so we can create a TextWindowPrint() command.
TODO: Switch to jinja2 templates instead of string format replacements.
"""
import os
import yaml
import re
import glovars
# Setup to use breakpt() for droppping into ipdb:
from IPython.core.debugger import Tracer
breakpt = Tracer()
# Regex for wrapping paragraphs...
# 60 columns then up to the next whitespace:
PARMAT = re.compile('(.{60}.+?)\s')
DOCSTRING = '''
' Wrapper stub for {filename} {cmdtypecaps}
'
' {cmddoc}
'
' Description:
' {descr}
'
' Returns:
' {returndoc}
'
import os
import numpy as np
import matplotlib
import matplotlib.pyplot as pl
import logging
_log = logging.getLogger('mcfost')
logging.basicConfig(level=logging.INFO)
# this lets you put "stop()" in your code to have a debugger breakpoint
from IPython.core.debugger import Tracer
stop = Tracer()
_VERBOSE = False
# some extremely simple classes to serve as structs.
class Star():
temp = 4000
radius = 1.0
mass = 1.0
x = 0.0
y = 0.0
z = 0.0
spectrum = ''
fUV = 0.0
flope_fuv = 0.0
import numpy as np
from pystruct.problems import GraphCRF
from pystruct.inference import inference_dispatch
from IPython.core.debugger import Tracer
tracer = Tracer()
class IgnoreVoidCRF(GraphCRF):
"""GraphCRF that ignores nodes with void label in ground truth.
"""
def __init__(self,
n_states=2,
n_features=None,
inference_method='qpbo',
void_label=21):
if void_label >= n_states:
raise ValueError("void_label must be one of the states!")
GraphCRF.__init__(self, n_states, n_features, inference_method)
self.void_label = void_label
def max_loss(self, y):
return np.sum(y != self.void_label)
def loss(self, y, y_hat):
# hamming loss:
return np.sum((y != y_hat)[y != self.void_label])
def loss_augmented_inference(self,
def stress_constraints(tri_data1, tri_data2, sm, pr):
out = []
E1, S1, x_to_xp1 = derive_stress(tri_data1, sm, pr)
E2, S2, x_to_xp2 = derive_stress(tri_data2, sm, pr)
tri1, _, tri_idx1, corner_idx1 = tri_data1
dof_start1 = tri_idx1 * 9 + corner_idx1 * 3
n1 = np.cross(tri1[1] - tri1[0], tri1[2] - tri1[0])
n1 /= np.linalg.norm(n1)
tri2, _, tri_idx2, corner_idx2 = tri_data2
dof_start2 = tri_idx2 * 9 + corner_idx2 * 3
n2 = np.cross(tri2[1] - tri2[0], tri2[2] - tri2[0])
n2 /= np.linalg.norm(n2)
S1xx = rotate_tensor(S1, x_to_xp1)
np.testing.assert_almost_equal(
Sdot(S1xx, n1), np.array([[0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]))
S2xx = rotate_tensor(S2, x_to_xp2)
np.testing.assert_almost_equal(
Sdot(S2xx, n2), np.array([[0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]))
def make_constraint(lhs, rhs):
terms = []
for k in range(3):
terms.append(Term(lhs[1 + k], dof_start1 + k))
terms.append(Term(-rhs[1 + k], dof_start2 + k))
out.append(ConstraintEQ(terms, -lhs[0] + rhs[0]))
lhs1 = n2.dot(Sdot(S1xx, n1))
rhs1 = n2.dot(Sdot(S2xx, n1))
make_constraint(lhs1, rhs1)
r2 = np.cross(n1, n2)
r2 /= np.linalg.norm(r2)
r1 = np.cross(n1, r2)
r3 = np.cross(n2, r2)
lhs2 = S1xx[0, 0] + S1xx[1, 1] + S1xx[2, 2]
rhs2 = S2xx[0, 0] + S2xx[1, 1] + S2xx[2, 2]
# norm = np.linalg.norm(lhs2[1:])
# lhs2 /= norm
# rhs2 /= norm
# lhs2[0] *= -1.5
# rhs2[0] *= -1.5
# make_constraint(lhs2, rhs2)
nmid = (n1 + n2) / 2.0
nmid /= np.linalg.norm(nmid)
lhs3 = nmid.dot(Sdot(S1xx, r2))
rhs3 = nmid.dot(Sdot(S2xx, r2))
norm = np.linalg.norm(lhs3[1:])
lhs3 /= norm
rhs3 /= norm
# lhs3[0] /= -7.4
# rhs3[0] /= -7.4
# make_constraint(lhs3, rhs3)
if np.any(S1xx[:, :, 0]) != 0:
print('C1: ', lhs1, rhs1)
print('C2: ', lhs2, rhs2)
print('C3: ', lhs3, rhs3)
from IPython.core.debugger import Tracer
Tracer()()
return out
from IPython.core.debugger import Tracer
dh = Tracer()
from pyjmi import transfer_model, transfer_optimization_problem, get_files_path
from pyjmi.optimization.casadi_collocation import BlockingFactors
from pymodelica import compile_fmu
from pyfmi import load_fmu
import matplotlib.pyplot as plt
import os
from pyjmi.common.io import ResultDymolaTextual
import time
import sys
sys.path.append('..')
import symbolic_processing as sp
from simulation import *
# Define problem
plt.rcParams.update({'text.usetex': False})
with_plots = True
#~ with_plots = False
blt = True
#~ blt = False
caus_opts = sp.CausalizationOptions()
#~ caus_opts['plots'] = True
#~ caus_opts['draw_blt'] = True
#~ caus_opts['solve_blocks'] = True
#~ caus_opts['ad_hoc_scale'] = True
#~ caus_opts['inline'] = False
#~ caus_opts['closed_form'] = True
#~ caus_opts['inline_solved'] = True
def extract_tweets(self, items_html):
"""
Parses Tweets from the given HTML
:param items_html: The HTML block with tweets
:return: A JSON list of tweets
"""
try:
soup = BeautifulSoup(items_html, "lxml")
tweets = []
twitter_username_re = re.compile(
r'(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z_]+[A-Za-z0-9_]+[A-Za-z]+[A-Za-z0-9])'
# r'(?<=@)\w+'
)
for li in soup.find_all("li", class_='js-stream-item'):
# Tracer()()
# If our li doesn't have a tweet-id, we skip it as it's not going
# to be a tweet.
if 'data-item-id' not in li.attrs:
continue
tweet = {
'tweet_id': li['data-item-id'],
'text': None,
# 'is_retweet':false,
'user_id': None,
'user_screen_name': None,
'user_name': None,
'created_at_ts': None,
'created_at_iso': None,
# 'convo_url': None,
# 'image_url': [],
'num_retweets': 0,
'num_favorites': 0,
'keyword': [],
'quote_tweet_id': None,
'quote_tweet_userid': None,
'quote_tweet_username': None,
'quote_tweet_screenname': None,
'quote_tweet_text': None,
'html': None
}
try:
tweet['html'] = str(li)
except Exception, e:
Tracer()()
pass
'''
Extract tweet text
'''
try:
text_p = li.find("p", class_="tweet-text")
if text_p is not None:
tweet['text'] = text_p.get_text()
# If there is any user mention containing the query, then pass the tweet.
# Tracer()()
# if self.query.find("from:") == -1:
user_mentions = twitter_username_re.match(tweet['text'])
if user_mentions and any([self.query_keyword.lower() in user_mention.lower() for user_mention in user_mentions.groups()]):
# Tracer()()
logging.log(logging.DEBUG, 'Found '+self.query_keyword+' in '+ str(user_mentions.groups())+': Drop tweet '+tweet['tweet_id'])
continue
# If the keyword was found in the text and was the same with query, then accept the tweet
# elif text_p.find("strong") and text_p.find("strong").get_text().lower() == self.query_keyword.lower():
# tweet['keyword'] = text_p.find("strong").get_text()
# elif tweet['text'].lower().find(self.query_keyword.lower()) != -1:
# tweet['keyword'] = self.query_keyword
# else:
# # The keyword is not in the text, then pass the tweet.
# # Tracer()()
# logging.log(logging.DEBUG, 'No '+self.query_keyword +' in the content of tweet'+': Drop tweet '+tweet['tweet_id'])
# continue
else:
# Tracer()()
logging.log(logging.DEBUG, 'No content in the tweet' + ': Drop tweet ' + tweet['tweet_id'])
continue
except Exception, e:
Tracer()()
logging.log(logging.DEBUG, "ERROR(extract_text_p): %s" % (str(e),))
traceback.print_exc()
'''
Extract quote tweet content if exists
'''
try:
quote_tweet = li.find("div", class_="QuoteTweet-innerContainer")
if quote_tweet is not None:
# Tracer()()
tweet['quote_tweet_id'] = quote_tweet['data-item-id']
tweet['quote_tweet_userid'] = quote_tweet['data-user-id']
tweet['quote_tweet_screenname'] = quote_tweet['data-screen-name']
tweet['quote_tweet_username'] = quote_tweet.find("b",class_="QuoteTweet-fullname").get_text()
tweet['quote_tweet_text'] = quote_tweet.find("div",class_="QuoteTweet-text").get_text()
except Exception, e:
Tracer()()
logging.log(logging.DEBUG, "ERROR(extract_quote_tweet): %s" % (str(e),))
traceback.print_exc()
# internal libraries
from templates import c_template, h_template, pyx_template, setup_template
# Python 2 vs 3 importing
try:
from string import letters as all_letters
except ImportError:
from string import ascii_letters as all_letters
# debugging
try:
from IPython.core.debugger import Tracer
except ImportError:
pass
else:
set_trace = Tracer()
class CythonGenerator(object):
def __init__(self,
filename_prefix,
mass_matrix,
forcing_vector,
constants,
coordinates,
speeds,
specified=None):
"""Instantiates an object that can generates a Cython shared object
module with a function that evaluates the provided mass_matrix and
the forcing vector given the numerical values of the input
variables.
def DebugHere():
if ipy_debug_available:
t = Tracer()
t()
else:
print 'ipython not available for debugging'
def main():
global g_options
parser = OptionParser()
parser.add_option("--scrape",
dest="scrape",
action="store_true",
help="Scrape random samples from XenoCanto")
parser.add_option(
"--scrape-conserve",
dest="scrape_conserve",
action="store_true",
help=
"Scrape random samples from XenoCanto, but only labels which already exist"
)
parser.add_option(
"--scrape-period",
dest="scrape_interval",
action="store",
type="int",
help=
"Scrape random samples from XenoCanto, at given interval in seconds")
parser.add_option("--stats",
dest="stats",
action="store_true",
help="Print statistics for local samples")
parser.add_option("-S",
"--make-spectrograms",
dest="spectrograms_build",
action="store_true",
help="Make and overwrite spectrograms to file.")
parser.add_option("-t",
"--load-templates",
dest="templates_load",
action="store_true",
help="Load templates from file")
parser.add_option(
"-T",
"--make-templates",
dest="templates_build",
action="store_true",
help="Make and overwrite templates to file. Equivalent to -sT")
parser.add_option("--show-templates",
dest="templates_interactive",
action="store_true",
help="Show template extraction results")
parser.add_option("-d", dest="data_load", action="store")
parser.add_option("-f",
"--load-features",
dest="features_load",
action="store_true",
help="Load features from file")
parser.add_option(
"-F",
"--make-features",
dest="features_build",
action="store",
help="Extract and overwrite features to file. Equivalent to -stF")
parser.add_option("-c",
"--classify",
dest="classify",
action="store_true",
help="Run classifier. Equivalent to -stfc")
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_true",
help="Print verbose output")
parser.add_option("-i",
"--informative",
dest="informative",
action="store_true",
help="Print informative output")
parser.add_option("-l",
"--filter-label",
dest="label_filter",
action="store",
help="Process only samples of a givel label value")
parser.add_option("--merge", dest="merge_results", action="store_true")
(options, args) = parser.parse_args()
g_options = options
logging.basicConfig(level=logging.INFO)
__loaddata = not options.classify
if __loaddata:
repository = SampleRepository(spectrograms_dir=DIR_SPECTROGRAMS,
samples_dir=DIR_SAMPLES)
repository.gather_samples()
if process_scrape_options(options, repository): exit()
if process_stats_options(options, repository): exit()
previous_data = load_feature_data(options.data_load) or None
logging.info('{} samples'.format(len(repository.samples)))
class_to_idx = previous_data.label_map \
if previous_data is not None \
else make_class_mapping(repository.samples)
idx_to_class = {v: k for k, v in class_to_idx.iteritems()}
logging.info('{} classes'.format(len(class_to_idx)))
# filter ids (keep/reject previous incl. labels)
previous_ids = previous_data.ids if previous_data is not None else []
preserve = True
if preserve:
logging.info('keeping ids: {}'.format(previous_ids))
if len(previous_ids) != 0:
repository.filter_uids(previous_ids, reject=False)
else:
logging.info('rejecting ids: {}'.format(previous_ids))
repository.filter_uids(previous_ids, reject=True)
logging.info(
'remove previous ids: filtered down to {} samples'.format(
len(repository.samples)))
previous_labels = set([idx_to_class[y] for y in previous_data.y])
logging.info('rejecting labels: {}'.format(previous_labels))
repository.filter_labels(previous_labels, reject=True)
logging.info('filter labels: filtered down to {} samples'.format(
len(repository.samples)))
repository.filter_labels(['Eurasian Blackcap', 'Garden Warbler'],
reject=False)
repository.reject_by_class_count(at_least=20)
logging.info(
'remove low bound: filtered down to {} samples'.format(
len(repository.samples)))
repository.keep_n_of_each_class(20)
if options.spectrograms_build:
logging.error('UNSUPPORTED ACTION -- NOT STORING SGRAMS DEV ONLY')
sgram_count = build_all_spectrograms(repository.samples)
logging.info('built {} spectrograms'.format(sgram_count))
#repository.store_all()
#store_all_spectrograms(samples)
logging.info('filtered down to {} samples'.format(
len(repository.samples)))
print_template_statistics(repository.samples)
if options.templates_build:
logging.error("WILL NOT STORE TEMPLATES .. DEV ONLY")
#Tracer()()
#delete_stored_templates(samples)
all_templates = build_all_templates(repository.samples, options,
options.label_filter)
#store_all_templates(samples)
logging.info('extracted {} templates'.format(len(all_templates)),
'', 'vvv after template build vvv')
print_template_statistics(samples)
exit()
repository.reject_by_template_count_per_class(at_least=2000)
print_template_statistics(repository.samples)
X = None
y = None
ids = None
if options.merge_results:
data_1 = load_feature_data('./merged_data_2.pkl')
data_2 = load_feature_data('./_new_data_1.pkl')
pids = data_1.ids + data_2.ids
Tracer()()
repository.filter_uids(pids, reject=False)
data = merge_results_d(data_1, data_2, repository)
pickle_safe(results, './_merged_results.pkl')
Tracer()()
elif options.features_build:
repository.filter_labels([
'Eurasian Reed Warbler', 'Garden Warbler',
'Western Meadowlark', 'Eurasian Blackcap'
],
reject=False)
all_templates = get_first_n_templates_per_class(repository, 3000)
logging.info('{} template(s) loaded'.format(len(all_templates)))
X, y, ids = extract_features(repository.samples, all_templates,
class_to_idx)
used_templates = [t.get_uid() for t in all_templates]
data = {
'X': X,
'y': y,
'ids': ids,
'label_map': class_to_idx,
'template_order': used_templates
}
pickle_safe(data, options.features_build)
elif options.features_load:
data = previous_data
if options.classify:
#with open('./cnfdata', 'r') as f:
# ce = pickle.load(f)
#plot_cnf_matrix(ce.cnf, data.y, idx_to_class, normalize=True, show_values=True)
#Tracer()()
data = load_feature_data(options.data_load)
ce = ClfEval(data, 10, 10, None)
#clf = ExtraTreesClassifier(
param_grid = {
'n_estimators': [300], #[10, 500, 5000, 10000],
'max_features':
[0.33
], #, 'sqrt', 'log2'],#, len(previous_data.template_order)*0.8],
'min_samples_split': [2], #, 10, 100],
'min_samples_leaf': [1], #, 10, 100],
'max_depth': [None] #, 5, 10, 100, 200]
}
__params_l = list(ParameterGrid(param_grid))
__i = 0
print('')
for p in __params_l:
print(' {}'.format(p))
print('')
# Evaluate all parameters. !! Results are dumped to console !!
t1 = time.time()
evaluate_clf_params(__params_l, 0, data)
print('total time {}'.format(time.time() - t1))
def get_kmeans_player_logs(n_clusters=100):
logs = get_logs_for_clustering()
kmeans = KMeans(n_clusters=n_clusters, random_state=0).fit(logs)
centers = kmeans.cluster_centers_
print kmeans.cluster_centers_
Tracer()()
def crazy_visual():
dataset = NYUSegmentation()
# load training data
data = load_nyu(n_sp=500)
data = add_edges(data)
for x, image_name, superpixels, y in zip(data.X, data.file_names,
data.superpixels, data.Y):
print(image_name)
if int(image_name) != 11:
continue
image = dataset.get_image(image_name)
plt.figure(figsize=(20, 20))
bounary_image = mark_boundaries(image, superpixels)
plt.imshow(bounary_image)
gridx, gridy = np.mgrid[:superpixels.shape[0], :superpixels.shape[1]]
edges = x[1]
points_normals = dataset.get_pointcloud_normals(image_name)
centers2d = get_superpixel_centers(superpixels)
centers3d = [
np.bincount(superpixels.ravel(), weights=c.ravel())
for c in points_normals[:, :, :3].reshape(-1, 3).T
]
centers3d = (np.vstack(centers3d) / np.bincount(superpixels.ravel())).T
sp_normals = get_sp_normals(points_normals[:, :, 3:], superpixels)
offset = centers3d[edges[:, 0]] - centers3d[edges[:, 1]]
offset = offset / np.sqrt(np.sum(offset**2, axis=1))[:, np.newaxis]
#mean_normal = (sp_normals[edges[:, 0]] + sp_normals[edges[:, 1]]) / 2.
mean_normal = sp_normals[edges[:, 0]]
#edge_features = np.arccos(np.abs((offset * mean_normal).sum(axis=1))) * 2. / np.pi
edge_features = 1 - np.abs((offset * mean_normal).sum(axis=1))
no_normals = (np.all(sp_normals[edges[:, 0]] == 0, axis=1) +
np.all(sp_normals[edges[:, 1]] == 0, axis=1))
edge_features[no_normals] = 0 # nan normals
if True:
coords = points_normals[:, :, :3].reshape(-1, 3)
perm = np.random.permutation(superpixels.max() + 1)
mv.points3d(coords[:, 0],
coords[:, 1],
coords[:, 2],
perm[superpixels.ravel()],
mode='point')
#mv.points3d(centers3d[:, 0], centers3d[:, 1], centers3d[:, 2], scale_factor=.04)
mv.quiver3d(centers3d[:, 0], centers3d[:, 1], centers3d[:, 2],
sp_normals[:, 0], sp_normals[:, 1], sp_normals[:, 2])
mv.show()
from IPython.core.debugger import Tracer
Tracer()()
for i, edge in enumerate(edges):
e0, e1 = edge
#color = (dataset.colors[y[e0]] + dataset.colors[y[e1]]) / (2. * 255.)
#f = edge_features[i]
#if f < 0:
#e0, e1 = e1, e0
#f = -f
#plt.arrow(centers[e0][0], centers[e0][1],
#centers[e1][0] - centers[e0][0], centers[e1][1] - centers[e0][1],
#width=f * 5
#)
color = "black"
plt.plot([centers2d[e0][0], centers2d[e1][0]],
[centers2d[e0][1], centers2d[e1][1]],
c=color,
linewidth=edge_features[i] * 5)
plt.scatter(centers2d[:, 0], centers2d[:, 1], s=100)
plt.tight_layout()
plt.xlim(0, superpixels.shape[1])
plt.ylim(superpixels.shape[0], 0)
plt.axis("off")
plt.savefig("figures/normal_relative/%s.png" % image_name,
bbox_inches="tight")
plt.close()
# k_means.py
# calculate k-means over the lyric database
# using tfidf scaling + cosine similarity
import sys
import sqlite3
from math import sqrt
import numpy as np
trace = None
try:
from IPython.core.debugger import Tracer
trace = Tracer()
except ImportError:
def disabled():
dbg("Tracepoint disabled -- IPython not found")
trace = disabled
MXM_DB = "mxm_dataset.db"
MXM_TFIDF = "mxm_tfidf_small.db"
tfidf = None
num_means = 6
total_docs = 0
modpct = 1
centroids = []
from __future__ import print_function
import torch
from seq2seq.loss import NLLLoss
from IPython.core.debugger import Tracer
debug_here = Tracer()
class Evaluator(object):
def __init__(self, loss=NLLLoss(), batch_size=64):
"""Class to initialize an evaluator
Args:
loss (seq2seq.loss, optional): loss for evaluator (default: seq2seq.loss.NLLLoss)
batch_size (int, optional): batch size for evaluator (default: 64)
"""
self.loss = loss
self.batch_size = batch_size
def evaluate(self, model, data):
""" Evaluate a model on given dataset and return performance.
Args:
model (seq2seq.models): model to evaluate
data (seq2seq.dataset.dataset.Dataset): dataset to evaluate against
Returns:
loss (float): loss of the given model on the given dataset
"""
def step(self, action):
"""
Take one step of interaction.
Parameters
----------
action: ndarray
action[0: smas_num] are action corresponding to sma
action[smas_num: end] are action corresponding to light color
Returns
-------
observation: ndarray
obervation of environment after taking an action
reward: float
reward after taking an action
done: bool
whether simulation is done or not.
info:
some information for debugging
"""
# To imporove system's robustness, we need to just ignore this action
# rather than throw an exception.
if np.sum(np.isnan(action)) != 0:
Tracer()()
raise ValueError("Find nan value in action!")
# Clip action to avoid improer action command
action = np.clip(action, self.act_min, self.act_max)
action = (action + 1) / 2.0
# Split action for light and sma
action = np.ndarray.flatten(action) # flatten array
action_smas = action[:self.smas_num]
action_lights_intensity = action[self.smas_num:]
# Taking action
#vrep.simxPauseCommunication(self.clientID,True) #temporarily halting the communication thread
self._set_all_joint_position(action_smas)
# Actually only set light color
self._set_all_light_intensity(action_lights_intensity)
#vrep.simxPauseCommunication(self.clientID,False) #and evaluated at the same time
# Set a small sleep time to avoid getting nan sensor data
time.sleep(0.01)
# Observe current state
try:
self.observation = self._self_observe()
except ValueError:
self._nanObervationFlag = 1
logging.error("Observation has NAN value.")
# This reward is non-interactive reward i.e. it's not affected by visitor.
# Therefore, it's only used for tunning hyper-parameters of LASAgent
if self.reward_function_type == 'occupancy':
# 1. 'IR_distance': based on IR distance from detected object to IR
# 2. 'IR_state_ratio': the ratio of # of detected objects and all #
# of IR sensors
# 3. 'IR_state_number': the number of detected objects
self.reward = self._reward_occupancy(self.observation,
'IR_distance')
else:
raise ValueError('No reward function: {}'.format(
self.reward_function_type))
done = False
info = []
return self.observation, self.reward, done, info