def shell():
"""Runs a shell in the app context.
Runs an interactive Python shell in the context of a given
Flask application. The application will populate the default
namespace of this shell according to it's configuration.
This is useful for executing small snippets of management code
without having to manually configuring the application.
"""
import IPython
from flask.globals import _app_ctx_stack
app = _app_ctx_stack.top.app
banner = 'Python %s on %s\nIPython: %s\nApp: %s%s\nInstance: %s\n' % (
sys.version,
sys.platform,
IPython.__version__,
app.import_name,
app.debug and ' [debug]' or '',
app.instance_path,
)
ctx = {}
# Support the regular Python interpreter startup script if someone
# is using it.
startup = os.environ.get('PYTHONSTARTUP')
if startup and os.path.isfile(startup):
with open(startup, 'r') as f:
eval(compile(f.read(), startup, 'exec'), ctx)
ctx.update(app.make_shell_context())
IPython.embed(banner1=banner, user_ns=ctx)
def impute_data(df, cohort):
#import IPython
#IPython.embed()
if isinstance(df, str):
df = ml.read_data(df)
#########################
## IMPUTE MISSING DATA ##
#########################
print "Imputing missing data..."
#change msam to missing is msam_NA==1
nanList = ['g6_g6msam_nan', 'g7_g7msam_nan', 'g8_g8msam_nan', 'g9_g8msam_nan']
varList = [[ 'g6_g6msam_Advanced', 'g6_g6msam_Basic', 'g6_g6msam_Proficient'], ['g7_g7msam_Advanced', 'g7_g7msam_Basic', 'g7_g7msam_Proficient'], ['g8_g8msam_Advanced', 'g8_g8msam_Basic', 'g8_g8msam_Proficient'],['g9_g8msam_Advanced', 'g9_g8msam_Basic', 'g9_g8msam_Proficient']]
for x in range(0,len(nanList)):
nacol = nanList[x]
colList = varList[x]
for col in colList:
df.loc[df[nacol] == 1, col] = np.nan
#pred missing data using any available data
wordList = ['absrate', 'mapr', 'msam_Advanced', 'msam_Basic', 'msam_Proficient', 'mobility', 'nsusp', 'mpa', 'tardyr', 'psatm', 'psatv', 'retained']
for word in wordList:
colList = [col for col in df.columns if word in col]
rowMean = df[colList].mean(axis=1)
for col in colList:
print df[col].value_counts(dropna=False)
df.loc[:,col].fillna(rowMean, inplace=True)
print df[col].value_counts(dropna=False)
'''
############################
# IMPUTE NEIGHBORHOOD DATA #
############################
print "Imputing missing school neighborhood data..."
## Fill missing school neighborhood data
print "Fixing neighborhood columns..."
neighborhood_cols = ['suspensionrate', 'mobilityrateentrantswithdra', 'attendancerate', 'avg_class_size', 'studentinstructionalstaffratio', 'dropoutrate', 'grade12documenteddecisionco', 'grade12documenteddecisionem', 'grade12documenteddecisionmi', 'grad12docdec_col_emp', 'graduationrate', 'studentsmeetinguniversitysyste', 'Est_Households_2012', 'Est_Population_2012', 'Med_Household_Income_2012', 'Mean_Household_Income_2012', 'Pop_Below_Poverty_2012', 'Percent_Below_Poverty_2012', 'Pop_Under18_2012', 'Under18_Below_Poverty_2012', 'Under18_Below_Poverty_Percent_2012', 'Housholds_on_Food_stamps_with_Children_Under18_2012', 'Housholds_Pop_on_Food_Stamps_2012', 'Pop_BlackAA_2012', 'Pop_White_2012', 'Bt_18_24_percent_less_than_High_School_2012', 'Bt_18_24_percent_High_School_2012', 'Bt_18_24_percent_Some_College_or_AA_2012', 'Bt_1824_percent_BA_or_Higher_2012', 'Over_25_percent_less_than_9th_grade_2012', 'Over_25_percent_9th_12th_2012', 'Over_25_percent_High_School_2012', 'Over_25__percent_Some_College_No_Deg_2012', 'Over_25_percent_AA_2012', 'Over_25_percent_Bachelors_2012', 'Over_25_percent_Graduate_or_Professionals_2012']
ml.replace_with_mean(df, neighborhood_cols)
'''
#summary = ml.summarize(df)
#print summary.T
#ml.print_to_csv(summary.T, 'updated_summary_stats_vertical.csv')
return_file = '/mnt/data2/education_data/mcps/DATA_DO_NOT_UPLOAD/imputed_data_cohort' + str(cohort) + '.csv'
ml.print_to_csv(df, return_file)
#IPython.embed()
print "Done!"
import IPython
IPython.embed()
return df
def transform_data():
from solaris.run import load_data
from sklearn.externals import joblib
data = load_data('data/data.pkl')
kringing = PertubatedKriging()
#kringing = PertubatedSpline()
data['description'] = '%r: %r' % (kringing, kringing.est)
print data['description']
print('_' * 80)
print(kringing)
print
for key in ['train', 'test']:
print('_' * 80)
print('transforming %s' % key)
print
X = data['X_%s' % key]
X = kringing.fit_transform(X)
data['X_%s' % key] = X
print
print('dumping data')
joblib.dump(data, 'data/interp10_data.pkl')
IPython.embed()
def corpus_sandbox(analysis_options):
print('Exploring SB-5 corpus')
session = create_session()
sb5b_documents = session.query(Document).join(Source).\
filter(Source.name == 'sb5b').all()
print('Found %d documents' % len(sb5b_documents))
rows = [dict(
label=document.label,
inferred=bool(document.details.get('Inferred')),
source=document.details.get('Source', 'NA')) for document in sb5b_documents]
df = pd.DataFrame.from_records(rows)
# df_agg = df.groupby(['label', 'inferred'])
# df.pivot_table(values=['label'], rows=['inferred'], aggfunc=[len])
df.pivot_table(rows=['label', 'inferred'], aggfunc=[len])
df.pivot_table(rows=['label', 'source'], aggfunc=[len])
df.pivot_table(rows=['source'], aggfunc=[len])
# df_agg.plot(x='train', y='accuracy')
for document in sb5b_documents:
# 'weareohio' in document.document.lower(), .document
print(document.details.get('Source'), document.label)
IPython.embed()
def user_console(user_email_address):
with global_session_scope() as db_session:
account = db_session.query(Account).filter_by(
email_address=user_email_address).one()
if account.provider == 'eas':
banner = """
You can access the account instance with the 'account' variable.
"""
else:
with writable_connection_pool(account.id, pool_size=1).get()\
as crispin_client:
if account.provider == 'gmail' \
and 'all' in crispin_client.folder_names():
crispin_client.select_folder(
crispin_client.folder_names()['all'][0],
uidvalidity_cb)
banner = """
You can access the crispin instance with the 'crispin_client' variable,
and the account instance with the 'account' variable.
IMAPClient docs are at:
http://imapclient.readthedocs.org/en/latest/#imapclient-class-reference
"""
IPython.embed(banner1=banner)
def _aggregate_batch(data_holder, use_list=False):
size = len(data_holder[0])
result = []
for k in range(size):
if use_list:
result.append(
[x[k] for x in data_holder])
else:
dt = data_holder[0][k]
if type(dt) in [int, bool]:
tp = 'int32'
elif type(dt) == float:
tp = 'float32'
else:
try:
tp = dt.dtype
except AttributeError:
raise TypeError("Unsupported type to batch: {}".format(type(dt)))
try:
result.append(
np.asarray([x[k] for x in data_holder], dtype=tp))
except Exception as e: # noqa
logger.exception("Cannot batch data. Perhaps they are of inconsistent shape?")
if isinstance(dt, np.ndarray):
s = pprint.pformat([x[k].shape for x in data_holder])
logger.error("Shape of all arrays to be batched: " + s)
try:
# open an ipython shell if possible
import IPython as IP; IP.embed() # noqa
except ImportError:
pass
return result
def remove_convex(self, start_state = None, peg = None):
left_peg = start_state.get_left_peg(peg)
right_peg = start_state.get_right_peg(peg)
new_contour_pegs = self.convex_hull(start_state, peg, left_peg, right_peg)
pegs_to_be_added = []
for elem in new_contour_pegs:
if (elem != left_peg) and (elem != right_peg):
pegs_to_be_added.append(elem)
prev_inside = start_state.inside
prev_contour = start_state.contour
new_outside = start_state.outside[:]
new_outside.append(peg)
new_inside = []
for elem in prev_inside:
if elem not in pegs_to_be_added:
new_inside.append(elem)
list_of_new_pegs = []
for elem in prev_contour.peg_order:
if elem != peg:
list_of_new_pegs.append(prev_contour.peg_dict[elem])
else:
for new_peg in pegs_to_be_added:
list_of_new_pegs.append(Peg(new_peg, True, 0))
end_state = State(list_of_new_pegs, new_inside, new_outside)
assert len(new_inside + new_outside + end_state.contour.peg_order) == 12
IPython.embed()
return end_state
def test_mapping_cost(
self,
other,
bend_coef=DEFAULT_LAMBDA[1],
outlierprior=1e-1,
outlierfrac=1e-2,
outliercutoff=1e-2,
T=5e-3,
norm_iters=DEFAULT_NORM_ITERS,
):
mapping_err = self.mapping_cost(other, outlierprior, outlierfrac, outliercutoff, T, norm_iters)
for i in range(self.N):
## compute error for 0 on cpu
s_gpu = mapping_err[i]
s_cpu = np.float32(0)
xt = self.pts_t[i].get()
xw = self.pts_w[i].get()
yt = other.pts_t[i].get()
yw = other.pts_w[i].get()
##use the trace b/c then numpy will use float32s all the way
s_cpu += np.trace(xt.T.dot(xt) + xw.T.dot(xw) - 2 * xw.T.dot(xt))
s_cpu += np.trace(yt.T.dot(yt) + yw.T.dot(yw) - 2 * yw.T.dot(yt))
if not np.isclose(s_cpu, s_gpu, atol=1e-4):
## high err tolerance is b/c of difference in cpu and gpu precision?
print "cpu and gpu sum sq differences differ!!!"
ipy.embed()
sys.exit(1)
def test_bending_cost(
self,
other,
bend_coef=DEFAULT_LAMBDA[1],
outlierprior=1e-1,
outlierfrac=1e-2,
outliercutoff=1e-2,
T=5e-3,
norm_iters=DEFAULT_NORM_ITERS,
):
self.get_target_points(other, outlierprior, outlierfrac, outliercutoff, T, norm_iters)
self.update_transform(bend_coef)
bending_costs = self.bending_cost(bend_coef)
for i in range(self.N):
c_gpu = bending_costs[i]
k_nn = self.kernels[i].get()
w_nd = self.w_nd[i].get()
c_cpu = np.float32(0)
for d in range(DATA_DIM):
r = np.dot(k_nn, w_nd[:, d]).astype(np.float32)
r = np.float32(np.dot(w_nd[:, d], r))
c_cpu += r
c_cpu *= np.float32(bend_coef)
if np.abs(c_cpu - c_gpu) > 1e-4:
## high err tolerance is b/c of difference in cpu and gpu precision?
print "cpu and gpu bend costs differ!!!"
ipy.embed()
sys.exit(1)
def pr2_flashlidar():
env = rave.Environment()
#env.Load('envs/testpr2sensors.env.xml')
env.Load('envs/pr2-table.env.xml')
env.SetViewer('qtcoin')
time.sleep(1)
start_time = time.time()
sensors = [s for s in env.GetSensors() if s.GetName().find("flashlidar") != -1]
lidar = sensors[0]
lidar.Configure(Sensor.ConfigureCommand.PowerOn)
#lidar.Configure(Sensor.ConfigureCommand.RenderDataOn)
while True:
start_time = time.time()
olddata = lidar.GetSensorData(Sensor.Type.Laser)
while True:
data = lidar.GetSensorData(Sensor.Type.Laser)
if data.stamp != olddata.stamp:
break
time.sleep(0.1)
print('Elapsed: {0}'.format(time.time() - start_time))
break
lidar.Configure(Sensor.ConfigureCommand.PowerOff)
#lidar.Configure(Sensor.ConfigureCommand.RenderDataOff)
IPython.embed()
def check_update(ctx, b):
ctx.tps_params[0] = ctx.default_tps_params.copy()
ctx.update_ptrs()
xt = ctx.pts_t[0].get()
p_mat = ctx.proj_mats[b][0].get()
o_mat = ctx.offset_mats[b][0].get()
true_res = np.dot(p_mat, xt) + o_mat
ctx.set_tps_params(ctx.offset_mats[b])
o_gpu = ctx.tps_params[0].get()
if not np.allclose(o_gpu, o_mat):
print "setting tps params failed"
diff = np.abs(o_mat - o_gpu)
nz = np.nonzero(diff)
print nz
ipy.embed()
sys.exit(1)
ctx.update_transform(b)
p1 = ctx.tps_params[0].get()
if not np.allclose(true_res, p1):
print "p1 and true res differ"
print p1[:3]
diff = np.abs(p1 - true_res)
print np.max(diff)
amax = np.argmax(diff)
print amax
nz = np.nonzero(diff)
print nz[0]
ipy.embed()
sys.exit(1)
def pr2_sensors():
env = rave.Environment()
#env.Load('robots/pr2-beta-sim.robot.xml')
env.Load('envs/testpr2sensors.env.xml')
r = env.GetRobots()[0]
env.SetViewer('qtcoin')
ienablesensor = 0
while True:
start_time = time.time()
sensors = env.GetSensors()
for i,sensor in enumerate(sensors):
if i==ienablesensor:
sensor.Configure(Sensor.ConfigureCommand.PowerOn)
sensor.Configure(Sensor.ConfigureCommand.RenderDataOn)
else:
sensor.Configure(Sensor.ConfigureCommand.PowerOff)
sensor.Configure(Sensor.ConfigureCommand.RenderDataOff)
print 'showing sensor %s, try moving obstacles'%(sensors[ienablesensor].GetName())
if sensors[ienablesensor].Supports(Sensor.Type.Laser):
# if laser, wait for the sensor data to be updated and then print it
olddata = sensors[ienablesensor].GetSensorData(Sensor.Type.Laser)
while True:
data = sensors[ienablesensor].GetSensorData(Sensor.Type.Laser)
if data.stamp != olddata.stamp:
break
time.sleep(0.1)
print 'sensor data: ',data.ranges
#time.sleep(5)
ienablesensor = (ienablesensor+1)%len(sensors)
print('Elapsed: {0}'.format(time.time() - start_time))
IPython.embed()
def eih():
env = rave.Environment()
env.Load('data/testwamcamera.env.xml')
env.SetViewer('qtcoin')
ienablesensor = 0
while True:
sensors = env.GetSensors()
for i,sensor in enumerate(sensors):
if i==ienablesensor:
sensor.Configure(Sensor.ConfigureCommand.PowerOn)
sensor.Configure(Sensor.ConfigureCommand.RenderDataOn)
else:
sensor.Configure(Sensor.ConfigureCommand.PowerOff)
sensor.Configure(Sensor.ConfigureCommand.RenderDataOff)
print 'showing sensor %s, try moving obstacles'%(sensors[ienablesensor].GetName())
if sensors[ienablesensor].Supports(Sensor.Type.Laser):
# if laser, wait for the sensor data to be updated and then print it
olddata = sensors[ienablesensor].GetSensorData(Sensor.Type.Laser)
while True:
data = sensors[ienablesensor].GetSensorData(Sensor.Type.Laser)
if data.stamp != olddata.stamp:
break
time.sleep(0.1)
print 'sensor data: ',data.ranges
time.sleep(5)
ienablesensor = (ienablesensor+1)%len(sensors)
IPython.embed()
def fileToHash(filename):
try:
assert(isfile(filename))
except:
IPython.embed(simple_prompt=True)
with open(filename, 'rb') as f:
return sha256(f.read()).hexdigest()
def main():
"""
program starting point
"""
shell = False
atomic = False
try:
optlist, _ = getopt.getopt(sys.argv[1:], 'p:n:', ["shell", "atomic"])
optdict = dict(optlist)
prefix = optdict['-p']
if '-' in prefix:
raise ValueError('"-" cannot exist in prefix=%r' % prefix)
num_workers = int(optdict['-n'])
if "--shell" in optdict:
shell = True
if "--atomic" in optdict:
atomic = True
except Exception:
print traceback.format_exc()
usage()
sys.exit(1)
orchestrator = Orchestrator(prefix, num_workers)
if shell:
# give me a ipython shell
IPython.embed()
return
orchestrator.start(atomic)
def segment(self):
# Specify segments and index
exit = False
i = 1
while not exit:
user_ret = raw_input("Enter new segment name: ")
self.map_index_labels[i] = str(user_ret)
self.map_index_data[i] = []
i += 1
user_ret = raw_input("Done with specifing segments?[y/n]")
if user_ret == 'y':
exit = True
print "----------------Collecting Segments ----------------------"
exit = False
while not exit:
self.print_all_index_labels()
index = int(raw_input("Which index?"))
start_frm = int(raw_input("Starting frame?"))
end_frm = int(raw_input("End frame?"))
new_segment = (start_frm, end_frm)
segment_list = self.map_index_data[index]
segment_list.append(new_segment)
self.map_index_data[index] = segment_list
user_ret = raw_input("Done with specifing segments?[y/n]")
if user_ret == 'y':
exit = True
IPython.embed()
def pause(self, message="Pause"):
if hasattr(self, 'services'):
self.services.start_interactive_mode()
import IPython
IPython.embed()
if hasattr(self, 'services'):
self.services.stop_interactive_mode()
def ArrayToCode(array, codes, code_lengths):
code_lengths = np.array(code_lengths, dtype = np.int32)
total_length = np.sum(code_lengths[array])
total_4bytes = (total_length - 1) / 32 + 1
compressed_codes= np.zeros(total_4bytes, dtype = np.uint32)
idx = 0
shift = 0
for i in range(len(array)):
number = array[i]
length = code_lengths[number]
code = codes[number]
while length > 0:
eff = min(length, 32-shift)
bits_to_write = code & ((1 << eff) - 1)
if idx == total_4bytes:
print i
import IPython
IPython.embed()
compressed_codes[idx] += bits_to_write << shift
code = code >> eff
length -= eff
idx += (shift + eff) / 32
shift = (shift + eff) % 32
assert idx * 32 + shift == total_length # For debug
return compressed_codes, total_length
def test(model, dataset, weights_filepath=BEST_WEIGHT_FILE):
model.load_weights(weights_filepath)
train_iterator = dataset.iterator(batch_size=batch_size,
num_batches=nb_test_batches)
batch_x, batch_y = train_iterator.next()
results_dir = 'results'
if not os.path.exists(results_dir):
os.makedirs(results_dir)
pred = model._predict(batch_x)
pred = pred.reshape(batch_size, patch_size, 1, patch_size, patch_size)
#pred_as_b012c = pred.transpose(0, 3, 4, 1, 2)
# for i in range(batch_size):
# v, t = mcubes.marching_cubes(pred_as_b012c[i, :, :, :, 0], 0.5)
# mcubes.export_mesh(v, t, results_dir + '/drill_' + str(i) + '.dae', 'drill')
# viz.visualize_batch_x(pred, i, str(i), results_dir + "/pred_" + str(i))
# viz.visualize_batch_x(batch_x, i, str(i), results_dir + "/input_" + str(i))
# viz.visualize_batch_x(batch_y, i, str(i), results_dir + "/expected_" + str(i))
for i in range(batch_size):
viz.visualize_batch_x_y_overlay(batch_x, batch_y, pred, i=i, title=str(i))
# viz.visualize_batch_x(pred, i, 'pred_' + str(i), )
# viz.visualize_batch_x(batch_x, i,'batch_x_' + str(i), )
# viz.visualize_batch_x(batch_y, i, 'batch_y_' + str(i), )
import IPython
IPython.embed()
def some_func(x, y): try: return x / y except: IPython.embed() raise
def update_alpha_beta_gamma(self): # Step 6
# At the moment, we can only fix C0 = 0
if self.parDict['CFit'] == 'fixed' and self.parDict['C0'] == 0.0:
X = np.ones((self.N, 2), dtype=float)
X[:, 1] = self.xi
# Eqn (77)
XTXinv = np.linalg.inv(np.dot(X.T, X))
Sigma_chat = XTXinv * self.sigsqr
# Eqn (76)
chat = np.dot(np.dot(XTXinv, X.T), self.eta)
# Eqn (75)
self.alpha, self.beta = self.rng.multivariate_normal(chat, Sigma_chat)
self.gamma=0.
else:
# This doesn't work if any parameters fixed
X = np.ones((self.N, 3), dtype=float)
X[:, 1] = self.xi
X[:, 2] = self.z
# Eqn (77)
XTXinv = np.linalg.inv(np.dot(X.T, X))
Sigma_chat = XTXinv * self.sigsqr
# Eqn (76)
chat = np.dot(np.dot(XTXinv, X.T), self.eta)
# Eqn (75)
try:
self.alpha, self.beta, self.gamma = self.rng.multivariate_normal(chat, Sigma_chat)
except:
print "multivariate_normal fail"
IPython.embed()
sys.exit()
def do_merge(self):
import cv2
import numpy as np
images = [cv2.imread(path.join(BUILD_FULL_PATH, graph)) for graph in self.graphs]
shape = self.rows, self.cols
try:
widths = np.array([img.shape[1] for img in images]).reshape(shape).max(axis=0)
heights = np.array([img.shape[0] for img in images]).reshape(shape).max(axis=1)
except AttributeError:
import IPython
IPython.embed()
width = widths.sum()
height = heights.sum()
merge_img = np.zeros((height, width, 3), np.uint8)
cumsum_width = np.r_[0, np.cumsum(widths)]
cumsum_height = np.r_[0, np.cumsum(heights)]
for i, img in enumerate(images):
row = i // self.cols
col = i % self.cols
y = cumsum_height[row]
x = cumsum_width[col]
h, w = img.shape[:2]
merge_img[y:y+h, x:x+w] = img
cv2.imwrite(path.join(BUILD_FULL_PATH, self.graphs[-1].replace(".png", ".merge.png")), merge_img)
def validExtend(self, start_config, end_config):
segmentNum = 10
IPython.embed()
x1=numpy.linspace(start_config[0],end_config[0],segmentNum)
x2=numpy.linspace(start_config[1],end_config[1],segmentNum)
x3=numpy.linspace(start_config[2],end_config[2],segmentNum)
x4=numpy.linspace(start_config[3],end_config[3],segmentNum)
x5=numpy.linspace(start_config[4],end_config[4],segmentNum)
x6=numpy.linspace(start_config[5],end_config[5],segmentNum)
x7=numpy.linspace(start_config[6],end_config[6],segmentNum)
for i in xrange(0,len(x1)):
newconfig = [x1[i], x2[i], x3[i], x4[i], x5[i], x6[i], x7[i]]
self.robot.SetActiveDOFValues(newconfig)
env = self.robot.GetEnv()
#check collision
isCollision = env.CheckCollision(env.GetBodies()[0],env.GetBodies()[1])
isSelfCollision = self.robot.CheckSelfCollision()
if(isCollision==True or isSelfCollision==True):
return False
return True
def python_shell(options):
logger = setup_logger("Robot", debug=options.verbose)
def conn_callback(*args):
sys.stdout.write(".")
sys.stdout.flush()
return True
if options.shell == "ipython":
import IPython
else:
import importlib
sys.path.append(os.path.abspath(""))
module_name, entrance_name = options.shell.rsplit(".", 1)
module_instance = importlib.import_module(module_name)
entrance = module_instance.__getattribute__(entrance_name)
robot, device = connect_robot_helper(options.target, options.clientkey)
if options.shell == "ipython":
logger.info("----> READY")
logger.info("""
* Hint: Try 'robot?' and 'dir(robot)' to get more informations)\n""")
IPython.embed()
return 0
else:
return entrance(robot, device)
def ipython_drop(msg, glo, loc):
"""
Add this function in your code to drop to an ipython shell
(provided that glVar.shell == True)
msg: informative message to print when shell is invoked
glo: globals()
loc: locals()
"""
import IPython
banner = 'Dropping into IPython, type CTR-D to exit' + msg
try:
# Try pre-0.11 syntax first
args = ['-pi1','In <\\#>: ','-pi2',' .\\D.: ',
'-po','Out<\\#>: ','-nosep']
ipshell = IPython.Shell.IPShellEmbed(
args,
banner = banner,
exit_msg = 'Leaving Interpreter, back to program.')
ipshell(global_ns = glo, local_ns = loc)
except AttributeError:
# try the new syntax: post-0.11
#from IPython.config.loader import Config
#cfg = Config()
# directly open the shell
IPython.embed(user_ns=loc, banner2=banner)
def expand_mlb_data(infile_data, pitcher=False, live=False):
new_feature_dataframes = [infile_data]
expanded_columns = []
for feature_name, (func, columns) in get_expansion_targets(pitcher=pitcher, expanding_live=live):
with Timer() as t:
print 'Expanding', feature_name, '(' + ', '.join(columns) + ')...'
raw_data = [func(row) for index, row in infile_data.iterrows()]
raw_columns = encode_names(feature_name, columns)
expanded_columns += raw_columns
try:
new_feature_data = pd.DataFrame(raw_data,
index=infile_data.index,
columns=raw_columns)
except AssertionError as ex:
print 'Debugging assertion error -- probably no data for some featurizer was loaded. ' \
'I suspect Numberfire scraping needs to happen!'
import IPython
IPython.embed()
except TypeError as ex:
print 'Debugging type error'
import IPython
IPython.embed()
new_feature_dataframes.append(new_feature_data)
print ' took %d seconds' % t.elapsed
expanded_data = pd.concat(new_feature_dataframes, axis=1)
# After doing all of that concatenation the index is super weird so just reset it
expanded_data.reset_index(drop=True, inplace=True)
# Transform categorical variables to indicator variables -- but only for expanded discrete columns.
# May need to tweak how this list is generated in the future.
categorical_cols = [c for c in expanded_columns if expanded_data[c].dtype.name == 'object']
expanded_discretized = pd.get_dummies(expanded_data, prefix_sep='=', columns=categorical_cols)
return expanded_discretized
def cookie_save(self):
t = time.strftime('%Y%m%dT%H%M')
import IPython; IPython.embed()
_f = '%s_cookiejar.pickle' % t
with open(_f, 'wb') as f:
pickle.dump(self.cj, f, pickle.HIGHEST_PROTOCOL)
print('File save on %s' % _f)
def test_window_distance(width, num_steps, plot=None):
import sdf_file, obj_file
np.random.seed(100)
mesh_file_name = 'data/test/meshes/Co_clean.obj'
sdf_3d_file_name = 'data/test/sdf/Co_clean.sdf'
sdf = sdf_file.SdfFile(sdf_3d_file_name).read()
mesh = obj_file.ObjFile(mesh_file_name).read()
graspable = GraspableObject3D(sdf, mesh)
grasp_axis = np.array([0, 1, 0])
grasp_width = 0.1
grasp1_center = np.array([0, 0, -0.025])
grasp1 = g.ParallelJawPtGrasp3D(grasp1_center, grasp_axis, grasp_width)
grasp2_center = np.array([0, 0, -0.030])
grasp2 = g.ParallelJawPtGrasp3D(grasp2_center, grasp_axis, grasp_width)
w1, w2 = graspable.surface_information(grasp1, width, num_steps)
v1, v2 = graspable.surface_information(grasp2, width, num_steps)
# IPython.embed()
if plot:
plot(w1.proj_win, num_steps)
plot(w2.proj_win, num_steps)
plot(v1.proj_win, num_steps)
plot(v2.proj_win, num_steps)
plt.show()
IPython.embed()
return
def ipython(user_ns=None):
try:
import IPython
from IPython.config.loader import Config
except ImportError:
return simple_repl(user_ns=user_ns)
defns = {'os':os, 're':re, 'sys':sys}
if not user_ns:
user_ns = defns
else:
defns.update(user_ns)
user_ns = defns
c = Config()
c.InteractiveShellApp.exec_lines = [
'from __future__ import division, absolute_import, unicode_literals, print_function',
]
c.TerminalInteractiveShell.confirm_exit = False
c.PromptManager.in_template = (r'{color.LightGreen}calibre '
'{color.LightBlue}[{color.LightCyan}%s{color.LightBlue}]'
r'{color.Green}|\#> '%get_version())
c.PromptManager.in2_template = r'{color.Green}|{color.LightGreen}\D{color.Green}> '
c.PromptManager.out_template = r'<\#> '
c.TerminalInteractiveShell.banner1 = BANNER
c.PromptManager.justify = True
c.TerminalIPythonApp.ipython_dir = ipydir
os.environ['IPYTHONDIR'] = ipydir
c.InteractiveShell.separate_in = ''
c.InteractiveShell.separate_out = ''
c.InteractiveShell.separate_out2 = ''
c.PrefilterManager.multi_line_specials = True
IPython.embed(config=c, user_ns=user_ns)
def main():
app = QApplication(sys.argv)
app.quitOnLastWindowClosed = True
form = PlotDialog(app)
form.show()
IPython.embed()
app.exit()
def gen_mat(nx, ny, nth):
dx = 1 / nx
dy = 1 / ny
xx = np.linspace(0, 1, nx)
yy = np.linspace(0, 1, ny)
# Made up shape. Little kelp on top, bulge near middle, zero at bottom.
kelp_lengths = 5 * (1 - yy)**2 * np.exp(5 * yy - 4)
# Number of individual kelps in each depth layer - more towards the top
ind = 2 - yy
# Assume that kelp and water scatter the same,
# but kelp absorbs much more light
# Choose these to create diagonal dominance
# non-DDOM coefs:
#sct_water = 5
#sct_kelp = 5
#abs_water = 1
#abs_kelp = 5
# DDOM coefs:
sct_water = 4 * nx
sct_kelp = 4 * nx
abs_water = 2 * np.pi * sct_water
abs_kelp = 2 * abs_water
iops = [vsf, abs_water, sct_water, abs_kelp, sct_kelp]
scenario = gm2.KelpScenario(surf_bc_fun, iops)
scenario.set_kelp(kelp_lengths, ind)
scenario.set_num_grid_points(nx, ny, nth)
scenario.calculate_pk()
# What to do
gen_sparsity_plots = True
interactive_load_mat = False
plot_kelp = False
plot_irrad = False
print("{}x{}x{}".format(nx, ny, nth))
if gen_sparsity_plots:
# Loop through all possible variable orderings
# Only generate 210 matrices for now
for ii, var_order in ([0,
[2, 1,
0]], ): #enumerate(it.permutations(range(3))):
print()
print("ii={}: {}".format(ii, var_order))
# Determine common name for files
# kelp1_[variable dimensions]_[variable order]
name = ('ddom_{}x{}x{}_{}{}{}'.format(nx, ny, nth, *var_order))
print("Creating matrix")
scenario.gen_rte_matrix(var_order)
print("Saving files")
# Solve system & plot result
print("Solving system")
scenario.solve_system()
scenario.reshape_rad()
print("Calculating irradiance")
scenario.calc_irrad()
scenario.plot_irrad('../img/irrad/irrad_' + name + '.png')
# Save mat file
scenario.write_rte_system_mat('../mat/' + name)
# Save sparsity plots - one coarse (spy) & one precise (int)
#scenario.write_int_matrix_png('../img/sparsity/int_'+name+'.png')
#scenario.plot_rte_matrix('../img/sparsity/spy_'+name+'.png')
if plot_kelp:
print("Plotting kelp")
plt.figure(1)
scenario.plot_kelp('../img/solve/kelp.png')
if plot_irrad:
print("Creating matrix")
plt.figure(2)
scenario.calculate_rte_matrix()
#scenario.write_int_matrix_png('solve/sparsity.png')
print("Solving system")
scenario.solve_system()
print("Calculating irradiance")
scenario.calc_irrad()
scenario.plot_irrad('../img/solve/irrad.png')
print("Done!")
if interactive_load_mat:
print("Loading matrix")
scenario.load_rte_system_mat('../mat/kelp1_50x50x32_012.mat',
[0, 1, 2])
print("Finished loading")
IPython.embed()
def main(argv):
parser = ArgumentParser(usage=__doc__.lstrip())
parser.add_argument("--verbose", "-v", action="count", default=1,
help="more verbosity")
parser.add_argument("--no-build", "-n", action="store_true", default=False,
help="do not build the project (use system installed version)")
parser.add_argument("--build-only", "-b", action="store_true", default=False,
help="just build, do not run any tests")
parser.add_argument("--doctests", action="store_true", default=False,
help="Run doctests in module")
parser.add_argument("--coverage", action="store_true", default=False,
help=("report coverage of project code. HTML output goes "
"under build/coverage"))
parser.add_argument("--mode", "-m", default="fast",
help="'fast', 'full', or something that could be "
"passed to nosetests -A [default: fast]")
parser.add_argument("--submodule", "-s", default=None,
help="Submodule whose tests to run (cluster, constants, ...)")
parser.add_argument("--pythonpath", "-p", default=None,
help="Paths to prepend to PYTHONPATH")
parser.add_argument("--tests", "-t", action='append',
help="Specify tests to run")
parser.add_argument("--python", action="store_true",
help="Start a Python shell with PYTHONPATH set")
parser.add_argument("--ipython", "-i", action="store_true",
help="Start IPython shell with PYTHONPATH set")
parser.add_argument("--shell", action="store_true",
help="Start Unix shell with PYTHONPATH set")
parser.add_argument("--debug", "-g", action="store_true",
help="Debug build")
parser.add_argument("args", metavar="ARGS", default=[], nargs=REMAINDER,
help="Arguments to pass to Nose")
args = parser.parse_args(argv)
if args.pythonpath:
for p in reversed(args.pythonpath.split(os.pathsep)):
sys.path.insert(0, p)
if not args.no_build:
site_dir = build_project(args)
sys.path.insert(0, site_dir)
os.environ['PYTHONPATH'] = site_dir
if args.python:
import code
code.interact()
sys.exit(0)
if args.ipython:
import IPython
IPython.embed()
sys.exit(0)
if args.shell:
shell = os.environ.get('SHELL', 'sh')
print("Spawning a Unix shell...")
os.execv(shell, [shell])
sys.exit(1)
extra_argv = args.args
if args.coverage:
dst_dir = os.path.join('build', 'coverage')
fn = os.path.join(dst_dir, 'coverage_html.js')
if os.path.isdir(dst_dir) and os.path.isfile(fn):
shutil.rmtree(dst_dir)
extra_argv += ['--cover-html',
'--cover-html-dir='+dst_dir]
if args.build_only:
sys.exit(0)
elif args.submodule:
modname = PROJECT_MODULE + '.' + args.submodule
try:
__import__(modname)
test = sys.modules[modname].test
except (ImportError, KeyError, AttributeError):
print("Cannot run tests for %s" % modname)
sys.exit(2)
elif args.tests:
def test(*a, **kw):
extra_argv = kw.pop('extra_argv', ())
extra_argv = extra_argv + args.tests[1:]
kw['extra_argv'] = extra_argv
from numpy.testing import Tester
return Tester(args.tests[0]).test(*a, **kw)
else:
__import__(PROJECT_MODULE)
test = sys.modules[PROJECT_MODULE].test
result = test(args.mode,
verbose=args.verbose,
extra_argv=args.args,
doctests=args.doctests,
coverage=args.coverage)
if result.wasSuccessful():
sys.exit(0)
else:
sys.exit(1)
def from_argv(cls, argv=None, **kwargs):
prog = cls.server and 'irc3d' or 'irc3'
doc = """
Run an {prog} instance from a config file
Usage: {prog} [options] <config>...
Options:
-h, --help Display this help and exit
--version Output version information and exit
--logdir DIRECTORY Log directory to use instead of stderr
--logdate Show datetimes in console output
--host HOST Server name or ip
--port PORT Server port
-v,--verbose Increase verbosity
-r,--raw Show raw irc log on the console
-d,--debug Add some debug commands/utils
-i,--interactive Load a ipython console with a bot instance
""".format(prog=prog)
if not cls.server:
doc += """
--help-page Output a reST page containing commands help
""".strip()
import os
import docopt
import textwrap
args = argv or sys.argv[1:]
args = docopt.docopt(textwrap.dedent(doc), args, version=version)
cfg = utils.parse_config(
cls.server and 'server' or 'bot', *args['<config>'])
cfg.update(
verbose=args['--verbose'],
debug=args['--debug'],
)
cfg.update(kwargs)
if args['--host']: # pragma: no cover
host = args['--host']
cfg['host'] = host
if host in ('127.0.0.1', 'localhost'):
cfg['ssl'] = False
if args['--port']: # pragma: no cover
cfg['port'] = args['--port']
if args['--logdir'] or 'logdir' in cfg:
logdir = os.path.expanduser(args['--logdir'] or cfg.get('logdir'))
cls.logging_config = config.get_file_config(logdir)
if args['--logdate']: # pragma: no cover
fmt = cls.logging_config['formatters']['console']
fmt['format'] = config.TIMESTAMPED_FMT
if args.get('--help-page'): # pragma: no cover
for v in cls.logging_config['handlers'].values():
v['level'] = 'ERROR'
if args['--raw']:
cfg['raw'] = True
context = cls.from_config(cfg)
if args.get('--help-page'): # pragma: no cover
context.print_help_page()
elif args['--interactive']: # pragma: no cover
import IPython
IPython.embed()
sys.exit(0)
else:
context.run(forever=not bool(kwargs))
if kwargs or argv:
return context
paveALICE.AddText("5 < #it{p}_{T,ch jet} < 30 GeV/#it{c}")
paveALICE.AddText("|#eta_{jet}| < 0.5")
paveALICE.Draw()
return canvas
def main():
ROOT.TH1.AddDirectory(False)
ROOT.gStyle.SetOptTitle(0)
ROOT.gStyle.SetOptStat(0)
f = open("LHC15i2response_Train1399.yaml", 'r')
config_b = yaml.load(f)
f.close()
f = open("LHC15i2response_Train1399.yaml", 'r')
config_c = yaml.load(f)
f.close()
canvas = EfficiencyComparison(config_c, config_b)
canvas.SaveAs("{0}/Efficiency_Paper.pdf".format(config_c["input_path"]))
canvas.SaveAs("{0}/Efficiency_Paper.C".format(config_c["input_path"]))
if __name__ == '__main__':
main()
IPython.embed()
def main(argv=None):
if argv is None:
argv = sys.argv
try:
# default values
prog_name = argv[0]
ignore_lock = True
filename_template = None
filename_output = None
no_output = False
list_invoices = False
invoice_number = None
invoice_id = None
filename_from_invoice = False
output_path = None
with_ipshell = False
try:
opts, args = getopt.getopt(argv[1:], "fhliI:t:o:OP:", ["help"])
except getopt.error as msg:
raise Usage(msg)
for opt in opts:
if opt[0] in ["-f"]:
print("ignoring lock")
ignore_lock = True
if opt[0] in ["-h", "--help"]:
raise Usage("Help:")
if opt[0] in ["-I"]:
invoice_id = opt[1]
print("using invoice ID '" + str(invoice_id) + "'.")
if opt[0] in ["-i"]:
print("Using ipshell")
with_ipshell = True
if opt[0] in ["-o"]:
filename_output = opt[1]
print("using output file", filename_output)
if opt[0] in ["-O"]:
if filename_output:
print("given output filename will be overwritten,")
print("creating output filename from Invoice data.")
filename_from_invoice = True
if opt[0] in ["-t"]:
filename_template = opt[1]
print("using template file", filename_template)
if opt[0] in ["-l"]:
list_invoices = True
print("listing invoices")
if opt[0] in ["-P"]:
output_path = opt[1]
print("output path is", output_path + ".")
# Check for correct input
if len(args) > 1:
print("opts:", opts, "args:", args)
raise Usage("Only one input possible !")
if len(args) == 0:
raise Usage("No input given !")
input_url = args[0]
# Check for correct template
if not filename_template:
no_output = True
if not list_invoices:
raise Usage("No template given !")
# Check for output file
if not (filename_output or filename_from_invoice):
if filename_template:
filename_output = filename_template + ".out"
print("no output filename given, will be:", filename_output)
except Usage as err:
if err.msg == "Help:":
retcode = 0
else:
print("Error:", err.msg, file=sys.stderr)
print("for help use --help", file=sys.stderr)
retcode = 2
print()
print("Usage:")
print()
print("Invoke with", prog_name, "gnucash_url.")
print("where input is")
print(" filename")
print("or file://filename")
print("or mysql://user:password@host/databasename")
print()
print("-f force open = ignore lock")
print("-l list all invoices")
print("-h or --help for this help")
print("-I ID use invoice ID")
print("-t filename use filename as template file")
print("-o filename use filename as output file")
print(
"-O create output filename by date, owner and invoice number"
)
print(
"-P path path for output file. Overwrites path in -o option"
)
return retcode
# Try to open the given input
try:
print("Opening", input_url, ".")
session = gnucash.Session(input_url, ignore_lock=ignore_lock)
except Exception as exception:
print("Problem opening input.")
print(exception)
return 2
book = session.book
root_account = book.get_root_account()
comm_table = book.get_table()
EUR = comm_table.lookup("CURRENCY", "EUR")
invoice_list = get_all_invoices(book)
if list_invoices:
for number, invoice in enumerate(invoice_list):
print(str(number) + ")")
print(invoice)
if not (no_output):
if invoice_id:
invoice = book.InvoiceLookupByID(invoice_id)
if not invoice:
print("ID not found.")
return 2
if invoice_number:
invoice = invoice_list[invoice_number]
print("Using the following invoice:")
print(invoice)
path_template = os.path.dirname(filename_template)
filename_template_basename = os.path.basename(filename_template)
loader = jinja2.FileSystemLoader(path_template)
env = jinja2.Environment(loader=loader)
template = env.get_template(filename_template_basename)
#company = gnucash_business.Company(book.instance)
output = template.render(invoice=invoice,
locale=locale) #, company=company)
if filename_from_invoice:
filename_date = invoice.GetDatePosted().strftime(
"%Y-%m-%d") # something like 2014-11-01
filename_owner_name = str(invoice.GetOwner().GetName())
filename_invoice_id = str(invoice.GetID())
filename_output = filename_date + "_" + filename_owner_name + "_" + filename_invoice_id + ".tex"
if output_path:
filename_output = os.path.join(output_path,
os.path.basename(filename_output))
print("Writing output", filename_output, ".")
with open(filename_output, 'w') as f:
f.write(output)
if with_ipshell:
import IPython
IPython.embed()
def interact(mydict=None, argv=None, mybanner=None, loglevel=20):
global session
import code, sys, pickle, os, getopt, re
from .config import conf
conf.interactive = True
if loglevel is not None:
conf.logLevel = loglevel
the_banner = "Welcome to Scapy (%s)"
if mybanner is not None:
the_banner += "\n"
the_banner += mybanner
if argv is None:
argv = sys.argv
import atexit
try:
import rlcompleter, readline
except ImportError:
log_loading.info("Can't load Python libreadline or completer")
READLINE = 0
else:
READLINE = 1
class ScapyCompleter(rlcompleter.Completer):
def global_matches(self, text):
matches = []
n = len(text)
for lst in [dir(__builtin__), list(session.keys())]:
for word in lst:
if word[:n] == text and word != "__builtins__":
matches.append(word)
return matches
def attr_matches(self, text):
m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text)
if not m:
return
expr, attr = m.group(1, 3)
try:
object = eval(expr)
except:
object = eval(expr, session)
if isinstance(object, Packet) or isinstance(
object, Packet_metaclass):
words = [x for x in dir(object) if x[0] != "_"]
words += [x.name for x in object.fields_desc]
else:
words = dir(object)
if hasattr(object, "__class__"):
words = words + rlcompleter.get_class_members(
object.__class__)
matches = []
n = len(attr)
for word in words:
if word[:n] == attr and word != "__builtins__":
matches.append("%s.%s" % (expr, word))
return matches
readline.set_completer(ScapyCompleter().complete)
readline.parse_and_bind("C-o: operate-and-get-next")
readline.parse_and_bind("tab: complete")
session = None
session_name = ""
STARTUP_FILE = DEFAULT_STARTUP_FILE
PRESTART_FILE = DEFAULT_PRESTART_FILE
iface = None
try:
opts = getopt.getopt(argv[1:], "hs:Cc:Pp:d")
for opt, parm in opts[0]:
if opt == "-h":
_usage()
elif opt == "-s":
session_name = parm
elif opt == "-c":
STARTUP_FILE = parm
elif opt == "-C":
STARTUP_FILE = None
elif opt == "-p":
PRESTART_FILE = parm
elif opt == "-P":
PRESTART_FILE = None
elif opt == "-d":
conf.logLevel = max(1, conf.logLevel - 10)
if len(opts[1]) > 0:
raise getopt.GetoptError("Too many parameters : [%s]" %
" ".join(opts[1]))
except getopt.GetoptError as msg:
log_loading.error(msg)
sys.exit(1)
if PRESTART_FILE:
_read_config_file(PRESTART_FILE)
scapy_builtins = __import__("all", globals(), locals(), ".").__dict__
builtins.__dict__.update(scapy_builtins)
globkeys = list(scapy_builtins.keys())
globkeys.append("scapy_session")
scapy_builtins = None # XXX replace with "with" statement
if mydict is not None:
builtins.__dict__.update(mydict)
globkeys += list(mydict.keys())
conf.color_theme = DefaultTheme()
if STARTUP_FILE:
_read_config_file(STARTUP_FILE)
if session_name:
try:
os.stat(session_name)
except OSError:
log_loading.info("New session [%s]" % session_name)
else:
try:
try:
session = pickle.load(gzip.open(session_name, "rb"))
except IOError:
session = pickle.load(open(session_name, "rb"))
log_loading.info("Using session [%s]" % session_name)
except EOFError:
log_loading.error("Error opening session [%s]" % session_name)
except AttributeError:
log_loading.error(
"Error opening session [%s]. Attribute missing" %
session_name)
if session:
if "conf" in session:
conf.configure(session["conf"])
session["conf"] = conf
else:
conf.session = session_name
session = {"conf": conf}
else:
session = {"conf": conf}
builtins.__dict__["scapy_session"] = session
if READLINE:
if conf.histfile:
try:
readline.read_history_file(conf.histfile)
except IOError:
pass
atexit.register(scapy_write_history_file, readline)
atexit.register(scapy_delete_temp_files)
IPYTHON = False
if conf.interactive_shell.lower() == "ipython":
try:
import IPython
IPYTHON = True
except ImportError as e:
log_loading.warning(
"IPython not available. Using standard Python shell instead.")
IPYTHON = False
if IPYTHON:
banner = the_banner % (
conf.version) + " using IPython %s" % IPython.__version__
# Old way to embed IPython kept for backward compatibility
try:
args = [''] # IPython command line args (will be seen as sys.argv)
ipshell = IPython.Shell.IPShellEmbed(args, banner=banner)
ipshell(local_ns=session)
except AttributeError as e:
pass
# In the IPython cookbook, see 'Updating-code-for-use-with-IPython-0.11-and-later'
IPython.embed(user_ns=session, banner2=banner)
else:
code.interact(banner=the_banner % (conf.version),
local=session,
readfunc=conf.readfunc)
if conf.session:
save_session(conf.session, session)
for k in globkeys:
try:
del (builtins.__dict__[k])
except:
pass
def outer_main(barquery=None,
beerfile=None,
get_taps=True,
get_cans=False,
interactive=False,
**kwargs):
if barquery:
barname, bar_url = get_bar(barquery)
# beerlst, n_on_tap = get_beers(bar_url)
# d_beers_beermenus, n_on_tap = get_beers(bar_url)
d_beermenus = get_beers(bar_url)
kwargs['d_beermenus'] = d_beermenus
def is_served_as(beer, *args):
servingtypes = {
d['type'].lower()
for d in d_beermenus[beer]['serving']
}
return any(arg in servingtypes for arg in args)
is_on_tap = lambda beer: is_served_as(beer, 'draft', 'cask', 'crowler',
'growler')
is_bottled = lambda beer: is_served_as(beer, 'bottle', 'can')
has_no_servinginfo = lambda beer: not d_beermenus[beer]['serving']
beerlst = [
beer for beer in d_beermenus.keys()
if (is_on_tap(beer) or has_no_servinginfo(beer))
]
beerlst_rest = [
beer for beer in d_beermenus.keys() if is_bottled(beer)
]
if barname.lower(
) == 'covenhoven': # TODO drafts listed as bottle - eventually fix for good
beerlst = list(d_beermenus.keys())
beerlst_rest = []
else:
barname = beerfile.split('_')[-1]
beerlst = get_beers_from_file(beerfile)
# on_draft = get_beers_from_file(beerfile)
d_beermenus = {} #; n_on_tap = len(beerlst)
print('\n what\'s on @ {} ?? \n'.format(barname.upper()))
if beerfile or (barquery and get_taps):
# beerlst_taps = beerlst[:n_on_tap]
beerlst_taps = beerlst
d_beers1 = alternate_main(beerlst_taps,
with_key=(not get_cans),
**kwargs)
if barquery and get_cans:
print('\nCANS & BOTTLES...\n')
# beerlst_cans = beerlst[n_on_tap:]
beerlst_cans = beerlst_rest
d_beers2 = alternate_main(beerlst_cans, with_key=get_cans, **kwargs)
if interactive:
from itertools import chain
for k, v in chain(d_beers1.items(), d_beers2.items()):
d_beers = d_beermenus[k].update(**v)
import IPython
IPython.embed()
log_once("UniformDiscretizer1D: value smaller than min!")
return 0
if v > self.maxv:
log_once("UniformDiscretizer1D: value larger than max!")
return self.nr_bin - 1
return int(np.clip((v - self.minv) / self.spacing, 0, self.nr_bin - 1))
def get_distribution(self, v, smooth_factor=0.05, smooth_radius=2):
""" return a smoothed one-hot distribution of the sample v.
"""
b = self.get_bin(v)
ret = np.zeros((self.nr_bin, ), dtype='float32')
ret[b] = 1.0
if v >= self.maxv or v <= self.minv:
return ret
try:
for k in range(1, smooth_radius + 1):
ret[b + k] = smooth_factor**k
except IndexError:
pass
for k in range(1, min(smooth_radius + 1, b + 1)):
ret[b - k] = smooth_factor**k
ret /= ret.sum()
return ret
if __name__ == '__main__':
u = UniformDiscretizer1D(-10, 10, 0.12)
import IPython as IP
IP.embed(config=IP.terminal.ipapp.load_default_config())
def main():
import IPython
IPython.embed()
import sys
from whoosh.index import open_dir
import IPython
try:
ix = open_dir('ex_index')
print "{} documents in index `ix`".format(ix.doc_count())
r = ix.reader()
IPython.embed(banner1="""
Got index reader in `r`.
Play around with the contents of the index!
See http://pythonhosted.org/Whoosh/api/reading.html for help.
""")
except OSError:
print >> sys.stderr, "Can't find index in 'ex_index'. Create it first!"
def main():
global args, best_loss1
args = parser.parse_args()
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size)
# create model
model = DenseNet121(N_CLASSES).cuda()
import IPython
IPython.embed()
model = torch.nn.DataParallel(model).cuda()
# define loss function (criterion) and optimizer
criterion = nn.BCELoss().cuda()
optimizer = torch.optim.Adam(model.parameters(),
lr=0.001,
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=0)
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
# Data loading code
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_idx = np.genfromtxt(TRAIN_INDEXES, delimiter=',').astype('int64')
train_dataset = ChestXrayDataSetHDF5(data_dir=HDF5_DATA_DIR,
set_idx=train_idx,
transform=transforms.Compose([
RandomHorizontalFlip(),
transforms.ToTensor(), normalize
]),
target_transform=None)
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
val_idx = np.genfromtxt(TRAIN_INDEXES, delimiter=',').astype('int64')
val_dataset = ChestXrayDataSetHDF5(data_dir=HDF5_DATA_DIR,
set_idx=val_idx,
transform=transforms.Compose([
RandomHorizontalFlip(),
transforms.ToTensor(), normalize
]),
target_transform=None)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion)
return
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
adjust_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
loss1 = validate(val_loader, model, criterion)
# remember best prec@1 and save checkpoint
is_best = loss1 < best_loss1
best_loss1 = max(loss1, best_loss1)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}, is_best)
def run_epoch_recurrence_one_batch(self,
batch,
is_training=False,
source='agent'):
if is_training:
self.acmodel.train()
else:
self.acmodel.eval()
batch_old = batch
batch = self.transform_demos(batch, source)
batch.sort(key=len, reverse=True)
# Constructing flat batch and indices pointing to start of each demonstration
obss = []
action_true = []
action_teacher = []
done = []
inds = [0]
for demo in batch:
obss.append(demo[0])
action_true.append(demo[1])
done.append(demo[2])
action_teacher.append(demo[3])
inds.append(inds[-1] + len(demo[0]))
# (batch size * avg demo length , 3), where 3 is for (state, action, done)
try:
obss = np.concatenate(obss)
except:
print("?")
import IPython
IPython.embed()
action_true = np.concatenate(action_true)
assert len(action_true.shape) == 1
done = np.concatenate(done)
action_teacher = np.concatenate(action_teacher)
inds = inds[:-1]
num_frames = len(obss)
mask = np.ones([len(obss)], dtype=np.float64)
try:
mask[inds] = 0
except:
print("???")
print("BATCH LENGTH", len(batch))
for demo in batch:
print("Obs", demo[0].shape)
print("LEN", len(demo))
print("INDS", inds, inds.shape)
print("MASK", mask.shape)
import IPython
IPython.embed()
mask = torch.tensor(mask, device=self.device,
dtype=torch.float).unsqueeze(1)
# Observations, true action, values and done for each of the stored demostration
action_true = torch.tensor([action for action in action_true],
device=self.device,
dtype=torch.long)
# Memory to be stored
memories = torch.zeros([len(obss), self.acmodel.memory_size],
device=self.device)
episode_ids = np.zeros(len(obss))
memory = torch.zeros([len(batch), self.acmodel.memory_size],
device=self.device)
preprocessed_first_obs = self.obss_preprocessor(
obss[inds], self.device)
instr = self.acmodel.get_instr(preprocessed_first_obs)
instr_embedding = self.acmodel._get_instr_embedding(instr)
# Loop terminates when every observation in the flat_batch has been handled
while True:
# taking observations and done located at inds
obs = obss[inds]
done_step = done[inds]
preprocessed_obs = self.obss_preprocessor(obs, self.device)
with torch.no_grad():
# taking the memory till len(inds), as demos beyond that have already finished
dist, info = self.acmodel(preprocessed_obs,
memory[:len(inds), :],
instr_embedding[:len(inds)],
self.distill_with_teacher)
new_memory = info['memory']
memories[inds, :] = memory[:len(inds), :]
memory[:len(inds), :] = new_memory
episode_ids[inds] = range(len(inds))
# Updating inds, by removing those indices corresponding to which the demonstrations have finished
inds = inds[:int(len(inds) - sum(done_step))]
if len(inds) == 0:
break
# Incrementing the remaining indices
inds = [index + 1 for index in inds]
# Here, actual backprop upto args.recurrence happens
final_loss = 0
per_token_correct = [0, 0, 0, 0, 0, 0, 0]
per_token_teacher_correct = [0, 0, 0, 0, 0, 0, 0]
per_token_count = [0, 0, 0, 0, 0, 0, 0]
per_token_teacher_count = [0, 0, 0, 0, 0, 0, 0]
per_token_agent_count = [0, 0, 0, 0, 0, 0, 0]
final_entropy, final_policy_loss, final_value_loss = 0, 0, 0
indexes = self.starting_indexes(num_frames)
memory = memories[indexes]
accuracy = 0
total_frames = len(indexes) * self.args.recurrence
accuracy_list = []
lengths_list = []
agent_running_count_long = 0
teacher_running_count_long = 0
for i in range(self.args.recurrence):
obs = obss[indexes]
preprocessed_obs = self.obss_preprocessor(obs, device=self.device)
action_step = action_true[indexes]
mask_step = mask[indexes]
dist, info = self.acmodel(preprocessed_obs, memory * mask_step,
instr_embedding[episode_ids[indexes]],
self.distill_with_teacher)
memory = info["memory"]
entropy = dist.entropy().mean()
policy_loss = -dist.log_prob(action_step).mean()
loss = policy_loss - self.args.entropy_coef * entropy
action_pred = dist.probs.max(1, keepdim=False)[1]
accuracy_list.append(float((action_pred == action_step).sum()))
lengths_list.append(
(action_pred.shape, action_step.shape, indexes.shape))
accuracy += float(
(action_pred == action_step).sum()) / total_frames
final_loss += loss
final_entropy += entropy
final_policy_loss += policy_loss
action_step = action_step.detach().cpu().numpy(
) # ground truth action
action_pred = action_pred.detach().cpu().numpy() # action we took
agent_running_count = 0
teacher_running_count = 0
for j in range(len(per_token_count)):
token_indices = np.where(action_step == j)[0]
count = len(token_indices)
correct = np.sum(
action_step[token_indices] == action_pred[token_indices])
per_token_correct[j] += correct
per_token_count[j] += count
action_teacher_index = action_teacher[indexes]
assert action_teacher_index.shape == action_pred.shape == action_step.shape, (
action_teacher_index.shape, action_pred.shape,
action_step.shape)
teacher_token_indices = np.where(action_teacher_index == j)[0]
teacher_count = len(teacher_token_indices)
teacher_correct = np.sum(
action_teacher_index[teacher_token_indices] ==
action_pred[teacher_token_indices])
per_token_teacher_correct[j] += teacher_correct
per_token_teacher_count[j] += teacher_count
teacher_running_count += teacher_count
agent_running_count += count
agent_running_count_long += teacher_count
teacher_running_count_long += teacher_count
assert np.min(action_step) < len(per_token_count), (
np.min(action_step), action_step)
assert np.max(action_step) >= 0, (np.max(action_step), action_step)
assert np.min(action_pred) < len(per_token_count), (
np.min(action_pred), action_pred)
assert np.max(action_pred) >= 0, (np.max(action_pred), action_pred)
assert np.min(action_teacher_index) < len(per_token_count), (
np.min(action_teacher_index), action_teacher_index)
assert np.max(action_teacher_index) >= 0, (
np.max(action_teacher_index), action_teacher_index)
indexes += 1
final_loss /= self.args.recurrence
if is_training:
self.optimizer.zero_grad()
final_loss.backward()
self.optimizer.step()
log = {}
log["Entropy"] = float(final_entropy / self.args.recurrence)
log["Loss"] = float(final_policy_loss / self.args.recurrence)
log["Accuracy"] = float(accuracy)
assert float(accuracy) <= 1, float(accuracy)
teacher_numerator = 0
teacher_denominator = 0
agent_numerator = 0
agent_denominator = 0
for i, (correct, count, teacher_correct, teacher_count) in enumerate(
zip(per_token_correct, per_token_count,
per_token_teacher_correct, per_token_teacher_count)):
assert correct <= count, (correct, count)
assert teacher_correct <= teacher_count, (teacher_correct,
teacher_count)
if count > 0:
log[f'Accuracy_{i}'] = correct / count
agent_numerator += correct
agent_denominator += count
if teacher_count > 0:
log[f'TeacherAccuracy_{i}'] = teacher_correct / teacher_count
teacher_numerator += teacher_correct
teacher_denominator += teacher_count
assert agent_denominator == teacher_denominator, (
agent_denominator, teacher_denominator, per_token_count,
per_token_teacher_count)
assert abs(float(accuracy) -
agent_numerator / agent_denominator) < .001, (
accuracy, agent_numerator / agent_denominator)
log["TeacherAccuracy"] = float(teacher_numerator / teacher_denominator)
return log
def optimize_policy(self, itr, all_samples_data, all_samples_data_latent):
assert len(
all_samples_data
) == self.num_grad_updates + 1 # we collected the rollouts to compute the grads and then the test!
sess = tf.get_default_session()
if not self.use_maml:
all_samples_data = [all_samples_data[0]]
input_list = []
for step in range(
len(all_samples_data)): # these are the gradient steps
obs_list, action_list, adv_list, noise_list, task_idx_list = [], [], [], [], []
for i in range(self.meta_batch_size):
inputs = ext.extract(all_samples_data[step][i], "observations",
"actions", "advantages", "noises",
"task_idxs")
print(inputs)
obs_list.append(inputs[0])
action_list.append(inputs[1])
adv_list.append(inputs[2])
noise_list.append(inputs[3])
task_idx_list.append(inputs[4])
input_list += obs_list + action_list + adv_list + noise_list + task_idx_list # [ [obs_0], [act_0], [adv_0], [obs_1], ... ]
if step == 0:
adv_list_latent, z_list_latent, task_idx_list_latent = [], [], []
for i in range(self.meta_batch_size):
inputs = ext.extract(all_samples_data_latent[step][i],
"advantages", "noises", "task_idxs")
#import ipdb
#ipdb.set_trace()
means = tf.gather(self.policy.all_params['latent_means'],
inputs[-1])
stds = tf.gather(self.policy.all_params['latent_stds'],
inputs[-1])
zs = sess.run(means + inputs[-2] * tf.exp(stds))
adv_list_latent.append(inputs[0])
z_list_latent.append(zs)
task_idx_list_latent.append(inputs[2])
input_list += adv_list_latent + z_list_latent + task_idx_list_latent
#import ipdb
#ipdb.set_trace()
if step == 0: ##CF not used?
init_inputs = input_list
if self.use_maml:
dist_info_list = []
for i in range(self.meta_batch_size):
agent_infos = all_samples_data[
self.kl_constrain_step][i]['agent_infos']
dist_info_list += [
agent_infos[k]
for k in self.policy.distribution.dist_info_keys
]
input_list += tuple(dist_info_list)
logger.log("Computing KL before")
mean_kl_before = self.optimizer.constraint_val(input_list)
if self.kl_scheme is None:
curr_kl_weighting = self.kl_weighting
elif self.kl_scheme == "0.01step4to0.05":
curr_kl_weighting = min(0.05, 0.01 + (itr // 10) * 0.01)
elif self.kl_scheme == "0.01step8to0.1":
curr_kl_weighting = min(0.1, 0.01 + (itr // 5) * 0.001)
elif self.kl_scheme == "0.01step8to0.05":
curr_kl_weighting = min(0.05, 0.01 + (itr // 5) * 0.0005)
elif self.kl_scheme == "0.01step8to0.2":
curr_kl_weighting = min(0.2, 0.01 + (itr // 5) * 0.002)
elif self.kl_scheme == "0.002step100to0.1":
curr_kl_weighting = min(0.1, 0.002 + (itr // 5) * 0.001)
elif self.kl_scheme == "0.002step100to0.02":
curr_kl_weighting = min(0.02, 0.002 + (itr // 5) * 0.0002)
elif self.kl_scheme == "0.002step100to0.05":
curr_kl_weighting = min(0.05, 0.002 + (itr // 5) * 0.0005)
elif self.kl_scheme == "0.01stepcontto0.05":
curr_kl_weighting = min(0.05, 0.01 + (itr) * 0.001)
elif self.kl_scheme == "0.01stepcontto0.1":
curr_kl_weighting = min(0.1, 0.01 + (itr) * 0.001)
elif self.kl_scheme == "0.01step8to0.3":
curr_kl_weighting = min(0.3, 0.01 + (itr) * 0.003)
else:
print("ERROR")
import IPython
IPython.embed()
input_list += ([curr_kl_weighting], )
logger.log("Computing loss before")
loss_before = self.optimizer.loss(input_list)
logger.log("Optimizing")
self.optimizer.optimize(input_list)
logger.log("Computing loss after")
loss_after = self.optimizer.loss(input_list)
if self.use_maml:
logger.log("Computing KL after")
mean_kl = self.optimizer.constraint_val(input_list)
logger.record_tabular('MeanKLBefore',
mean_kl_before) # this now won't be 0!
logger.record_tabular('MeanKL', mean_kl)
logger.record_tabular('LossBefore', loss_before)
logger.record_tabular('LossAfter', loss_after)
logger.record_tabular('dLoss', loss_before - loss_after)
return dict()
params.problem_file = '/tmp/lkh/m{0}n{1}task.m-pdtsp'.format(m, n)
params.max_trials = 1
params.runs = 3
params.special = True
lkh.parser.write_tsplib(params.problem_file,
graph,
params,
nodelist=nodelist,
demand=demand,
capacity=2,
depot=0)
tour, info = lkh.solver.lkh_solver(params)
import IPython
IPython.embed(banner1="")
"""
# Reduce the velocity limits
velocity_limits = robot.GetDOFVelocityLimits()
robot.SetDOFVelocityLimits(0.25*velocity_limits)
# Set the viewer camera
Tcam = br.euler.to_transform(*np.deg2rad([-120, 0, 90]))
Tcam[:3,3] = [2.5, 0.25, 2]
viewer.SetCamera(Tcam)
# Find the IK solutions for both arms
Tleft = env.GetKinBody('cube_05').GetTransform()
Tleft[:3,3] += [0, 0, 0.065]
Tright = env.GetKinBody('cube_01').GetTransform()
Tright[:3,3] += [0, 0, 0.065]
# Estimate torso angle
torso_angle = bimanual.estimate_torso_angle(Tleft[:3,3], Tright[:3,3])
def embed_ipython(
args, context
): # allow embedding IPython session in the middle of Scheme code
import IPython
IPython.embed()
def main(_):
if FLAGS.self_test:
print('Running self-test.')
train_data, train_labels = fake_data(256)
validation_data, validation_labels = fake_data(EVAL_BATCH_SIZE)
test_data, test_labels = fake_data(EVAL_BATCH_SIZE)
num_epochs = 1
else:
# Get the data.
train_data_filename = maybe_download('train-images-idx3-ubyte.gz')
train_labels_filename = maybe_download('train-labels-idx1-ubyte.gz')
test_data_filename = maybe_download('t10k-images-idx3-ubyte.gz')
test_labels_filename = maybe_download('t10k-labels-idx1-ubyte.gz')
# Extract it into numpy arrays.
train_data = extract_data(train_data_filename, 60000)
train_labels = extract_labels(train_labels_filename, 60000)
test_data = extract_data(test_data_filename, 10000)
test_labels = extract_labels(test_labels_filename, 10000)
# Generate a validation set.
validation_data = train_data[:VALIDATION_SIZE, ...]
validation_labels = train_labels[:VALIDATION_SIZE]
train_data = train_data[VALIDATION_SIZE:, ...]
train_labels = train_labels[VALIDATION_SIZE:]
num_epochs = NUM_EPOCHS
train_size = train_labels.shape[0]
# This is where training samples and labels are fed to the graph.
# These placeholder nodes will be fed a batch of training data at each
# training step using the {feed_dict} argument to the Run() call below.
train_data_node = tf.placeholder(data_type(),
shape=(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE,
NUM_CHANNELS))
train_labels_node = tf.placeholder(tf.int64, shape=(BATCH_SIZE, ))
eval_data = tf.placeholder(data_type(),
shape=(EVAL_BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE,
NUM_CHANNELS))
# The variables below hold all the trainable weights. They are passed an
# initial value which will be assigned when we call:
# {tf.global_variables_initializer().run()}
conv1_weights = tf.Variable(
tf.truncated_normal(
[5, 5, NUM_CHANNELS, 32], # 5x5 filter, depth 32.
stddev=0.1,
seed=SEED,
dtype=data_type()))
conv1_biases = tf.Variable(tf.zeros([32], dtype=data_type()))
conv2_weights = tf.Variable(
tf.truncated_normal([5, 5, 32, 64],
stddev=0.1,
seed=SEED,
dtype=data_type()))
conv2_biases = tf.Variable(tf.constant(0.1, shape=[64], dtype=data_type()))
fc1_weights = tf.Variable( # fully connected, depth 512.
tf.truncated_normal([IMAGE_SIZE // 4 * IMAGE_SIZE // 4 * 64, 512],
stddev=0.1,
seed=SEED,
dtype=data_type()))
fc1_biases = tf.Variable(tf.constant(0.1, shape=[512], dtype=data_type()))
fc2_weights = tf.Variable(
tf.truncated_normal([512, NUM_LABELS],
stddev=0.1,
seed=SEED,
dtype=data_type()))
fc2_biases = tf.Variable(
tf.constant(0.1, shape=[NUM_LABELS], dtype=data_type()))
# We will replicate the model structure for the training subgraph, as well
# as the evaluation subgraphs, while sharing the trainable parameters.
def model(data, train=False):
"""The Model definition."""
# 2D convolution, with 'SAME' padding (i.e. the output feature map has
# the same size as the input). Note that {strides} is a 4D array whose
# shape matches the data layout: [image index, y, x, depth].
conv = tf.nn.conv2d(data,
conv1_weights,
strides=[1, 1, 1, 1],
padding='SAME')
# Bias and rectified linear non-linearity.
relu = tf.nn.relu(tf.nn.bias_add(conv, conv1_biases))
# Max pooling. The kernel size spec {ksize} also follows the layout of
# the data. Here we have a pooling window of 2, and a stride of 2.
pool = tf.nn.max_pool(relu,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
conv = tf.nn.conv2d(pool,
conv2_weights,
strides=[1, 1, 1, 1],
padding='SAME')
relu = tf.nn.relu(tf.nn.bias_add(conv, conv2_biases))
pool = tf.nn.max_pool(relu,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# Reshape the feature map cuboid into a 2D matrix to feed it to the
# fully connected layers.
pool_shape = pool.get_shape().as_list()
reshape = tf.reshape(
pool,
[pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
# Fully connected layer. Note that the '+' operation automatically
# broadcasts the biases.
hidden = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)
# Add a 50% dropout during training only. Dropout also scales
# activations such that no rescaling is needed at evaluation time.
if train:
hidden = tf.nn.dropout(hidden, 0.5, seed=SEED)
return tf.matmul(hidden, fc2_weights) + fc2_biases
# Training computation: logits + cross-entropy loss.
logits = model(train_data_node, True)
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=train_labels_node, logits=logits))
# L2 regularization for the fully connected parameters.
regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) +
tf.nn.l2_loss(fc2_weights) + tf.nn.l2_loss(fc2_biases))
# Add the regularization term to the loss.
loss += 5e-4 * regularizers
# Optimizer: set up a variable that's incremented once per batch and
# controls the learning rate decay.
batch = tf.Variable(0, dtype=data_type())
# Decay once per epoch, using an exponential schedule starting at 0.01.
learning_rate = tf.train.exponential_decay(
0.01, # Base learning rate.
batch * BATCH_SIZE, # Current index into the dataset.
train_size, # Decay step.
0.95, # Decay rate.
staircase=True)
# Use simple momentum for the optimization.
optimizer = tf.train.MomentumOptimizer(learning_rate,
0.9).minimize(loss,
global_step=batch)
# Predictions for the current training minibatch.
train_prediction = tf.nn.softmax(logits)
# Predictions for the test and validation, which we'll compute less often.
eval_logits = model(eval_data)
eval_prediction = tf.nn.softmax(eval_logits)
# Small utility function to evaluate a dataset by feeding batches of data to
# {eval_data} and pulling the results from {eval_predictions}.
# Saves memory and enables this to run on smaller GPUs.
def eval_in_batches(data, sess):
"""Get all predictions for a dataset by running it in small batches."""
size = data.shape[0]
if size < EVAL_BATCH_SIZE:
raise ValueError("batch size for evals larger than dataset: %d" %
size)
predictions = numpy.ndarray(shape=(size, NUM_LABELS),
dtype=numpy.float32)
for begin in xrange(0, size, EVAL_BATCH_SIZE):
end = begin + EVAL_BATCH_SIZE
if end <= size:
predictions[begin:end, :] = sess.run(
eval_prediction,
feed_dict={eval_data: data[begin:end, ...]})
else:
batch_predictions = sess.run(
eval_prediction,
feed_dict={eval_data: data[-EVAL_BATCH_SIZE:, ...]})
predictions[begin:, :] = batch_predictions[begin - size:, :]
return predictions
# Create a local session to run the training.
start_time = time.time()
saver = tf.compat.v1.train.Saver(keep_checkpoint_every_n_hours=2,
max_to_keep=2)
import IPython
IPython.embed()
raise 1
with tf.Session() as sess:
# Run all the initializers to prepare the trainable parameters.
tf.global_variables_initializer().run()
print('Initialized!')
# Loop through training steps.
for step in xrange(int(num_epochs * train_size) // BATCH_SIZE):
# Compute the offset of the current minibatch in the data.
# Note that we could use better randomization across epochs.
offset = (step * BATCH_SIZE) % (train_size - BATCH_SIZE)
batch_data = train_data[offset:(offset + BATCH_SIZE), ...]
batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
# This dictionary maps the batch data (as a numpy array) to the
# node in the graph it should be fed to.
feed_dict = {
train_data_node: batch_data,
train_labels_node: batch_labels
}
# Run the optimizer to update weights.
sess.run(optimizer, feed_dict=feed_dict)
# print some extra information once reach the evaluation frequency
if step % EVAL_FREQUENCY == 0:
# fetch some extra nodes' data
l, lr, predictions = sess.run(
[loss, learning_rate, train_prediction],
feed_dict=feed_dict)
elapsed_time = time.time() - start_time
start_time = time.time()
print('Step %d (epoch %.2f), %.1f ms' %
(step, float(step) * BATCH_SIZE / train_size,
1000 * elapsed_time / EVAL_FREQUENCY))
print('Minibatch loss: %.3f, learning rate: %.6f' % (l, lr))
print('Minibatch error: %.1f%%' %
error_rate(predictions, batch_labels))
print('Validation error: %.1f%%' % error_rate(
eval_in_batches(validation_data, sess), validation_labels))
sys.stdout.flush()
saver.save(sess, os.path.join(MODEL_DIR, 'model'))
# Finally print the result!
test_error = error_rate(eval_in_batches(test_data, sess), test_labels)
print('Test error: %.1f%%' % test_error)
if FLAGS.self_test:
print('test_error', test_error)
assert test_error == 0.0, 'expected 0.0 test_error, got %.2f' % (
test_error, )
def warp_slice(inp_src: DataSource,
patch_shape: Union[Tuple[int, ...], np.ndarray],
M: np.ndarray,
target_src: Optional[DataSource] = None,
target_patch_shape: Optional[Union[Tuple[int],
np.ndarray]] = None,
target_discrete_ix: Optional[Sequence[int]] = None,
input_discrete_ix: Optional[Sequence[int]] = None,
debug: bool = False) -> Tuple[np.ndarray, Optional[np.ndarray]]:
"""
Cuts a warped slice out of the input image and out of the target_src image.
Warping is applied by multiplying the original source coordinates with
the inverse of the homogeneous (forward) transformation matrix ``M``.
"Source coordinates" (``src_coords``) signify the coordinates of voxels in
``inp_src`` and ``target_src`` that are used to compose their respective warped
versions. The idea here is that not the images themselves, but the
coordinates from where they are read are warped. This allows for much higher
efficiency for large image volumes because we don't have to calculate the
expensive warping transform for the whole image, but only for the voxels
that we eventually want to use for the new warped image.
The transformed coordinates usually don't align to the discrete
voxel grids of the original images (meaning they are not integers), so the
new voxel values are obtained by linear interpolation.
Parameters
----------
inp_src
Input image source (in HDF5)
patch_shape
(spatial only) Patch shape ``(D, H, W)``
(spatial shape of the neural network's input node)
M
Forward warping tansformation matrix (4x4).
Must contain translations in source and target_src array.
target_src
Optional target source array to be extracted from in the same way.
target_patch_shape
Patch size for the ``target_src`` array.
target_discrete_ix
List of target channels that contain discrete values.
By default (``None``), every channel is is seen as discrete (this is
generally the case for classification tasks).
This information is used to decide what kind of interpolation should
be used for reading target data:
- discrete targets are obtained by nearest-neighbor interpolation
- non-discrete (continuous) targets are linearly interpolated.
input_discrete_ix
List of input channels that contain discrete values.
By default (``None``), no channel is seen as discrete (generally
inputs are real world images).
See target_discrete_ix for the effect on input interpolation.
debug: If ``True`` (default), enable additional sanity checks to catch
warping issues early.
Returns
-------
inp
Warped input image slice
target
Warped target_src image slice
or ``None``, if ``target_src is None``.
"""
patch_shape = tuple(patch_shape)
if len(inp_src.shape) == 3:
n_f = 1
elif len(inp_src.shape) == 4:
n_f = inp_src.shape[0]
else:
raise ValueError(f'Can\'t handle inp_src shape {inp_src.shape}')
# Spatial shapes of input and target data sources
inp_src_shape = np.array(inp_src.shape[-3:])
M_inv = np.linalg.inv(M.astype(np.float64)).astype(floatX) # stability...
dest_corners = make_dest_corners(patch_shape)
src_corners = np.dot(M_inv, dest_corners.T).T
if np.any(M[3, :3] != 0): # homogeneous divide
src_corners /= src_corners[:, 3][:, None]
# check corners
src_corners = src_corners[:, :3]
lo = np.min(np.floor(src_corners), 0).astype(np.int)
hi = np.max(np.ceil(src_corners + 1), 0).astype(np.int)
# compute/transform dense coords
dest_coords = make_dest_coords(patch_shape)
src_coords = np.tensordot(dest_coords, M_inv, axes=[[-1], [1]])
if np.any(M[3, :3] != 0): # homogeneous divide
src_coords /= src_coords[..., 3][..., None]
# cut patch
src_coords = src_coords[..., :3]
# TODO: WIP code, integrate this into the warping pipeline with config options
# Perform elastic deformation on warped coordinates so we don't have
# to interpolate twice.
# For more details, see elektronn3.data.transforms.ElasticTransform
elastic = False
if elastic:
sigma = 4
alpha = 40
aniso_factor = 2
for i in range(3):
# For each coordinate of dimension i, build a random displacement,
# smooth it with sigma and multiply it by alpha
elastic_displacement = gaussian_filter(
np.random.rand(*patch_shape) * 2 - 1,
sigma,
mode='constant',
cval=0) * alpha
# Apply anisotropy correction
if i == 0 and aniso_factor != 1:
elastic_displacement /= aniso_factor
# Apply deformation
src_coords[..., i] += elastic_displacement
# Clip out-of-bounds coordinates back to original cube edges to
# prevent out-of-bounds reading
np.clip(src_coords[..., i],
lo[i],
hi[i] - 1,
out=src_coords[..., i])
if target_src is not None:
target_src_shape = np.array(target_src.shape[-3:])
target_patch_shape = tuple(target_patch_shape)
n_f_t = target_src.shape[0] if target_src.ndim == 4 else 1
target_src_offset = np.subtract(inp_src_shape, target_src.shape[-3:])
if np.any(np.mod(target_src_offset, 2)):
raise ValueError("targets must be centered w.r.t. images")
target_src_offset //= 2
target_offset = np.subtract(patch_shape, target_patch_shape)
if np.any(np.mod(target_offset, 2)):
raise ValueError("targets must be centered w.r.t. images")
target_offset //= 2
src_coords_target = src_coords[
target_offset[0]:(target_offset[0] + target_patch_shape[0]),
target_offset[1]:(target_offset[1] + target_patch_shape[1]),
target_offset[2]:(target_offset[2] + target_patch_shape[2])]
# shift coords to be w.r.t. to origin of target_src array
lo_targ = np.floor(
src_coords_target.min(2).min(1).min(0) - target_src_offset).astype(
np.int)
hi_targ = np.ceil(
src_coords_target.max(2).max(1).max(0) + 1 -
target_src_offset).astype(np.int)
if np.any(lo_targ < 0) or np.any(hi_targ >= target_src_shape - 1):
raise WarpingOOBError("Out of bounds for target_src")
if np.any(lo < 0) or np.any(hi >= inp_src_shape - 1):
raise WarpingOOBError("Out of bounds for inp_src")
# Slice and interpolate input
# Slice to hi + 1 because interpolation potentially needs this value.
img_cut = slice_3d(inp_src, lo, hi + 1, dtype=floatX)
if img_cut.ndim == 3:
img_cut = img_cut[None]
inp = np.zeros((n_f, ) + patch_shape, dtype=floatX)
lo = lo.astype(floatX)
if debug and np.any(
(src_coords - lo).max(2).max(1).max(0) >= img_cut.shape[-3:]):
raise WarpingSanityError(
f'src_coords check failed (too high).\n{(src_coords - lo).max(2).max(1).max(0), img_cut.shape[-3:]}'
)
if debug and np.any((src_coords - lo).min(2).min(1).min(0) < 0):
raise WarpingSanityError(
f'src_coords check failed (negative indices).\n{(src_coords - lo).min(2).min(1).min(0)}'
)
if input_discrete_ix is None:
input_discrete_ix = [False for i in range(img_cut.shape[0])]
else:
input_discrete_ix = [
i in input_discrete_ix for i in range(img_cut.shape[0])
]
for k, discr in enumerate(input_discrete_ix):
(map_coordinates_nearest if discr else map_coordinates_linear)(
img_cut[k], src_coords, lo, inp[k])
# Slice and interpolate target
if target_src is not None:
# dtype is float as well here because of the static typing of the
# numba-compiled map_coordinates functions
# Slice to hi + 1 because interpolation potentially needs this value.
target_cut = slice_3d(target_src, lo_targ, hi_targ + 1, dtype=floatX)
if target_cut.ndim == 3:
target_cut = target_cut[None]
src_coords_target = np.ascontiguousarray(src_coords_target,
dtype=floatX)
target = np.zeros((n_f_t, ) + target_patch_shape, dtype=floatX)
lo_targ = (lo_targ + target_src_offset).astype(floatX)
if target_discrete_ix is None:
target_discrete_ix = [True for i in range(n_f_t)]
else:
target_discrete_ix = [
i in target_discrete_ix for i in range(n_f_t)
]
if debug and np.any(
(src_coords_target -
lo_targ).max(2).max(1).max(0) >= target_cut.shape[-3:]):
raise WarpingSanityError(
f'src_coords_target check failed (too high).\n{(src_coords_target - lo_targ).max(2).max(1).max(0)}\n{target_cut.shape[-3:]}'
)
if debug and np.any(
(src_coords_target - lo_targ).min(2).min(1).min(0) < 0):
raise WarpingSanityError(
f'src_coords_target check failed (negative indices).\n{(src_coords_target - lo_targ).min(2).min(1).min(0)}'
)
for k, discr in enumerate(target_discrete_ix):
if discr:
map_coordinates_nearest(target_cut[k], src_coords_target,
lo_targ, target[k])
if debug:
unique_cut = set(list(np.unique(target_cut[k])))
unique_warp = set(list(np.unique(target[k])))
# If new values appear in discrete targets, there is something wrong.
# unique_warp can have less values than unique_cut though, for example
# if the warping transform coincidentally slices away all values of a class.
if not unique_warp.issubset(unique_cut):
print(
f'Invalid target encountered:\n\nunique_cut=\n{unique_cut}\n'
f'unique_warp=\n{unique_warp}\nM_inv=\n{M_inv}\n'
f'src_coords_target - lo_targ=\n{src_coords_target - lo_targ}\n'
)
# Try dropping to an IPython shell (Won't work with num_workers > 0).
import IPython
IPython.embed()
raise SystemExit
else:
map_coordinates_linear(target_cut[k], src_coords_target,
lo_targ, target[k])
else:
target = None
if debug and np.any(np.isnan(inp)):
raise RuntimeError('Warping is broken: inp contains NaN.')
if debug and np.any(np.isnan(target)):
raise RuntimeError('Warping is broken: target contains NaN.')
return inp, target
def run_trial(opt):
oc = svm.OneClassSVM(kernel='rbf', nu=.01, gamma=.01)
est = knet.Network(opt.arch, learning_rate=opt.lr, epochs=opt.epochs)
lnr = learner.Learner(est)
plot_dir = utils.generate_plot_dir('initial', 'experts', vars(opt))
data_dir = utils.generate_data_dir('initial', 'experts', vars(opt))
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
if not os.path.exists(data_dir):
os.makedirs(data_dir)
opt.plot_dir = plot_dir
opt.data_dir = data_dir
opt.num_valid_trajs = max(1, int(.25 * opt.iters))
opt.samples = 10
train_trajs = []
valid_trajs = []
for i in range(opt.iters):
print "Iteration: " + str(i)
states, int_actions, taken_actions, r = statistics.collect_traj(
opt.env, opt.sup, opt.t)
lnr.add_data(states, int_actions)
oc.fit(lnr.X)
preds = oc.predict(lnr.X)
train_err = len(preds[preds == -1]) / float(len(preds))
print "\nTraining error: " + str(train_err)
lnr.train()
sup_rewards = np.zeros((20))
lnr_rewards = np.zeros((20))
X_valid = []
X_test = []
for i in range(20):
states_valid, int_actions_valid, _, r_valid = statistics.collect_traj(
opt.env, opt.sup, opt.t, False)
states_test, int_actions_test, _, r_test = statistics.collect_traj(
opt.env, lnr, opt.t, False)
sup_rewards[i] = r_valid
lnr_rewards[i] = r_test
X_valid += states_valid
X_test += states_test
valid_preds = oc.predict(X_valid)
valid_err = len(valid_preds[valid_preds == -1]) / float(len(valid_preds))
print "Validation erorr: " + str(valid_err)
test_preds = oc.predict(X_test)
test_err = len(test_preds[test_preds == -1]) / float(len(test_preds))
print "Test erorr: " + str(test_err)
print "\n\n"
print "Average sup reward: " + str(np.mean(sup_rewards)) + " +/- " + str(
scipy.stats.sem(sup_rewards))
print "Average lnr reward: " + str(np.mean(lnr_rewards)) + " +/- " + str(
scipy.stats.sem(lnr_rewards))
print "\n\n"
def dec(u):
x = opt.env.get_x()
s, _, _, _ = opt.env.step(u)
opt.env.set_x(x)
return oc.decision_function([s])[0, 0]
rewards = np.zeros((20))
rec_counts = np.zeros((20))
X_robust = []
for i in range(20):
s = opt.env.reset()
states = [s]
for t in range(opt.t):
score = oc.decision_function([s])[0, 0]
print "Decision score: " + str(score)
if score < .1:
alpha = .1
a = alpha * utils.finite_diff1(
np.zeros(opt.env.action_space.shape), dec)
IPython.embed()
rec_counts[i] += 1.0
s, r, done, _ = opt.env.step(a)
a = lnr.intended_action(s)
s, r, done, _ = opt.env.step(a)
rewards[i] += r
states.append(s)
if done == True:
break
X_robust += states
robust_preds = oc.predict(X_robust)
robust_err = len(robust_preds[robust_preds == -1]) / float(
len(robust_preds))
print "Robust erorr: " + str(robust_err)
rec_freq = np.mean(rec_counts / float(opt.t))
print "Recovery frequency: " + str(rec_freq)
print "Robust rewards: " + str(np.mean(rewards)) + " +/- " + str(
scipy.stats.sem(rewards))
def run(self):
IPython.embed(header=__header__, banner1=__content__)
print "Detaching from the target process"
self.session.detach()
return
def scrape_single_game(sport, browser, debug=False):
# Since we are separating contests by their start time now, we don't currently try to choose a
# best contest to scrape. We should certainly upgrade this in the future!
contest_element_id_regex = re.compile(r"contest_(?P<gameid>\d+)-(?P<tableid>\d+)")
browser_elem = browser.find_element_by_class_name('contest-list-item')
match_groups = contest_element_id_regex.match(browser_elem.get_attribute("id"))
fd_game_id = int(match_groups.group("gameid"))
fd_table_id = int(match_groups.group("tableid"))
fd_game_title = browser_elem.find_element_by_class_name("contest-name-text").text
entry_fee = browser_elem.find_element_by_class_name("entry-fee-cell").text.replace('$','').replace(',','')
game_time = browser_elem.find_element_by_class_name("startdate-cell").text
cal = Calendar()
new_parsed_dt, ret_code = cal.parseDT(game_time)
# If it is between 9 PM and midnight locally (Pacific), we'll have an issue:
# FD shows us the game time as e.g. "7 PM" but we'll interpret that as today.
# Workaround: if the game time is in the past, its probably tomorrow. Add 1 day.
# Break if it's not between 9PM and midnight so we know this happened
if new_parsed_dt < datetime.datetime.now():
if datetime.datetime.now().hour < 21:
print 'Parsed time of FanDuel game start is in the past, but this isnt a TZ issue!'
IPython.embed()
else:
new_parsed_dt += datetime.timedelta(days=1)
assert new_parsed_dt > datetime.datetime.now()
browser_elem.click()
print 'Scraping this game:'
print ' ', fd_game_title
print ' on', new_parsed_dt.isoformat()
game_entry_url = 'https://www.fanduel.com/games/{game_id}/contests/{game_id}-{table_id}/enter'.format(game_id=fd_game_id, table_id=fd_table_id)
# Go to the details for the game, to find list of eligible players for this game.
browser.get(game_entry_url)
time.sleep(1)
# Get the salary cap for the game directly
salary_text = browser.find_element_by_xpath('//*[@id="ui-skeleton"]/div/section/div[2]/div[4]/div[2]/section/header/'
'remaining-salary/div/div[1]/figure').text
cap = int(salary_text.replace('$','').replace(',',''))
# Download the master player / salary list
player_list_link = browser.find_element_by_link_text("Download players list")
# Sometimes this doesn't work the first time...
download_filename = get_csv_file(sport, new_parsed_dt, fd_game_id)
attempts = 1
while not os.path.exists(download_filename) and attempts <= 10:
print '...trying to download file (attempt %d)' % attempts
time.sleep(3)
player_list_link.click()
time.sleep(3)
attempts += 1
if attempts > 5:
print "Problem downloading player list -- not saving anything"
return False
else:
print '...success!'
# Get player lineups
if sport == 'mlb':
print "Accessing lineups..."
lineups_link = browser.find_element_by_link_text("Lineup Info")
# You can't just click it, it opens in a new tab :( So we visit it with get() and then go back()
lineup_url = lineups_link.get_attribute('href')
browser.get(lineup_url)
lineup_info = parse_lineup_page(browser.page_source)
browser.back()
print " %d / %d lineups submitted (%d players)" % (len(lineup_info.loaded_teams),
len(lineup_info.loaded_teams) + len(lineup_info.unloaded_teams),
len(lineup_info.parsed_players))
else:
# Should consider grabbing injury / GTD status here
lineup_info = None
# Except that doesn't work too well
# parse_nba_player_list(browser.page_source)
add_game_info_to_db(sport,
fd_game_id,
cap,
entry_fee,
new_parsed_dt,
fd_table_id,
game_title=fd_game_title,
lineup=lineup_info)
# Return to the main lobby page with back()
browser.back()
return True
print 'Index error on antenna %i'%ant
continue
avg_p2 = np.nanmean(np.absolute(dp2[tup][pol_strs[1]]))
ratio = avg_p2/avg_p1
std_p1 = np.nanstd(np.absolute(dp1[tup][pol_strs[0]]))
std_p2 = np.nanstd(np.absolute(dp2[tup][pol_strs[1]]))
std = ratio*np.sqrt((std_p1/avg_p1)**2. + (std_p2/avg_p2)**2.)
x1,y1 = antpos[ant]['top_x'],antpos[ant]['top_y']
dx,dy = np.abs(x0-x1),np.abs(y0-y1)
L = np.sqrt(dx**2. + dy**2.)
avg_ratios.append(ratio)
stdevs.append(std)
bl_length.append(L)
#indiv plot
if opts.mb and ant in badants: continue
plt.errorbar(L,ratio,yerr=std,fmt='o',ecolor='b',color='b')
plt.text(L,ratio,str(ant))
if ant in badants: plt.plot(L,ratio,'kx',ms=10)
if opts.verb: print np.where(avg_ratios > np.nanmean(avg_ratios)+2*np.nanstd(avg_ratios))
#format
plt.xlabel('Basline length [m]')
plt.xlim(0,300)
plt.ylabel(r'$\langle | V_{a,j} | \rangle_{t,\nu}$',size=20)
plt.suptitle('Relative to antenna %i'%anchor_ant,size=15)
if not opts.nobl: plt.show()
plt.close()
import IPython;IPython.embed()
from exampleapp import app import IPython app.testing = True test_client = app.test_client() welcome_message = """Welcome to your Flask CLI environment. The following variables are available to use: app -> Your Flask app instance. test_client -> Your Flask app.test_client(). """ IPython.embed(header=welcome_message)
def main(argv):
parser = ArgumentParser(usage=__doc__.lstrip())
parser.add_argument("--verbose",
"-v",
action="count",
default=1,
help="more verbosity")
parser.add_argument(
"--no-build",
"-n",
action="store_true",
default=False,
help="do not build the project (use system installed version)")
parser.add_argument("--build-only",
"-b",
action="store_true",
default=False,
help="just build, do not run any tests")
parser.add_argument("--doctests",
action="store_true",
default=False,
help="Run doctests in module")
parser.add_argument(
"--coverage",
action="store_true",
default=False,
help=("report coverage of project code. HTML output goes "
"under build/coverage"))
parser.add_argument(
"--gcov",
action="store_true",
default=False,
help=("enable C code coverage via gcov (requires GCC). "
"gcov output goes to build/**/*.gc*"))
parser.add_argument("--lcov-html",
action="store_true",
default=False,
help=("produce HTML for C code coverage information "
"from a previous run with --gcov. "
"HTML output goes to build/lcov/"))
parser.add_argument("--mode",
"-m",
default="fast",
help="'fast', 'full', or something that could be "
"passed to nosetests -A [default: fast]")
parser.add_argument(
"--submodule",
"-s",
default=None,
help="Submodule whose tests to run (cluster, constants, ...)")
parser.add_argument("--pythonpath",
"-p",
default=None,
help="Paths to prepend to PYTHONPATH")
parser.add_argument("--tests",
"-t",
action='append',
help="Specify tests to run")
parser.add_argument("--python",
action="store_true",
help="Start a Python shell with PYTHONPATH set")
parser.add_argument("--ipython",
"-i",
action="store_true",
help="Start IPython shell with PYTHONPATH set")
parser.add_argument("--shell",
action="store_true",
help="Start Unix shell with PYTHONPATH set")
parser.add_argument("--debug",
"-g",
action="store_true",
help="Debug build")
parser.add_argument("--parallel",
"-j",
type=int,
default=0,
help="Number of parallel jobs during build")
parser.add_argument("--show-build-log",
action="store_true",
help="Show build output rather than using a log file")
parser.add_argument("--bench",
action="store_true",
help="Run benchmark suite instead of test suite")
parser.add_argument(
"--bench-compare",
action="store",
metavar="COMMIT",
help=("Compare benchmark results to COMMIT. "
"Note that you need to commit your changes first!"))
parser.add_argument("--raise-warnings",
default=None,
type=str,
choices=('develop', 'release'),
help="if 'develop', warnings are treated as errors")
parser.add_argument("args",
metavar="ARGS",
default=[],
nargs=REMAINDER,
help="Arguments to pass to Nose, Python or shell")
args = parser.parse_args(argv)
if args.bench_compare:
args.bench = True
args.no_build = True # ASV does the building
if args.lcov_html:
# generate C code coverage output
lcov_generate()
sys.exit(0)
if args.pythonpath:
for p in reversed(args.pythonpath.split(os.pathsep)):
sys.path.insert(0, p)
if args.gcov:
gcov_reset_counters()
if args.debug and args.bench:
print("*** Benchmarks should not be run against debug "
"version; remove -g flag ***")
if not args.no_build:
site_dir = build_project(args)
sys.path.insert(0, site_dir)
os.environ['PYTHONPATH'] = site_dir
extra_argv = args.args[:]
if extra_argv and extra_argv[0] == '--':
extra_argv = extra_argv[1:]
if args.python:
# Debugging issues with warnings is much easier if you can see them
print("Enabling display of all warnings")
import warnings
import types
warnings.filterwarnings("always")
if extra_argv:
# Don't use subprocess, since we don't want to include the
# current path in PYTHONPATH.
sys.argv = extra_argv
with open(extra_argv[0], 'r') as f:
script = f.read()
sys.modules['__main__'] = types.ModuleType('__main__')
ns = dict(__name__='__main__', __file__=extra_argv[0])
exec_(script, ns)
sys.exit(0)
else:
import code
code.interact()
sys.exit(0)
if args.ipython:
# Debugging issues with warnings is much easier if you can see them
print("Enabling display of all warnings and pre-importing numpy as np")
import warnings
warnings.filterwarnings("always")
import IPython
import numpy as np
IPython.embed(user_ns={"np": np})
sys.exit(0)
if args.shell:
shell = os.environ.get('SHELL', 'sh')
print("Spawning a Unix shell...")
os.execv(shell, [shell] + extra_argv)
sys.exit(1)
if args.coverage:
dst_dir = os.path.join(ROOT_DIR, 'build', 'coverage')
fn = os.path.join(dst_dir, 'coverage_html.js')
if os.path.isdir(dst_dir) and os.path.isfile(fn):
shutil.rmtree(dst_dir)
extra_argv += ['--cover-html', '--cover-html-dir=' + dst_dir]
if args.bench:
# Run ASV
items = extra_argv
if args.tests:
items += args.tests
if args.submodule:
items += [args.submodule]
bench_args = []
for a in items:
bench_args.extend(['--bench', a])
if not args.bench_compare:
cmd = ['asv', 'run', '-n', '-e', '--python=same'] + bench_args
os.chdir(os.path.join(ROOT_DIR, 'benchmarks'))
os.execvp(cmd[0], cmd)
sys.exit(1)
else:
commits = [x.strip() for x in args.bench_compare.split(',')]
if len(commits) == 1:
commit_a = commits[0]
commit_b = 'HEAD'
elif len(commits) == 2:
commit_a, commit_b = commits
else:
p.error("Too many commits to compare benchmarks for")
# Check for uncommitted files
if commit_b == 'HEAD':
r1 = subprocess.call(
['git', 'diff-index', '--quiet', '--cached', 'HEAD'])
r2 = subprocess.call(['git', 'diff-files', '--quiet'])
if r1 != 0 or r2 != 0:
print("*" * 80)
print("WARNING: you have uncommitted changes --- "
"these will NOT be benchmarked!")
print("*" * 80)
# Fix commit ids (HEAD is local to current repo)
p = subprocess.Popen(['git', 'rev-parse', commit_b],
stdout=subprocess.PIPE)
out, err = p.communicate()
commit_b = out.strip()
p = subprocess.Popen(['git', 'rev-parse', commit_a],
stdout=subprocess.PIPE)
out, err = p.communicate()
commit_a = out.strip()
cmd = [
'asv', 'continuous', '-e', '-f', '1.05', commit_a, commit_b
] + bench_args
os.chdir(os.path.join(ROOT_DIR, 'benchmarks'))
os.execvp(cmd[0], cmd)
sys.exit(1)
test_dir = os.path.join(ROOT_DIR, 'build', 'test')
if args.build_only:
sys.exit(0)
elif args.submodule:
modname = PROJECT_MODULE + '.' + args.submodule
try:
__import__(modname)
test = sys.modules[modname].test
except (ImportError, KeyError, AttributeError):
print("Cannot run tests for %s" % modname)
sys.exit(2)
elif args.tests:
def fix_test_path(x):
# fix up test path
p = x.split(':')
p[0] = os.path.relpath(os.path.abspath(p[0]), test_dir)
return ':'.join(p)
tests = [fix_test_path(x) for x in args.tests]
def test(*a, **kw):
extra_argv = kw.pop('extra_argv', ())
extra_argv = extra_argv + tests[1:]
kw['extra_argv'] = extra_argv
from numpy.testing import Tester
return Tester(tests[0]).test(*a, **kw)
else:
__import__(PROJECT_MODULE)
test = sys.modules[PROJECT_MODULE].test
# Run the tests under build/test
try:
shutil.rmtree(test_dir)
except OSError:
pass
try:
os.makedirs(test_dir)
except OSError:
pass
cwd = os.getcwd()
try:
os.chdir(test_dir)
result = test(args.mode,
verbose=args.verbose,
extra_argv=extra_argv,
doctests=args.doctests,
raise_warnings=args.raise_warnings,
coverage=args.coverage)
finally:
os.chdir(cwd)
if result.wasSuccessful():
sys.exit(0)
else:
sys.exit(1)
def forward(self, x):
sources = list() # locとconfへの入力source1～6を格納
loc = list() # locの出力を格納
conf = list() # confの出力を格納
# vggのconv4_3まで計算する
for k in range(23):
x = self.vgg[k](x)
# conv4_3の出力をL2Normに入力し、source1を作成、sourcesに追加
source1 = self.L2Norm(x)
sources.append(source1)
# vggを最後まで計算し、source2を作成、sourcesに追加
for k in range(23, len(self.vgg)):
x = self.vgg[k](x)
sources.append(x)
# extrasのconvとReLUを計算
# source3～6を、sourcesに追加
for k, v in enumerate(self.extras):
x = F.relu(v(x), inplace=True)
if k % 2 == 1: # conv→ReLU→cov→ReLUをしたらsourceに入れる
sources.append(x)
# source1～6に、それぞれ対応する畳み込みを1回ずつ適用する
# zipでforループの複数のリストの要素を取得
# source1～6まであるので、6回ループが回る
for (x, l, c) in zip(sources, self.loc, self.conf):
# Permuteは要素の順番を入れ替え
loc.append(l(x).permute(0, 2, 3, 1).contiguous())
conf.append(c(x).permute(0, 2, 3, 1).contiguous())
# l(x)とc(x)で畳み込みを実行
# l(x)とc(x)の出力サイズは[batch_num, 4*アスペクト比の種類数, featuremapの高さ, featuremap幅]
# sourceによって、アスペクト比の種類数が異なり、面倒なので順番入れ替えて整える
# permuteで要素の順番を入れ替え、
# [minibatch数, featuremap数, featuremap数,4*アスペクト比の種類数]へ
# （注釈）
# torch.contiguous()はメモリ上で要素を連続的に配置し直す命令です。
# あとでview関数を使用します。
# このviewを行うためには、対象の変数がメモリ上で連続配置されている必要があります。
# さらにlocとconfの形を変形
# locのサイズは、torch.Size([batch_num, 34928])
# confのサイズはtorch.Size([batch_num, 183372])になる
loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1)
conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1)
# さらにlocとconfの形を整える
# locのサイズは、torch.Size([batch_num, 8732, 4])
# confのサイズは、torch.Size([batch_num, 8732, 21])
loc = loc.view(loc.size(0), -1, 4)
try:
conf = conf.view(conf.size(0), -1, self.num_classes)
except:
import traceback
traceback.print_exc()
import IPython
IPython.embed()
# 最後に出力する
output = (loc, conf, self.dbox_list)
if self.phase == "inference": # 推論時
# クラス「Detect」のforwardを実行
# 返り値のサイズは torch.Size([batch_num, 21, 200, 5])
return self.detect(output[0], output[1], output[2])
else: # 学習時
return output
def generate_compose_tree(self, treex, latent_canvas_size):
for i in range(0, treex.num_children):
treex.children[i] = self.generate_compose_tree(treex.children[i], latent_canvas_size)
# one hot embedding of a word
ohe = self.get_code(self.dictionary, treex.word)
if treex.function == 'combine':
vis_dist = self.vis_dist(ohe)
pos_dist = self.pos_dist(ohe)
if treex.num_children > 0:
# visual content
vis_dist_child = treex.children[0].vis_dist
vis_dist = self.combine(vis_dist, vis_dist_child, 'vis')
# visual position
pos_dist_child = treex.children[0].pos_dist
pos_dist = self.combine(pos_dist, pos_dist_child, 'pos')
treex.vis_dist = vis_dist
treex.pos_dist = pos_dist
elif treex.function == 'describe':
# blend visual words
vis_dist = self.vis_dist(ohe)
pos_dist = self.pos_dist(ohe)
if treex.num_children > 0:
# visual content
vis_dist_child = treex.children[0].vis_dist
vis_dist = self.describe(vis_dist_child, vis_dist, 'vis')
# visual position
pos_dist_child = treex.children[0].pos_dist
pos_dist = self.describe(pos_dist_child, pos_dist, 'pos')
treex.pos_dist = pos_dist
# regress bbox
treex.pos = np.clip(self.box_vae.generate(prior=treex.pos_dist).data.cpu().numpy().astype(int),
int(self.ds),
self.im_size).flatten() // self.ds
#print treex.pos
#treex.pos = [6, 5]
if treex.parent == None:
ones = self.get_ones(torch.Size([1, 1]))
if not self.bg_bias:
bg_vis_dist = [Variable(torch.zeros(latent_canvas_size)).cuda(), \
Variable(torch.zeros(latent_canvas_size)).cuda()]
else:
bg_vis_dist = [self.bias_mean(ones).view(*latent_canvas_size), \
self.bias_var(ones).view(*latent_canvas_size)]
b = [int(latent_canvas_size[2]) // 2 - treex.pos[0] // 2,
int(latent_canvas_size[3]) // 2 - treex.pos[1] // 2, treex.pos[0], treex.pos[1]]
#b[:2] = [7, 8]
#b[:2] = [6, 6]
bg_vis_dist = [self.assign_util(bg_vis_dist[0], b, self.transform(vis_dist[0], treex.pos),
'assign'), \
self.assign_util(bg_vis_dist[1], b,
self.transform(vis_dist[1], treex.pos, variance=True),
'assign')]
vis_dist = bg_vis_dist
treex.offsets = b
else:
# resize vis_dist
vis_dist = [self.transform(vis_dist[0], treex.pos), \
self.transform(vis_dist[1], treex.pos, variance=True)]
treex.vis_dist = vis_dist
elif treex.function == 'layout':
# get pos word as position prior
treex.pos_dist = self.pos_dist(ohe)
assert (treex.num_children > 0)
# get offsets: use gt for training
l_pos = treex.children[0].pos
r_pos = treex.children[1].pos
offsets = np.clip(self.offset_vae.generate(prior=treex.pos_dist).data.cpu().numpy().astype(int), 0,
self.im_size).flatten() // self.ds
countdown = 0
while self.check_valid(offsets, l_pos, r_pos, self.im_size // self.ds) == False:
offsets = np.clip(self.offset_vae.generate(prior=treex.pos_dist).data.cpu().numpy().astype(int), 0,
self.im_size).flatten() // self.ds
if countdown >= 100:
print('Tried proposing more than 100 times.')
if self.debug_mode:
import IPython;
IPython.embed()
print('Warning! Manually adapt offsets')
lat_size = self.im_size // self.ds
if offsets[0] + l_pos[0] > lat_size:
offsets[0] = lat_size - l_pos[0]
if offsets[1] + l_pos[1] > lat_size:
offsets[1] = lat_size - l_pos[1]
if offsets[2] + r_pos[0] > lat_size:
offsets[2] = lat_size - r_pos[0]
if offsets[3] + r_pos[1] > lat_size:
offsets[3] = lat_size - r_pos[1]
countdown += 1
treex.offsets = offsets
l_offset = offsets[:2]
r_offset = offsets[2:]
######################### constructing latent map ###############################
# bias filled mean&var
ones = self.get_ones(torch.Size([1, 1]))
if not self.bg_bias:
bg_vis_dist = [Variable(torch.zeros(latent_canvas_size)).cuda(), \
Variable(torch.zeros(latent_canvas_size)).cuda()]
else:
bg_vis_dist = [self.bias_mean(ones).view(*latent_canvas_size), \
self.bias_var(ones).view(*latent_canvas_size)]
vis_dist = bg_vis_dist
try:
# arrange the layout of two children
vis_dist[0] = self.assign_util(vis_dist[0], list(l_offset) + list(l_pos), treex.children[0].vis_dist[0],
'assign')
vis_dist[1] = self.assign_util(vis_dist[1], list(l_offset) + list(l_pos), treex.children[0].vis_dist[1],
'assign')
vis_dist[0] = self.assign_util(vis_dist[0], list(r_offset) + list(r_pos), treex.children[1].vis_dist[0],
'assign')
vis_dist[1] = self.assign_util(vis_dist[1], list(r_offset) + list(r_pos), treex.children[1].vis_dist[1],
'assign')
except:
print('latent distribution doesnt fit size.')
import IPython;
IPython.embed()
if treex.parent != None:
p = [min(l_offset[0], r_offset[0]), min(l_offset[1], r_offset[1]), \
max(l_offset[0] + l_pos[0], r_offset[0] + r_pos[0]),
max(l_offset[1] + l_pos[1], r_offset[1] + r_pos[1])]
treex.pos = [p[2] - p[0], p[3] - p[1]]
treex.vis_dist = [vis_dist[0][:, :, p[0]:p[2], p[1]:p[3]], \
vis_dist[1][:, :, p[0]:p[2], p[1]:p[3]]]
else:
treex.vis_dist = vis_dist
return treex
def main():
# Runs code in "functions.py" files, provided in configuration directories.
config.exec_custom_functions()
# Sets up logging
if config.debug_mode:
logging.getLogger().setLevel(logging.DEBUG)
logging.debug("Setting log level to DEBUG (debug_mode is enabled)")
# Set up graceful exit for SIGTERM (so finally clauses might have a chance to execute)
def handle_sigterm(*args):
logging.warn("Exiting due to SIGTERM")
sys.exit(0)
signal.signal(signal.SIGTERM, handle_sigterm)
# Validates constraints on the configuration data (does not touch database)
validate_config()
# Performs database migrations as needed
migrate()
# Validates constraints on the configuration data and database data in conjunction
validate_database_constraints()
if config.mode == "ipython":
# If we're running --ipython mode, STOP here (don't interfere with a server that may be
# running simultaneuosly). Launch the IPython shell and wait for user input.
import IPython
return IPython.embed()
elif config.mode == "server":
# Run ob2 in server mode.
#
# First, we clean up our resumable queues by re-enqueuing any half-completed transactions.
# Then, we reset the state of the local Docker daemon.
# Then, we start all our worker threads.
# Finally, the main thread goes to sleep until we receive a signal.
# Recovers the resumable queue used for the mailer thread (if mailer is enabled)
if config.mailer_enabled:
mailer_queue.recover()
# Recovers the resumable queue used for the GitHub API thread (if GitHub is NOT in read-only
# mode)
if not config.github_read_only_mode:
repomanager_queue.recover()
# Clears out stray Docker containers and images
reset_grader()
# Start background threads for all the apps
# Warning: Do not try to start more than 1 web thread. The web server is already threaded.
apps = [(ob2.dockergrader, config.num_workers), (ob2.web, 1)]
if config.mailer_enabled:
apps.append((ob2.mailer, 1))
if not config.github_read_only_mode:
# The GitHub repo manager thread is only needed if GitHub is NOT in read-only mode
apps.append((ob2.repomanager, 1))
for app, num_workers in apps:
for _ in range(num_workers):
worker = Thread(target=app.main)
worker.daemon = True
worker.start()
# Wait until we're asked to quit
while True:
try:
signal.pause()
except (KeyboardInterrupt, SystemExit):
logging.warn("Shutting down.. Goodbye world.")
break
except:
traceback.print_exc()
def forward(self, x, treex, treeindex=None, alpha=1.0, ifmask=False, maskweight=1.0):
################################
## input: images, trees ##
################################
# if multigpu_full, pick the trees by treeindex
if self.multigpu_full:
treex_pick = [treex[ele[0]] for ele in treeindex.data.cpu().numpy().astype(int)]
treex = treex_pick
if ifmask == True:
mask = []
for i in range(0, len(treex)):
mask += [self.get_mask_from_tree(treex[i], x[0:1, :, :, :].size())]
mask = torch.cat(mask, dim=0)
# encoding the images
h = self.reader(x)
# proposal distribution
latent_mean = self.h_mean(h)
latent_var = self.h_var(h)
# losses
kld_loss, rec_loss, pos_loss = 0, 0, 0
# forward GNMN
prior_mean_all = []
prior_var_all = []
trees = []
for i in range(0, len(treex)): # iterate through every tree of the batch
trees.append(self.compose_tree(treex[i], self.latent_canvas_size))
prior_mean_all += [trees[i].vis_dist[0]]
prior_var_all += [trees[i].vis_dist[1]]
pos_loss += trees[i].pos_loss
if np.isnan(trees[i].pos_loss.data.cpu().numpy()):
print('found nan pos loss')
import IPython;
IPython.embed()
prior_mean = torch.cat(prior_mean_all, dim=0)
prior_var = torch.cat(prior_var_all, dim=0)
prior_mean, prior_var = self.renderer([prior_mean, prior_var])
# sample z map
z_map = self.sampler(latent_mean, latent_var)
# kld loss
kld_loss = alpha * self.bikld([latent_mean, latent_var], [prior_mean, prior_var]) + \
(1 - alpha) * self.bikld([latent_mean.detach(), latent_var.detach()], [prior_mean, prior_var])
rec = self.writer(z_map)
if ifmask is True:
mask = (mask + maskweight) / (maskweight + 1.0)
rec_loss = self.pixelrecon_criterion(mask * rec, mask * x)
else:
rec_loss = self.pixelrecon_criterion(rec, x)
rec_loss = rec_loss.sum()
return rec_loss, kld_loss, pos_loss, rec
def run(self, config_dict_file, config_dict):
config_dict['n_hidden_to3Dpose'] = config_dict.get(
'n_hidden_to3Dpose', 2)
if 1: # load small example data
import pickle
data_loader = pickle.load(open('../examples/test_set.pickl', "rb"))
else:
data_loader = self.load_data_test(config_dict)
# save example data
if 0:
import pickle
IPython.embed()
data_iterator = iter(data_loader)
data_cach = [next(data_iterator) for i in range(10)]
data_cach = tuple(data_cach)
pickle.dump(data_cach, open('../examples/test_set.pickl',
"wb"))
# load model
model = self.load_network(config_dict)
model = model.to(device)
def tensor_to_npimg(torch_array):
return np.swapaxes(np.swapaxes(torch_array.numpy(), 0, 2), 0, 1)
def denormalize(np_array):
return np_array * np.array(config_dict['img_std']) + np.array(
config_dict['img_mean'])
# extract image
def tensor_to_img(output_tensor):
output_img = tensor_to_npimg(output_tensor)
output_img = denormalize(output_img)
output_img = np.clip(output_img, 0, 1)
return output_img
def rotationMatrixXZY(theta, phi, psi):
Ax = np.matrix([[1, 0, 0], [0, np.cos(theta), -np.sin(theta)],
[0, np.sin(theta), np.cos(theta)]])
Ay = np.matrix([[np.cos(phi), 0, -np.sin(phi)], [0, 1, 0],
[np.sin(phi), 0, np.cos(phi)]])
Az = np.matrix([
[np.cos(psi), -np.sin(psi), 0],
[np.sin(psi), np.cos(psi), 0],
[0, 0, 1],
])
return Az * Ay * Ax
# get next image
input_dict, label_dict = None, None
data_iterator = iter(data_loader)
def nextImage():
nonlocal input_dict, label_dict
input_dict, label_dict = next(data_iterator)
input_dict['external_rotation_global'] = torch.from_numpy(
np.eye(3)).float().to(device)
nextImage()
# apply model on images
output_dict = None
def predict():
nonlocal output_dict
model.eval()
with torch.no_grad():
input_dict_cuda, label_dict_cuda = utils_data.nestedDictToDevice(
(input_dict, label_dict), device=device)
output_dict_cuda = model(input_dict_cuda)
output_dict = utils_data.nestedDictToDevice(output_dict_cuda,
device='cpu')
predict()
# init figure
my_dpi = 400
fig, ax_blank = plt.subplots(figsize=(5 * 800 / my_dpi,
5 * 300 / my_dpi))
plt.axis('off')
# gt skeleton
ax_gt_skel = fig.add_subplot(111, projection='3d')
ax_gt_skel.set_position([0.8, 0.0, 0.2, 0.98])
handle_gt_skel = utils_plt.plot_3Dpose_simple(
ax_gt_skel,
label_dict['3D'][0].numpy().reshape([-1, 3]).T,
bones=utils_skel.bones_h36m,
linewidth=5,
plot_handles=None) # , colormap='Greys')
ax_gt_skel.invert_zaxis()
ax_gt_skel.grid(False)
ax_gt_skel.set_axis_off()
ax_gt_skel.set_title("GT pose")
# output skeleton
ax_pred_skel = fig.add_subplot(111, projection='3d')
ax_pred_skel.set_position([0.65, 0.0, 0.2, 0.98])
handle_pred_skel = utils_plt.plot_3Dpose_simple(
ax_pred_skel,
label_dict['3D'][0].numpy().reshape([-1, 3]).T,
bones=utils_skel.bones_h36m,
linewidth=5,
plot_handles=None) # , colormap='Greys')
ax_pred_skel.invert_zaxis()
ax_pred_skel.grid(False)
ax_pred_skel.set_axis_off()
ax_pred_skel.set_title("Pred. pose")
# input image
ax_in_img = plt.axes([-0.16, 0.2, 0.7, 0.7])
ax_in_img.axis('off')
im_input = plt.imshow(tensor_to_img(input_dict['img_crop'][0]),
animated=True)
ax_in_img.set_title("Input img")
# output image
ax_out_img = plt.axes([0.15, 0.2, 0.7, 0.7])
ax_out_img.axis('off')
im_pred = plt.imshow(tensor_to_img(output_dict['img_crop'][0]),
animated=True)
ax_out_img.set_title("Output img")
# update figure with new data
def update_figure():
# images
im_input.set_array(tensor_to_img(input_dict['img_crop'][0]))
im_pred.set_array(tensor_to_img(output_dict['img_crop'][0]))
# gt 3D poses
gt_pose = label_dict['3D'][0]
R_cam_2_world = label_dict['extrinsic_rot_inv'][0].numpy()
R_world_in_cam = la.inv(R_cam_2_world) @ input_dict[
'external_rotation_global'].cpu().numpy() @ R_cam_2_world
pose_rotated = R_world_in_cam @ gt_pose.numpy().reshape([-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_gt_skel,
pose_rotated,
bones=utils_skel.bones_h36m,
plot_handles=handle_gt_skel)
# prediction 3D poses
pose_mean = label_dict['pose_mean'][0].numpy()
pose_std = label_dict['pose_std'][0].numpy()
pred_pose = (output_dict['3D'][0].numpy().reshape(pose_mean.shape)
* pose_std) + pose_mean
pose_rotated = R_world_in_cam @ pred_pose.reshape([-1, 3]).T
utils_plt.plot_3Dpose_simple(ax_pred_skel,
pose_rotated,
bones=utils_skel.bones_h36m,
plot_handles=handle_pred_skel)
# flush drawings
fig.canvas.draw_idle()
def update_rotation(event):
rot = slider_yaw_glob.val
print("Rotationg ", rot)
batch_size = input_dict['img_crop'].size()[0]
input_dict['external_rotation_global'] = torch.from_numpy(
rotationMatrixXZY(theta=0, phi=0, psi=rot)).float().to(device)
input_dict['external_rotation_cam'] = torch.from_numpy(
np.eye(3)
).float().to(
device
) # torch.from_numpy(rotationMatrixXZY(theta=0, phi=rot, psi=0)).float().cuda()
predict()
update_figure()
ax_next = plt.axes([0.05, 0.1, 0.15, 0.04])
button_next = Button(ax_next,
'Next image',
color='lightgray',
hovercolor='0.975')
def nextButtonPressed(event):
nextImage()
predict()
update_figure()
button_next.on_clicked(nextButtonPressed)
ax_yaw_glob = plt.axes([0.25, 0.1, 0.65, 0.015], facecolor='lightgray')
slider_range = 2 * np.pi
slider_yaw_glob = Slider(ax_yaw_glob,
'Yaw',
-slider_range,
slider_range,
valinit=0)
slider_yaw_glob.on_changed(update_rotation)
plt.show()
def farthest_point_sampling(pts,
k,
initial_idx=None,
metrics=l2_norm,
skip_initial=False,
indices_dtype=numpy.int32,
distances_dtype=numpy.float32):
"""Batch operation of farthest point sampling
Code referenced from below link by @Graipher
https://codereview.stackexchange.com/questions/179561/farthest-point-algorithm-in-python
Args:
pts (numpy.ndarray or cupy.ndarray): 2-dim array (num_point, coord_dim)
or 3-dim array (batch_size, num_point, coord_dim)
When input is 2-dim array, it is treated as 3-dim array with
`batch_size=1`.
k (int): number of points to sample
initial_idx (int): initial index to start farthest point sampling.
`None` indicates to sample from random index,
in this case the returned value is not deterministic.
metrics (callable): metrics function, indicates how to calc distance.
skip_initial (bool): If True, initial point is skipped to store as
farthest point. It stabilizes the function output.
xp (numpy or cupy):
indices_dtype (): dtype of output `indices`
distances_dtype (): dtype of output `distances`
Returns (tuple): `indices` and `distances`.
indices (numpy.ndarray or cupy.ndarray): 2-dim array (batch_size, k, )
indices of sampled farthest points.
`pts[indices[i, j]]` represents `i-th` batch element of `j-th`
farthest point.
distances (numpy.ndarray or cupy.ndarray): 3-dim array
(batch_size, k, num_point)
"""
if pts.ndim == 2:
# insert batch_size axis
pts = pts[None, ...]
assert pts.ndim == 3
xp = cuda.get_array_module(pts)
batch_size, num_point, coord_dim = pts.shape
indices = xp.zeros((
batch_size,
k,
), dtype=indices_dtype)
# distances[bs, i, j] is distance between i-th farthest point `pts[bs, i]`
# and j-th input point `pts[bs, j]`.
distances = xp.zeros((batch_size, k, num_point), dtype=distances_dtype)
if initial_idx is None:
indices[:, 0] = xp.random.randint(len(pts))
else:
indices[:, 0] = initial_idx
batch_indices = xp.arange(batch_size)
farthest_point = pts[batch_indices, indices[:, 0]]
# minimum distances to the sampled farthest point
try:
min_distances = metrics(farthest_point[:, None, :], pts)
except Exception as e:
import IPython
IPython.embed()
if skip_initial:
# Override 0-th `indices` by the farthest point of `initial_idx`
indices[:, 0] = xp.argmax(min_distances, axis=1)
farthest_point = pts[batch_indices, indices[:, 0]]
min_distances = metrics(farthest_point[:, None, :], pts)
distances[:, 0, :] = min_distances
for i in range(1, k):
indices[:, i] = xp.argmax(min_distances, axis=1)
farthest_point = pts[batch_indices, indices[:, i]]
dist = metrics(farthest_point[:, None, :], pts)
distances[:, i, :] = dist
min_distances = xp.minimum(min_distances, dist)
return indices, distances
def compose_tree(self, treex, latent_canvas_size):
for i in range(0, treex.num_children):
treex.children[i] = self.compose_tree(treex.children[i], latent_canvas_size)
# one hot embedding of a word
ohe = self.get_code(self.dictionary, treex.word)
if treex.function == 'combine':
vis_dist = self.vis_dist(ohe)
pos_dist = self.pos_dist(ohe)
if treex.num_children > 0:
# visual content
vis_dist_child = treex.children[0].vis_dist
vis_dist = self.combine(vis_dist, vis_dist_child, 'vis')
# visual position
pos_dist_child = treex.children[0].pos_dist
pos_dist = self.combine(pos_dist, pos_dist_child, 'pos')
treex.vis_dist = vis_dist
treex.pos_dist = pos_dist
elif treex.function == 'describe':
# blend visual words
vis_dist = self.vis_dist(ohe)
pos_dist = self.pos_dist(ohe)
if treex.num_children > 0:
# visual content
vis_dist_child = treex.children[0].vis_dist
vis_dist = self.describe(vis_dist_child, vis_dist, 'vis')
# visual position
pos_dist_child = treex.children[0].pos_dist
pos_dist = self.describe(pos_dist_child, pos_dist, 'pos')
treex.pos_dist = pos_dist
# regress bbox
treex.pos = np.maximum(treex.bbox[2:] // self.ds, [1, 1])
target_box = Variable(torch.from_numpy(np.array(treex.bbox[2:])[np.newaxis, ...].astype(np.float32))).cuda()
regress_box, kl_box = self.box_vae(target_box, prior=treex.pos_dist)
treex.pos_loss = self.pos_criterion(regress_box, target_box) + kl_box
if treex.parent == None:
ones = self.get_ones(torch.Size([1, 1]))
if not self.bg_bias:
bg_vis_dist = [Variable(torch.zeros(latent_canvas_size)).cuda(), \
Variable(torch.zeros(latent_canvas_size)).cuda()]
else:
bg_vis_dist = [self.bias_mean(ones).view(*latent_canvas_size), \
self.bias_var(ones).view(*latent_canvas_size)]
b = np.maximum(treex.bbox // self.ds, [0, 0, 1, 1])
bg_vis_dist = [self.assign_util(bg_vis_dist[0], b, self.transform(vis_dist[0], treex.pos),
'assign'), \
self.assign_util(bg_vis_dist[1], b,
self.transform(vis_dist[1], treex.pos, variance=True),
'assign')]
vis_dist = bg_vis_dist
else:
try:
# resize vis_dist
vis_dist = [self.transform(vis_dist[0], treex.pos), \
self.transform(vis_dist[1], treex.pos, variance=True)]
except:
import IPython;
IPython.embed()
treex.vis_dist = vis_dist
elif treex.function == 'layout':
# get pos word as position prior
treex.pos_dist = self.pos_dist(ohe)
assert (treex.num_children > 0)
# get offsets: use gt for training
l_pos = treex.children[0].pos
l_offset = np.maximum(treex.children[0].bbox[:2] // self.ds, [1, 1])
r_pos = treex.children[1].pos
r_offset = np.maximum(treex.children[1].bbox[:2] // self.ds, [1, 1])
# regress offsets
target_offset = np.append(l_offset * self.ds, r_offset * self.ds).astype(np.float32)
target_offset = Variable(torch.from_numpy(target_offset[np.newaxis, ...])).cuda()
regress_offset, kl_offset = self.offset_vae(target_offset, prior=treex.pos_dist)
treex.pos_loss = self.pos_criterion(regress_offset, target_offset) + kl_offset + treex.children[
0].pos_loss + \
treex.children[1].pos_loss
######################### constructing latent map ###############################
# bias filled mean&var
ones = self.get_ones(torch.Size([1, 1]))
if not self.bg_bias:
vis_dist = [Variable(torch.zeros(latent_canvas_size)).cuda(), \
Variable(torch.zeros(latent_canvas_size)).cuda()]
else:
vis_dist = [self.bias_mean(ones).view(*latent_canvas_size), \
self.bias_var(ones).view(*latent_canvas_size)]
# arrange the layout of two children
vis_dist[0] = self.assign_util(vis_dist[0], list(l_offset) + list(l_pos), treex.children[0].vis_dist[0],
'assign')
vis_dist[1] = self.assign_util(vis_dist[1], list(l_offset) + list(l_pos), treex.children[0].vis_dist[1],
'assign')
vis_dist[0] = self.assign_util(vis_dist[0], list(r_offset) + list(r_pos), treex.children[1].vis_dist[0],
'assign')
vis_dist[1] = self.assign_util(vis_dist[1], list(r_offset) + list(r_pos), treex.children[1].vis_dist[1],
'assign')
# continue layout
if treex.parent != None:
p = [min(l_offset[0], r_offset[0]), min(l_offset[1], r_offset[1]), \
max(l_offset[0] + l_pos[0], r_offset[0] + r_pos[0]),
max(l_offset[1] + l_pos[1], r_offset[1] + r_pos[1])]
treex.pos = [p[2] - p[0], p[3] - p[1]]
treex.vis_dist = [vis_dist[0][:, :, p[0]:p[2], p[1]:p[3]], \
vis_dist[1][:, :, p[0]:p[2], p[1]:p[3]]]
else:
treex.vis_dist = vis_dist
return treex
