def get_ai_move(board: List[int], player: int, tic_net: Network, randomness: float) -> int:
""" return which space to put player's mark in """
# copy board to not modify original
board_copy = np.copy(board)
# make it so I am player 1
board_copy *= player
# work with preferred transformation
transform_used, preferred_transformation = choose_transformation(board_copy)
L.log("preferred transform:")
L.log(preferred_transformation)
outputs = tic_net.predict(np.array([preferred_transformation]))
L.log(outputs[0])
valid_moves = get_valid_moves(preferred_transformation)
max_move = next(iter(valid_moves))
for valid_move in valid_moves:
if outputs[0][valid_move] > outputs[0][max_move]:
max_move = valid_move
prob_of_using_max_move = (outputs[0][max_move] / (randomness + 0.0000001) - 1)
if random() < 1 - prob_of_using_max_move:
L.log("made a random choice from probability", 1 - prob_of_using_max_move)
return choice(tuple(get_valid_moves(board)))
# now transform max_move back to original board
return transform(max_move, invert_transform[transform_used])
def test_transform():
new_df, _ = transforms.transform(df,
rename=True,
replace=False,
trans_type="quadratic",
scaler=None,
symmetric=None,
verbose=0)
assert utils.checkEqual(new_df["col1_trans_quadratic"].values, [1, 4])
assert utils.checkEqual(new_df["col2_trans_quadratic"].values, [9, 16])
# Verify the others at least run
for trans_type in transforms_list:
new_df, scaler = transforms.transform(df, trans_type=trans_type)
if trans_type in ["normal", "scale01"]:
new_df, _ = transforms.transform(df, scaler=scaler)
for trans_type in comb_ops:
new_df, _ = transforms.transform(df, trans_type=trans_type)
def __init__(self,
data_path,
image_size=IMAGE_SIZE,
batch_size=BATCH_SIZE,
val_size=VAL_SIZE,
num_workers=NUM_WORKERS):
super().__init__()
self.data_path = data_path
self.image_size = image_size
self.batch_size = batch_size
self.val_size = val_size
self.num_workers = num_workers
self.transforms = transform()
def get_data(self, data_file):
coord, label = self.extract_coord_label(self.data[data_file])
coord = self.convert_coord_min_max_xy(coord)
image = self.read_image(data_file)
image = self.convert_to_PIL(image)
coord = torch.FloatTensor(coord) # (n_objects, 4)
label = torch.LongTensor(label)
image, coord, label = transform(image,
coord,
label,
self.resize,
is_train=True)
return image, coord, label
def train(move_record: List[Tuple[List[int], int]], winner: int, tic_net: Network):
""" train network based on outcome """
training_sets = []
target_output = []
player = 1
for board, moved in move_record:
valid_moves = get_valid_moves(board)
tie_value = 0.7 # target output for training data leading to tie
invalid_value = 0.5 # target output for training data of invalid move
# for moves I could have taken but didn't
default_value = 0.3 if player == winner else (0.5 if winner == (-1 * player) else 0.5)
# TODO: maybe, instead of 0.5, use the value that the net already predicts for that space?
out = [(default_value if v == 0 else invalid_value) for v in board]
equals = equal_indexes(board, moved)
for i in equals:
# don't know what values I should use if I'm not sure how good of a move this is
out[i] = 0.9 if winner == player else (0.1 if winner == (-1 * player) else tie_value)
# overwrite that if there are winning moves or blocking moves or only 1 possible move
winning_moves = get_winning_moves(board, player)
if len(winning_moves) > 0:
out = [1 if i in winning_moves else (0 if (i in valid_moves) else invalid_value)
for i in range(9)]
else: # no winning moves
can, blocking_move = can_block(board, player)
if can:
out = [1 if i == blocking_move else (0 if (i in valid_moves) else invalid_value)
for i in range(9)]
else: # no wining moves and no blocking moves
if len(valid_moves) == 1:
L.log("only 1 valid on this board")
out[next(iter(valid_moves))] = 1
board_copy = np.copy(board)
board_copy *= player
transform_used, preferred_transform_of_board = choose_transformation(board_copy)
transformed_output = transform(np.array(out), transform_used)
L.log("training:")
L.log(preferred_transform_of_board)
L.log(transformed_output)
training_sets.append(preferred_transform_of_board)
target_output.append(transformed_output)
player *= -1
tic_net.train(np.array(training_sets), np.array(target_output), 1, 0.0625, L.logging)
def test_transform(test_df, spark, transform):
"""Test transform returns adoptions per year"""
schema = StructType([
StructField('AdoptedYear', IntegerType()),
StructField('Species', StringType()),
StructField('count', LongType(), False)
])
expected = spark.createDataFrame([(2017, 'Cat', 1), (2017, 'Dog', 1),
(2018, 'Cat', 1), (2018, 'Dog', 2),
(2019, 'Cat', 1), (2019, 'Dog', 2)],
schema=schema)
result = transform(test_df)
assert_dataframe_equal(expected, result)
num_classes = len(data_legend.emotion.unique())
batch_size = 20
learning_rate = 0.0001
#CUDA
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = torchvision.models.resnet18(pretrained=True)
model = model.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
#optimizer = torch.optim.Adagrad(model.parameters(), lr=learning_rate)
model.conv1.in_channels = 1
train_loader, valid_loader = transform(label_path, image_path, batch_size)
train_losses = []
valid_losses = []
print('---train---')
for epoch in range(num_epoch):
train_loss = 0.0
valid_loss = 0.0
model.train()
for datas in train_loader:
image, target = datas['image'].cuda(), datas['label'].cuda()
optimizer.zero_grad()
print("Estimating memory usage of {}GB".format(GB))
# Set the various sheares, scales, first in terms of range
sheares, scales = set_shear_and_scale_ranges(
padded_images[0].shape, shear_steps=shear_steps, scale_steps=scale_steps, pix=2
)
# Then in terms of transform matrices
print("Calculating transform matrices")
transform_matrices = set_transform_matrices(angles, sheares, scales)
rotation_matrices, shear_matrices, scale_matrices = transform_matrices
# Scale and shear the masked data
print("Transforming data")
data = transform(
padded_images, rotation_matrices, shear_matrices, scale_matrices, weights=weights
)
data = data.astype("float32")
data = data.reshape((len(padded_images), -1) + padded_images[0].shape)
data = data.swapaxes(0, 1)
print("Preparing correlation data")
correlation_data = prepare_correlation_data(
data, weights, method=correlation_method, gpu=gpu
)
print("Correlating")
max_indexes, shifts_list = correlate_images(
correlation_data, method=correlation_method, gpu=gpu
def reDraw(self):
pyc.background(255)
pyc.lightSpecular(255 * 30 / 640, 255 * 30 / 640, 255 * 30 / 640)
pyc.directionalLight(255, 255, 255, 0, 0, 1)
pyc.specular(102, 102, 102)
# Project self.pts into image, and display it
imgTmp = cv.cvCloneImage(self.img)
imNP = cv.adaptors.Ipl2NumPy(scale(imgTmp, self.zoom))
color_list = [(255, 255, 0), (255, 0, 0), (0, 255, 255), (0, 255, 0),
(0, 0, 255), (0, 100, 100), (100, 100, 0), (100, 0, 100),
(100, 200, 100), (200, 100, 100), (100, 100, 200),
(100, 0, 200), (0, 200, 100), (0, 100, 200),
(200, 0, 100), (100, 0, 100), (255, 152, 7)]
XformPts = tr.transform(self.transFunc(self.vals), self.pts)
camPts = self.cam_proj_mat * tr.xyzToHomogenous(XformPts)
camPts = camPts / camPts[2]
camPts[0] = (camPts[0] + self.cam_centers[0]) * self.zoom
camPts[1] = (camPts[1] + self.cam_centers[1]) * self.zoom
camPts = np.matrix(np.round(camPts), 'int')
conditions = np.concatenate([
camPts[0] >= 0, camPts[0] < imNP.shape[1], camPts[1] >= 0,
camPts[1] < imNP.shape[0]
], 0)
r, c = np.where(np.all(conditions, 0))
camPts_bound = camPts[:, c.A[0]]
x = np.asarray(self.pts[0])[0][c.A[0]]
x = x - x.min()
x = x / x.max() * 256 #512 #number of colors
x = np.floor(x)
x = np.asarray(np.matrix(x, 'int'))[0]
if self.display_laser:
map2d = np.asarray(camPts_bound[0:2])
n, m = map2d.shape
for i in range(0, m):
imNP[map2d[1, i],
map2d[0, i], :] = [x[i], 256 - x[i],
128 + x[i] / 2] #color_list[x[i]]
imgTmp = cv.adaptors.NumPy2Ipl(imNP)
#imgSmall = cv.cvCreateImage(cv.cvSize(imgTmp.width/3, imgTmp.height/3), cv.IPL_DEPTH_8U, 3)
#cv.cvResize(imgTmp, imgSmall, cv.CV_INTER_AREA)
im = cv.adaptors.Ipl2PIL(imgTmp)
#pyc.rotate(math.radians(90))
pyc.image(im, 0, 0, self.width, self.height)
#pyc.rotate(math.radians(-90))
# Display the current values of the parameter vector (and highlight the selected one)
pyc.textSize(10)
for i, val in enumerate(self.vals):
if np.nonzero(self.selected)[0] == i:
print 'x',
print '%8.4f ' % self.vals[i],
pval = '%7s: %8.4f' % (self.names[i], self.vals[i])
pyc.text(pval, self.width + 15, 20 + 20 * i, 0)
if np.nonzero(self.selected)[0] == i:
pyc.fill(255, 0, 0)
pyc.quad(self.width + 4.0, 15.0 + 20.0 * i - 7.0,
self.width + 4.0, 15.0 + 20.0 * i + 7.0,
self.width + 13.0, 15.0 + 20.0 * i, self.width + 13.0,
15.0 + 20.0 * i)
print '\n'
self.move(pyc.escape_handler(pyc.draw()))
def main(train_dir,
val_dir,
checkpoint_dir,
batch_size,
image_size=512,
num_epochs=10,
checkpoint_name=None,
num_workers=1,
pin_memory=True,
log_dir="logs",
model_name=None,
train_csv=None,
val_csv=None):
# declare datasets
train_ds = DataFolder(root_dir=train_dir,
transform=transform(image_size, is_training=True),
csv_path=train_csv)
val_ds = DataFolder(root_dir=val_dir,
transform=transform(image_size, is_training=False),
csv_path=val_csv)
train_loader = DataLoader(train_ds,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=pin_memory,
shuffle=True)
val_loader = DataLoader(val_ds,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=pin_memory,
shuffle=True)
#init model
model = MainModel(128, model_name)
# configure parameter
loss_fn = nn.CrossEntropyLoss()
model = model.to(device)
optimizer = optim.Adam(model.parameters(), lr=1e-4)
scaler = torch.cuda.amp.GradScaler()
# checkpoint = {'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict()}
# save_checkpoint(checkpoint, os.path.join(checkpoint_dir, f"checkpoint_initialilze.pth.tar"))
# return
if checkpoint_name:
ckp_path = os.path.join(checkpoint_dir, checkpoint_name)
load_checkpoint(torch.load(ckp_path), model, optimizer)
check_accuracy(val_loader, model, device)
#training
for epoch in range(num_epochs):
train_fn(train_loader,
model,
optimizer,
loss_fn,
scaler,
device,
epoch,
log_dir=log_dir)
check_accuracy(val_loader, model, device)
checkpoint = {
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict()
}
save_checkpoint(
checkpoint,
os.path.join(checkpoint_dir, f"checkpoint_{epoch}.pth.tar"))
def transform_features(features, scaler_normal=None, scaler_01=None):
"""Prepare features, including handling missing values, normalization, PCA, etc.
Args:
url (string): The download address.
path (string): The location to save the download.
Returns:
bool: True if successful, False otherwise.
"""
# Split categorical features
tags = {
'Condition1_RRAe_orig': 'categ',
'HouseStyle_SFoyer_orig': 'categ',
'MSSubClass_20_orig': 'categ',
'RoofMatl_Tar&Grv_orig': 'categ',
'MSSubClass_45_orig': 'categ',
'MoSold_orig': 'cont',
'HouseStyle_1.5Fin_orig': 'categ',
'Heating_GasW_orig': 'categ',
'Exterior1st_VinylSd_orig': 'categ',
'Exterior1st_AsphShn_orig': 'categ',
'PavedDrive_orig': 'ord',
'LotShape_IR3_orig': 'categ',
'Exterior1st_ImStucc_orig': 'categ',
'LotShape_IR1_orig': 'categ',
'MSSubClass_160_orig': 'categ',
'SaleCondition_Partial_orig': 'categ',
'CentralAir_orig': 'ord',
'OpenPorchSF_orig': 'cont',
'MSZoning_FV_orig': 'categ',
'BldgType_TwnhsE_orig': 'categ',
'SaleCondition_Alloca_orig': 'categ',
'Exterior1st_BrkFace_orig': 'categ',
'LandContour_Lvl_orig': 'categ',
'SaleCondition_Normal_orig': 'categ',
'GarageType_Attchd_orig': 'categ',
'BsmtFullBath_orig': 'cont',
'GarageIdx_orig': 'cont',
'Exterior1st_Wd Sdng_orig': 'categ',
'SaleCondition_AdjLand_orig': 'categ',
'Exterior2nd_AsbShng_orig': 'categ',
'Exterior2nd_Wd Shng_orig': 'categ',
'Exterior1st_MetalSd_orig': 'categ',
'Exterior2nd_CmentBd_orig': 'categ',
'Neighborhood_NoRidge_orig': 'categ',
'PoolArea_orig': 'cont',
'3SsnPorch_orig': 'cont',
'RoofMatl_Metal_orig': 'categ',
'Neighborhood_Gilbert_orig': 'categ',
'Foundation_CBlock_orig': 'categ',
'KitchenAbvGr_orig': 'cont',
'Street_Pave_orig': 'categ',
'RoofStyle_Gable_orig': 'categ',
'HouseStyle_1Story_orig': 'categ',
'LotArea_orig': 'cont',
'Condition2_RRAe_orig': 'categ',
'MiscFeature_Othr_orig': 'categ',
'Fireplaces_orig': 'cont',
'Exterior2nd_MetalSd_orig': 'categ',
'Exterior2nd_HdBoard_orig': 'categ',
'SummerSale_orig': 'categ',
'SaleCondition_Abnorml_orig': 'categ',
'Neighborhood_Crawfor_orig': 'categ',
'Neighborhood_CollgCr_orig': 'categ',
'Neighborhood_Veenker_orig': 'categ',
'Condition1_Norm_orig': 'categ',
'GarageType_0_orig': 'categ',
'HouseStyle_SLvl_orig': 'categ',
'Neighborhood_SawyerW_orig': 'categ',
'MSSubClass_85_orig': 'categ',
'OverallQual_orig': 'cont',
'Exterior1st_Plywood_orig': 'categ',
'LotConfig_FR3_orig': 'categ',
'Heating_Wall_orig': 'categ',
'Season_0_orig': 'categ',
'LandContour_Low_orig': 'categ',
'RemodelAge_orig': 'cont',
'RoofStyle_Shed_orig': 'categ',
'MSSubClass_70_orig': 'categ',
'PoolQC_orig': 'ord',
'BsmtFinType1_orig': 'ord',
'Exterior2nd_CBlock_orig': 'categ',
'MSZoning_RH_orig': 'categ',
'MSSubClass_75_orig': 'categ',
'SQFperRoom_orig': 'cont',
'Neighborhood_Blmngtn_orig': 'categ',
'MSSubClass_120_orig': 'categ',
'Neighborhood_StoneBr_orig': 'categ',
'MSSubClass_60_orig': 'categ',
'MiscFeature_Shed_orig': 'categ',
'Exterior2nd_Wd Sdng_orig': 'categ',
'Foundation_Slab_orig': 'categ',
'Fence_orig': 'ord',
'YrSold_2006_orig': 'categ',
'Condition2_PosA_orig': 'categ',
'OverallCond_orig': 'cont',
'BsmtCond_orig': 'ord',
'BsmtExposure_orig': 'ord',
'Foundation_Stone_orig': 'categ',
'BedroomAbvGr_orig': 'cont',
'LandContour_Bnk_orig': 'categ',
'MSSubClass_30_orig': 'categ',
'Foundation_Wood_orig': 'categ',
'Exterior2nd_VinylSd_orig': 'categ',
'BsmtFinSF1_orig': 'cont',
'BldgType_Duplex_orig': 'categ',
'MSSubClass_90_orig': 'categ',
'Neighborhood_MeadowV_orig': 'categ',
'FullBath_orig': 'cont',
'BldgType_Twnhs_orig': 'categ',
'FireplaceQu_orig': 'ord',
'RoofStyle_Mansard_orig': 'categ',
'Exterior1st_CBlock_orig': 'categ',
'Condition1_PosA_orig': 'categ',
'Season_3_orig': 'categ',
'MSSubClass_80_orig': 'categ',
'ExterCond_orig': 'ord',
'GarageType_2Types_orig': 'categ',
'LargeHouse_orig': 'categ',
'Exterior1st_CemntBd_orig': 'categ',
'HouseStyle_2.5Fin_orig': 'categ',
'SaleType_WD_orig': 'categ',
'RoofMatl_CompShg_orig': 'categ',
'Exterior1st_HdBoard_orig': 'categ',
'Electrical_SBrkr_orig': 'categ',
'RoofStyle_Flat_orig': 'categ',
'Foundation_PConc_orig': 'categ',
'BsmtFinSF2_orig': 'cont',
'Neighborhood_IDOTRR_orig': 'categ',
'SaleType_ConLw_orig': 'categ',
'LandSlope_Mod_orig': 'categ',
'Exterior1st_Stone_orig': 'categ',
'Exterior2nd_Stucco_orig': 'categ',
'Heating_GasA_orig': 'categ',
'RoofMatl_WdShake_orig': 'categ',
'HouseAge_orig': 'cont',
'Neighborhood_NPkVill_orig': 'categ',
'Utilities_orig': 'ord',
'Exterior2nd_AsphShn_orig': 'categ',
'BsmtQual_orig': 'ord',
'GarageAgeInv_orig': 'cont',
'Exterior1st_BrkComm_orig': 'categ',
'Electrical_Mix_orig': 'categ',
'Neighborhood_ClearCr_orig': 'categ',
'LotConfig_Corner_orig': 'categ',
'SaleType_ConLD_orig': 'categ',
'BsmtHalfBath_orig': 'cont',
'YrSold_2010_orig': 'categ',
'Electrical_FuseF_orig': 'categ',
'LotShape_Reg_orig': 'categ',
'MasVnrType_orig': 'ord',
'Electrical_FuseP_orig': 'categ',
'Heating_Floor_orig': 'categ',
'GarageQual_orig': 'ord',
'RoofStyle_Gambrel_orig': 'categ',
'Condition2_Norm_orig': 'categ',
'time_index_orig': 'cont',
'GrLivArea_orig': 'cont',
'SaleType_Con_orig': 'categ',
'neighborhood_index_orig': 'cont',
'GarageType_CarPort_orig': 'categ',
'Condition1_PosN_orig': 'categ',
'MiscVal_orig': 'cont',
'Electrical_FuseA_orig': 'categ',
'Exterior1st_WdShing_orig': 'categ',
'BldgType_1Fam_orig': 'categ',
'GarageCond_orig': 'ord',
'Neighborhood_BrkSide_orig': 'categ',
'Condition2_PosN_orig': 'categ',
'LandContour_HLS_orig': 'categ',
'YrSold_2007_orig': 'categ',
'Neighborhood_BrDale_orig': 'categ',
'MasVnrArea_orig': 'cont',
'SaleType_CWD_orig': 'categ',
'Heating_Grav_orig': 'categ',
'KitchenQual_orig': 'ord',
'Neighborhood_NridgHt_orig': 'categ',
'LotConfig_Inside_orig': 'categ',
'RoofMatl_ClyTile_orig': 'categ',
'WoodDeckSF_orig': 'cont',
'HeatingQC_orig': 'ord',
'Condition2_RRNn_orig': 'categ',
'Neighborhood_Somerst_orig': 'categ',
'MSSubClass_40_orig': 'categ',
'MSZoning_C (all)_orig': 'categ',
'ExterQual_orig': 'ord',
'MSSubClass_190_orig': 'categ',
'Exterior2nd_Stone_orig': 'categ',
'Alley_Grvl_orig': 'categ',
'Neighborhood_Sawyer_orig': 'categ',
'Neighborhood_NWAmes_orig': 'categ',
'LotFrontage_orig': 'cont',
'Exterior2nd_Brk Cmn_orig': 'categ',
'MSSubClass_180_orig': 'categ',
'Season_2_orig': 'categ',
'Condition2_RRAn_orig': 'categ',
'BsmtFinType2_orig': 'ord',
'Condition2_Artery_orig': 'categ',
'HasPool_orig': 'categ',
'GarageFinish_orig': 'ord',
'SaleCondition_Family_orig': 'categ',
'EnclosedPorch_orig': 'cont',
'Foundation_BrkTil_orig': 'categ',
'Condition1_RRAn_orig': 'categ',
'Exterior2nd_Other_orig': 'categ',
'HouseStyle_1.5Unf_orig': 'categ',
'LotShape_IR2_orig': 'categ',
'HalfBath_orig': 'cont',
'Heating_OthW_orig': 'categ',
'LandSlope_Gtl_orig': 'categ',
'TotRmsAbvGrd_orig': 'cont',
'Condition1_RRNe_orig': 'categ',
'MSZoning_RM_orig': 'categ',
'Condition1_Feedr_orig': 'categ',
'GarageType_Detchd_orig': 'categ',
'TotalBsmtSF_orig': 'cont',
'Exterior2nd_BrkFace_orig': 'categ',
'NewHouse_orig': 'categ',
'YrSold_2008_orig': 'categ',
'RoofMatl_Roll_orig': 'categ',
'LotConfig_FR2_orig': 'categ',
'Neighborhood_Timber_orig': 'categ',
'Neighborhood_Blueste_orig': 'categ',
'Condition2_Feedr_orig': 'categ',
'2ndFlrSF_orig': 'cont',
'LotConfig_CulDSac_orig': 'categ',
'Street_Grvl_orig': 'categ',
'Exterior1st_Stucco_orig': 'categ',
'YrSold_2009_orig': 'categ',
'RoofStyle_Hip_orig': 'categ',
'BsmtUnfSF_orig': 'cont',
'Neighborhood_NAmes_orig': 'categ',
'ScreenPorch_orig': 'cont',
'Functional_orig': 'ord',
'GarageType_BuiltIn_orig': 'categ',
'Alley_Pave_orig': 'categ',
'Condition1_RRNn_orig': 'categ',
'BldgType_2fmCon_orig': 'categ',
'LandSlope_Sev_orig': 'categ',
'Condition1_Artery_orig': 'categ',
'Neighborhood_Edwards_orig': 'categ',
'GarageType_Basment_orig': 'categ',
'SaleType_New_orig': 'categ',
'Season_1_orig': 'categ',
'MSSubClass_50_orig': 'categ',
'Neighborhood_SWISU_orig': 'categ',
'Exterior2nd_ImStucc_orig': 'categ',
'HouseStyle_2.5Unf_orig': 'categ',
'HouseStyle_2Story_orig': 'categ',
'RoofMatl_WdShngl_orig': 'categ',
'SaleType_COD_orig': 'categ',
'GarageArea_orig': 'cont',
'MSZoning_RL_orig': 'categ',
'LowQualFinSF_orig': 'cont',
'Exterior1st_AsbShng_orig': 'categ',
'Neighborhood_Mitchel_orig': 'categ',
'PrchSQ_orig': 'cont',
'Neighborhood_OldTown_orig': 'categ',
'RoofMatl_Membran_orig': 'categ',
'MiscFeature_Gar2_orig': 'categ',
'1stFlrSF_orig': 'cont',
'Exterior2nd_Plywood_orig': 'categ',
'SaleType_ConLI_orig': 'categ',
'GarageCars_orig': 'cont',
'TotalSQF_orig': 'cont',
'MiscFeature_TenC_orig': 'categ',
'SaleType_Oth_orig': 'categ'
}
# Add orig tag to columns
features.columns = [
str(col) + '_orig' if col != "Id" else "Id" for col in features.columns
]
# For now, treat ordinal and continuous variables the same
dont_rescale = features[["Id", "MiscVal_orig"]]
continuous_features = features[[
feat for feat in tags.keys()
if tags[feat] == "cont" and feat not in dont_rescale
]]
ordinal_features = features[[
feat for feat in tags.keys()
if tags[feat] == "ord" and feat not in dont_rescale
]]
categorical_features = features[[
feat for feat in tags.keys()
if tags[feat] == "categ" and feat not in dont_rescale
]]
# Add epithets
continuous_features.columns = [
str(col) + '_cont' for col in continuous_features.columns
]
ordinal_features.columns = [
str(col) + '_ord' for col in ordinal_features.columns
]
categorical_features.columns = [
str(col) + '_categ' for col in categorical_features.columns
]
continuous_features_log, _ = transform(continuous_features,
rename=True,
replace=True,
trans_type="log")
continuous_features_inverse, _ = transform(continuous_features,
rename=True,
replace=True,
trans_type="inverse")
continuous_features_normal, scaler_normal = transform(continuous_features,
rename=True,
replace=True,
trans_type="normal",
scaler=scaler_normal)
continuous_features01, scaler_01 = transform(continuous_features,
rename=True,
replace=True,
trans_type="scale01",
scaler=scaler_01)
continuous_features_root, _ = transform(continuous_features,
rename=True,
replace=True,
trans_type="root")
continuous_features_quad, _ = transform(continuous_features,
rename=True,
replace=True,
trans_type="quadratic")
df_list = [
continuous_features_log, continuous_features_inverse,
continuous_features_root, continuous_features,
continuous_features_normal, continuous_features01,
continuous_features_quad
]
continuous_features = pd.concat(df_list, axis=1)
# Recombine
features = pd.concat([
dont_rescale, continuous_features, categorical_features,
ordinal_features
],
axis=1)
return features, scaler_normal, scaler_01
def create_new_features(features,
market_index=None,
neighborhood_index=None,
MoSold_or_Season_index="Season"):
""" Combine features to generate new features.
Args:
features (DataFrame): The dataframe containing training data.
market_index (DataFrame): The dataframe mapping time periods to index values.
neighborhood_index (DataFrame): The dataframe mapping neighborhoods to index values.
MoSold_or_Season_index (str): Whether to index seasonly or monthly
Returns:
(tuple): tuple containing:
features(DataFrame): DataFrame with original features combined with newly created ones
market_index (DataFrame): The dataframe mapping time periods to index values.
neighborhood_index (DataFrame): The dataframe mapping neighborhoods to index values.
"""
# Create new features
features['TotalSQF'] = features['TotalBsmtSF'] + features['GrLivArea']
features['NewHouse'] = np.less(
abs(features['YrSold'] - features['YearBuilt']), 2)
features['SummerSale'] = np.isin(features["MoSold"], range(4, 9))
features['Season'] = ((features["MoSold"] / 4)).astype(int)
features['RemodelAge'] = (features["YrSold"] -
features["YearRemodAdd"]).clip(0, None)
temp = (features["YrSold"] - features["GarageYrBlt"]).clip(0, None)
features['GarageAgeInv'] = transform(temp,
rename=False,
replace=True,
trans_type="inverse")[0]
features['GarageIdx'] = features['GarageAgeInv'] * features[
'GarageFinish'] * features['GarageCars']
features['HouseAge'] = (features['YrSold'] - features['YearBuilt']).clip(
0, None)
features['LargeHouse'] = features["TotalSQF"] > 4000
features['SQFperRoom'] = features["TotalSQF"] / features.TotRmsAbvGrd
features["HasPool"] = features["PoolArea"] > 0
features["PrchSQ"] = np.nansum(features[get_features(features, "porch")],
axis=1)
merge_vars = ["YrSold", MoSold_or_Season_index]
if market_index is None:
market_index = create_market_index(features, "time", merge_vars)
features = pd.merge(features,
market_index,
how='left',
left_on=merge_vars,
right_on=merge_vars)
merge_vars = ["Neighborhood"]
if neighborhood_index is None:
neighborhood_index = create_market_index(features, "neighborhood",
merge_vars)
features = pd.merge(features,
neighborhood_index,
how='left',
left_on=merge_vars,
right_on=merge_vars)
return features, market_index, neighborhood_index
def specific_transform(features, column_name, verbose=0):
"""Backout transformation from variable name. Pretty messy.
Args:
features (DataFrame): Dataframe with base features
column_name (str): Name of column from which to derive transformation
verbose (int, optional): Verbose messages for verbose>0
Returns:
DataFrame: DataFrame with only the newly created variable
"""
vprint = print if verbose else lambda *a, **k: None
# List all transform suffixes
var_types = ["categ", "ord", "cont"]
misc_keywords = ["trans"]
comb_ops = ["add", "subtract", "multiply", "divide"]
transforms = [
"log", "inverse", "quadratic", "cubic", "root", "normal", "scale01"
]
all_keywords = misc_keywords + comb_ops + transforms
c = column_name.replace(" ", "^%^")
l = c.split("_")
# Get the "base" variables
base_variables = '_'.join([x if x not in all_keywords else ' ' for x in l])
base_variables = re.sub("(_ )+_", " ", base_variables).strip().split(" ")
base_variables = [b.replace("^%^", " ") for b in base_variables
] # ugly hack for dealing with spaces
vprint(base_variables)
# Process "base" variable to get original, untransformed variable (e.g. add orig to it etc.)
original_variables = base_variables[:]
for n, o in enumerate(original_variables):
for var_type in var_types:
original_variables[n] = original_variables[n].replace(
var_type, "orig_" + var_type)
vprint(original_variables)
# Get string defining transformations
for o in set(base_variables):
column_name = column_name.replace(o, "|" + o + "|")
parsed_variables = column_name.split("|")[1:]
vprint(parsed_variables)
# Create a list containing [[original_var_name, str_of_transforms],...]
var_list = []
for n in range(0, len(parsed_variables), 2):
var = original_variables[int(n / 2)]
trans = parsed_variables[n + 1]
var_list.append([var, trans])
# Check for individual transforms
vprint(var_list)
for var in var_list:
feat_col = features[[var[0]]]
for t in transforms:
if t in var[1]:
new_df = transform(feat_col,
rename=True,
replace=True,
trans_type=t,
verbose=0)[0]
var.append(new_df)
break # Assume one transform
if len(var) < 3:
var.append(feat_col)
# Check for variable combinations
if len(var_list) > 1:
new_features = pd.concat([var_list[0][2], var_list[1][2]], axis=1)
for op in comb_ops:
if op in var_list[0][1]:
combined_feature = transform(new_features,
rename=True,
replace=True,
trans_type=op,
verbose=0)[0]
break
else:
combined_feature = var_list[0][2]
combined_feature = pd.concat([features, combined_feature], axis=1)
return combined_feature
def main(test_dir,
checkpoint_path,
batch_size,
num_workers=1,
pin_memory=True,
test_csv=None,
model_name='efficientnet-b3'):
# declare datasets
test_ds = DataFolder(root_dir=test_dir,
transform=transform(is_training=False),
is_test=True,
csv_path=test_csv)
test_loader = DataLoader(test_ds,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=pin_memory,
shuffle=True)
#init model
model = MainModel(test_ds.__num_class__(), model_name)
model = model.to(device)
# load checkpoint
load_checkpoint(torch.load(checkpoint_path), model)
model.eval()
iterator = tqdm(test_loader)
num_correct = 0
num_samples = 0
preds = []
groundtruths = []
print(test_ds.class_names)
with torch.no_grad():
for x, y, image_paths in iterator:
#convert to device
x = x.to(device=device)
y = y.to(device=device)
# inference
scores = torch.sigmoid(model(x))
# get prediction
max_score = torch.argmax(scores, dim=1)
# add to global comparing value
preds += max_score.to("cpu").numpy().tolist()
groundtruths += y.to("cpu").numpy().tolist()
#calculate score
predictions = max_score.float()
num_correct += (predictions == y).sum()
num_samples += predictions.shape[0]
iterator.set_postfix(
accuracy=f'{float(num_correct) / float(num_samples) * 100:.2f}'
)
# break
print(
classification_report(groundtruths,
preds,
zero_division=0,
target_names=test_ds.class_names))
def reDraw(self):
pyc.background(255)
pyc.lightSpecular(255*30/640, 255*30/640, 255*30/640)
pyc.directionalLight(255,255,255,0,0,1)
pyc.specular(102, 102, 102)
# Project self.pts into image, and display it
imgTmp = cv.cvCloneImage(self.img)
imNP = cv.adaptors.Ipl2NumPy(scale(imgTmp, self.zoom))
color_list = [(255,255,0),(255,0,0),(0,255,255),(0,255,0),(0,0,255),(0,100,100),(100,100,0),
(100,0,100),(100,200,100),(200,100,100),(100,100,200),(100,0,200),(0,200,100),
(0,100,200),(200,0,100),(100,0,100),(255,152,7) ]
XformPts = tr.transform( self.transFunc(self.vals), self.pts )
camPts = self.cam_proj_mat * tr.xyzToHomogenous(XformPts)
camPts = camPts / camPts[2]
camPts[0] = (camPts[0] + self.cam_centers[0]) * self.zoom
camPts[1] = (camPts[1] + self.cam_centers[1]) * self.zoom
camPts = np.matrix( np.round(camPts), 'int')
conditions = np.concatenate([camPts[0] >= 0,
camPts[0] < imNP.shape[1],
camPts[1] >= 0,
camPts[1] < imNP.shape[0]], 0)
r, c = np.where(np.all(conditions, 0))
camPts_bound = camPts[:, c.A[0]]
x = np.asarray(self.pts[0])[0][c.A[0]]
x = x - x.min()
x = x / x.max() * 256 #512 #number of colors
x = np.floor(x)
x = np.asarray(np.matrix(x,'int'))[0]
if self.display_laser:
map2d = np.asarray(camPts_bound[0:2])
n,m = map2d.shape
for i in range(0,m):
imNP[map2d[1,i],map2d[0,i], :] = [x[i],256-x[i],128+x[i]/2]#color_list[x[i]]
imgTmp = cv.adaptors.NumPy2Ipl(imNP)
#imgSmall = cv.cvCreateImage(cv.cvSize(imgTmp.width/3, imgTmp.height/3), cv.IPL_DEPTH_8U, 3)
#cv.cvResize(imgTmp, imgSmall, cv.CV_INTER_AREA)
im = cv.adaptors.Ipl2PIL(imgTmp)
#pyc.rotate(math.radians(90))
pyc.image(im, 0,0, self.width, self.height)
#pyc.rotate(math.radians(-90))
# Display the current values of the parameter vector (and highlight the selected one)
pyc.textSize(10)
for i, val in enumerate(self.vals):
if np.nonzero(self.selected)[0] == i:
print 'x',
print '%8.4f ' % self.vals[i],
pval = '%7s: %8.4f' % (self.names[i], self.vals[i])
pyc.text(pval, self.width + 15, 20 + 20*i, 0)
if np.nonzero(self.selected)[0] == i:
pyc.fill(255,0,0)
pyc.quad(self.width+4.0, 15.0 + 20.0*i - 7.0,
self.width+4.0, 15.0 + 20.0*i + 7.0,
self.width+13.0, 15.0 + 20.0*i,
self.width+13.0, 15.0 + 20.0*i)
print '\n'
self.move(pyc.escape_handler(pyc.draw()))
def preprocess_features(features, scaler_normal = None, scaler_01 = None):
"""Prepare features, including handling missing values, normalization, PCA, etc.
Args:
url (string): The download address.
path (string): The location to save the download.
Returns:
bool: True if successful, False otherwise.
"""
# Impute NaNs using the median
#g = features.columns.to_series().groupby(features.dtypes).groups
#print([x for x in g])
# [dtype('float64'), dtype('uint8'), dtype('bool'), dtype('int64')]
tags = features.tags
#features = smart_impute(features, ["LotFrontage"])
print(features["LotFrontage"])
features = impute(features, strategy='median')
print(features["LotFrontage"])
# Split categorical features
print_feature_dtypes(features)
features.tags = tags
id, continuous_features, categorical_features, ordinal_features = split_features(features, dont_rescale=["Id","MiscVal"])
skewed_features = get_skewed_features(continuous_features)
non_skewed_cont_features = [x for x in list(continuous_features) if x not in list(skewed_features)]
#print(non_skewed_cont_features)
continuous_features_log, _ = transform(continuous_features, rename=True, replace=True, trans_type="log")
continuous_features_inverse, _ = transform(continuous_features, rename=True, replace=True, trans_type="inverse")
continuous_features_normal, scaler_normal = transform(continuous_features, rename=True, replace=True, trans_type="normal", scaler=scaler_normal)
continuous_features01, scaler_01 = transform(continuous_features, rename=True, replace=True, trans_type="scale01", scaler=scaler_01)
continuous_features_root, _ = transform(continuous_features, rename=True, replace=True, trans_type="root")
continuous_features_quad, _ = transform(continuous_features, rename=True, replace=True, trans_type="quadratic")
df_list = [continuous_features_log, continuous_features_inverse, continuous_features_root, continuous_features, continuous_features_normal, continuous_features01, continuous_features_quad]
#
for df in df_list:
if df.isnull().values.any():
print(df.columns[0])
print(df[df.isnull()])
continuous_features = pd.concat(df_list, axis=1)
"""
if USE_LOG:
continuous_features[skewed_features] = np.log1p(continuous_features[skewed_features])
continuous_features[non_skewed_cont_features], scaler = normalize(continuous_features[non_skewed_cont_features])
else:
# Normalize
continuous_features, scaler = normalize(continuous_features, scaler)
"""
# PCA
#continuous_features = use_pca(continuous_features)
# Recombine
features = pd.concat([id, continuous_features, categorical_features, ordinal_features], axis=1)
return features, scaler_normal, scaler_01
import tqdm
import datahandling as dh
import ease_grid as eg
from validate import RETRIEVAL_MIN
from validation_db_orm import (
date_to_int,
get_db_session,
DbWMOMetDailyTempRecord,
DbWMOMetStation,
)
from transforms import N45_VIEW_TRANS as transform
pred = np.load("../runs/gscc/2020-11-02-19:51:31.769613-gold/pred.npy")
land_mask = ~transform(np.load("../data/masks/ft_esdr_water_mask.npy"))
dates, _, _ = dh.read_accuracies_file(
"../runs/gscc/2020-11-02-19:51:31.769613-gold/acc.csv"
)
d2i = {d: i for i, d in enumerate(dates)}
assert len(dates) == len(pred)
d2p = {d: p.ravel() for d, p in zip(dates, pred)}
lon, lat = [transform(x) for x in eg.v1_get_full_grid_lonlat(eg.ML)]
df = dh.get_aws_full_data_for_dates(
dates,
"../data/dbs/wmo_gsod.db",
land_mask,
lon,
lat,
