def open_annot(self):
try:
file = sf.open_annot_window(self)
self.log(file)
self.annot_handler.parse_xml(file)
sf.fill_annotation_table(self, self.ui.annot_table, self.annot_handler)
except we.FileNotSelected as msg:
self.log(msg)
def open_epub(self):
try:
file = sf.open_epub_window(self)
self.log(file)
sf.unpack_epub(file,self.main_path)
self.filename = sf.get_filename(file)
self.setWindowTitle(self.filename)
self.log("File loaded")
except we.FileNotSelected as msg:
self.log(msg)
def IHT_Joint(self, Y, max_IHT_iters, alpha):
# print("Running IHT, projecting onto support cardinality k = {0:1.0f}".format(self.k))
y_dims = list(Y.data.size())
w_dims = list(self.D_trans.weight.data.size())
# Initialise X as zero tensor
X1 = Variable(
torch.zeros(y_dims[0], w_dims[0], (y_dims[2] - w_dims[2] + 1),
(y_dims[3] - w_dims[3] + 1)).to(device, dtype=dtype))
X1_error = np.sum((Y).data.cpu().numpy()**2)
X2_error = 0
i = 0
run = True
while run == True and i < max_IHT_iters:
g = self.dropout(self.D_trans(Y - self.D(self.dropout(X1))))
HT_arg = X1 + self.alpha * g
X2, filters_selected = sf.hard_threshold_joint_k(HT_arg, self.k)
X2_error = np.sum(
((Y - self.D(self.dropout(X2))).data.cpu().numpy())**2)
if X2_error < X1_error:
X1 = X2
X1_error = X2_error
else:
run = False
if i == 0 or (i + 1) % 1 == 0:
# After run IHT print out the result
l2_error = X1_error
av_num_zeros_per_image = X1.data.nonzero().cpu().numpy(
).shape[0] / y_dims[0]
percent_zeros_per_image = 100 * av_num_zeros_per_image / (
y_dims[2] * y_dims[3])
error_percent = l2_error * 100 / (np.sum(
(Y).data.cpu().numpy()**2))
# print("After " +repr(i+1) + " iterations of IHT, average l2 error over batch: {0:1.2f}".format(error_percent) + "% , Av. sparsity per image: {0:1.2f}".format(percent_zeros_per_image) +"%")
i = i + 1
return X1, error_percent, l2_error, i, filters_selected
def save_paragraph_changes(self):
text = self.ui.after.toPlainText().__str__()
annot = sf.get_selected(self.ui.annot_table)
container = self.epub_container.parse(os.path.join("tmp_epub",annot[1]))
path = [int(i) for i in annot[2].split("/")[2:]]
container.set_content(path,text)
ee = EpubExport()
ee.export_container_to_file(os.path.join("tmp_epub",annot[1]), container)
def load_selected_annotation(self):
annot = sf.get_selected(self.ui.annot_table)
container = self.epub_container.parse(os.path.join("tmp_epub",annot[1]))
path = [int(i) for i in annot[2].split("/")[2:]]
print container.get_content(path)[int(annot[3]):int(annot[4])]
text = container.get_content(path)
sf.set_text_in_ctrl(self.ui.before, text)
sf.set_text_in_ctrl(self.ui.after, text)
sf.set_text_selection(self.ui.after,int(annot[3]),int(annot[4]))
def Infer_Repeat(self, utterance, samples=1000):
"""
Take an existing hypothesis space and compute the posterior
"""
[VW_HypothesisSpace, VW_Priors] = SF.BuildVWHypSpace(
self.VisualWorld, self.CommonGroundPrior, self.Filter)
[SpeakerBias_HypothesisSpace, SpeakerBias_Priors] = SF.BuildBiasHypSpace(
self.BiasPriors)
BiasNames = [x.name for x in self.BiasPriors]
index = 0
for SB_index in range(len(SpeakerBias_Priors)):
CurrentBias = Bias.Bias(
BiasNames, SpeakerBias_HypothesisSpace[SB_index])
for VW_index in range(len(VW_Priors)):
TestSpeaker = Speaker.Speaker(
VW_HypothesisSpace[VW_index], CurrentBias, self.SpeakerRationalityNoise)
for testobject in TestSpeaker.VisualWorld.objects:
p = TestSpeaker.GetUtteranceProbability(
utterance, testobject, samples)
self.HypothesisSpace.Grow(
VW_HypothesisSpace[VW_index].GetIDs(), CurrentBias, testobject.Id, utterance, p)
#self.HypothesisSpace.AddTrial(index, utterance, p)
index += 1
def QueryFourSquare():
print(">>> Action 2 : QueryFourSquare")
CLIENT_ID = 'ARRNPU155D51DSX4JHDRNJLSAR3GS2JB0BZJX1N550ACFA5A' # your Foursquare ID
CLIENT_SECRET = 'OGUGTHQP15HJCC31I3QXXEK2FCAESKJV5HMCINXK023514EQ' # your Foursquare Secret
VERSION = '20191231' # Foursquare API version
radius = 500
LIMIT = 100
print(">>> Reading File Into Data Frame : " + QueryLocations)
df = pd.read_csv(QueryLocations)
FourSquare = []
for index, row in df.iterrows():
if "#" in str(df['Location ID'][index]):
continue
Location = df[' Location Name'][index]
LatValue = df[' Latitude'][index]
LonValue = df[' Longitude'][index]
print(' $ Latitude and longitude values of {} are {}, {}.'.format(Location, LatValue, LonValue))
url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
CLIENT_ID,
CLIENT_SECRET,
VERSION,
LatValue,
LonValue,
radius,
LIMIT)
results = requests.get(url).json()
if "There aren't a lot of results near you" in str(results["response"]):
FourSquare.append("No")
continue
else:
FourSquare.append("Yes")
SF.jsonwrite(venues + "\\" + str(Location) + ".json", results)
FSQ_DF = ProcessFourSquareData(results)
FSQ_DF.to_csv(venues + "\\" + str(Location) + "_FSQ.csv")
df["FourSquare"] = FourSquare
def Infer_New(self, utterance, samples=1000):
"""
Build a hypothesis space and compute the posterior
"""
# Each speaker object needs a visual world, and a bias object.
# Build space of possible visual worlds
#sys.stdout.write("Building visual world space.\n")
[VW_HypothesisSpace, VW_Priors] = SF.BuildVWHypSpace(
self.VisualWorld, self.CommonGroundPrior, self.Filter)
# Now build hypothesis spaces over speaker biases.
#sys.stdout.write("Building speaker bias space.\n")
[SpeakerBias_HypothesisSpace, SpeakerBias_Priors] = SF.BuildBiasHypSpace(
self.BiasPriors)
# Next we need to combine these two to build a massive hypothesis
# space.
#sys.stdout.write("Building full hypothesis space.\n")
BiasNames = [x.name for x in self.BiasPriors]
for SB_index in range(len(SpeakerBias_Priors)):
# Pack things into a set of Bias objects
#sys.stdout.write("\tbuilding speaker biases.\n")
CurrentBias = Bias.Bias(
BiasNames, SpeakerBias_HypothesisSpace[SB_index])
for VW_index in range(len(VW_Priors)):
# Now we also need to build the space of possible referents.
# First build the speaker object.
#sys.stdout.write("\tbuilding speaker object.\n")
TestSpeaker = Speaker.Speaker(
VW_HypothesisSpace[VW_index], CurrentBias, self.SpeakerRationalityNoise)
# Now iterate over the space of referents and get
# the probability of producing the utterance.
#sys.stdout.write("\t\titerating over referents.\n")
for testobject in TestSpeaker.VisualWorld.objects:
#sys.stdout.write(".")
p = TestSpeaker.GetUtteranceProbability(
utterance, testobject, samples)
self.HypothesisSpace.InsertHypothesis(Hypothesis.Hypothesis(VW_HypothesisSpace[
VW_index], CurrentBias, testobject, utterance, VW_Priors[VW_index], SpeakerBias_Priors[SB_index], p))
def QueryLocations():
print(">>> Action 1 : QueryLocations")
api = "getPlanningareaNames?"
myurl = url + api + OneMapAuth
SF.log_to_report(QueryLocations, "w", "Location ID, Location Name, Latitude, Longitude")
for value in SF.Query(myurl):
my_dict = dict(value)
location = str(my_dict['pln_area_n'])
lat, lon = SF.QueryLocationData(location, "Singapore")
SF.log_to_report(QueryLocations, "a", str(my_dict['id']) + "," + location + "," + str(lat) + "," + str(lon))
PlanningArea.append(my_dict['pln_area_n'])
["large", "rectangle"], ["large", "square"]]
Utterance_Unr = [["large", "square"], ["small", "circle"],
["small", "triangle"], ["large", "rectangle"], ["", "star"]]
Utterance_Rel = [["", "square"], ["", "circle"], ["small", "triangle"],
["", "rectangle"], ["", "star"]]
Utterance_List = Utterance_Unr
Verbose = False
BetaParameters = [0.32, 1.12]
SpeakerRationalityNoise = 0.057
MyFilter = Filter.Filter(3, 3) # Common ground must have three objects.
BiasPriors = [SF.BuildBeta(BetaParameters[0], BetaParameters[1], "size")]
RawUtterance = Utterance_List[0]
if RawUtterance[0] != "":
FirstUtterance = UT.Utterance(RawUtterance[1],
[OF.ObjectFeature(RawUtterance[0], "size")])
else:
FirstUtterance = UT.Utterance(RawUtterance[1])
TL = PO.PhysicalObject(TL_Keys[0][1],
[OF.ObjectFeature(TL_Keys[0][0], "size")], "TL")
TR = PO.PhysicalObject(TR_Keys[0][1],
[OF.ObjectFeature(TR_Keys[0][0], "size")], "TR")
BL = PO.PhysicalObject(BL_Keys[0][1],
[OF.ObjectFeature(BL_Keys[0][0], "size")], "BL")
BR = PO.PhysicalObject(BR_Keys[0][1],
[OF.ObjectFeature(BR_Keys[0][0], "size")], "BR")
CGPrior = BF.Belief(4, [TL.Id, TR.Id, BL.Id, BR.Id],
def forward_training(self, Y, T, p):
print(
"Running Joint IHT, projecting onto support cardinality k = {0:1.0f}"
.format(self.k))
# Define IHT parameters
alpha = 0.2 #0.005
max_IHT_iters = 30
# Extract data and weight filter dimensions
y_dims = list(Y.data.size())
filter_dims = list(self.D_trans.weight.data.size())
# Order the data so that member fom the same set are processed at the same time
temp = {
"0": [],
"1": [],
"2": [],
"3": [],
"4": [],
"5": [],
"6": [],
"7": [],
"8": [],
"9": []
}
Z = {}
X = {}
Z_list = []
X_list = []
error = {}
total_l2_error = 0
error_percent = {}
joint_supp = {}
numb_runs = {}
# Put data into into lists of tensors by label
for i in range(y_dims[0]):
temp[str(int(T[i].item()))].append(Y[i])
# Take list of tensors and form stacked tensor, process as go along
for key, tensor_list in temp.items():
if len(temp[key]) > 0:
Z[key] = torch.stack(temp[key], dim=0)
input_dims = list(Z[key].size())
active_filter_inds = sf.sample_filters(filter_dims[0], p,
self.k)
self.mask = sf.create_dropout_mask(
input_dims[0], filter_dims[0],
(input_dims[2] - filter_dims[2] + 1),
(input_dims[3] - filter_dims[3] + 1), active_filter_inds)
X[key], error_percent[key], error[key], numb_runs[
key], joint_supp[key] = self.IHT_Joint(
Z[key], max_IHT_iters, alpha)
total_l2_error = total_l2_error + error[key]
X_list.append(X[key])
Z_list.append(Z[key])
X_tensor = torch.cat(X_list, dim=0)
Z_tensor = torch.cat(Z_list, dim=0)
total_error_percent = total_l2_error * 100 / (np.sum(
(Y).data.cpu().numpy()**2))
av_number_runs = 0
for key, sc_iterations in numb_runs.items():
av_number_runs = av_number_runs + sc_iterations
av_number_runs = av_number_runs / len(numb_runs)
filters_selected = []
for key, js in joint_supp.items():
filters_selected = np.append(filters_selected, js)
print("Error breakdown by class label:")
for key, item in error_percent.items():
print("Class: {}".format(key) +
", number of iterations: {}".format(numb_runs[key]) +
", error percentage: {0:1.2f}%".format(item))
return X_tensor, Z_tensor, total_error_percent, av_number_runs, joint_supp, filters_selected, X
# self-defined module
import sys
#sys.path.append(r'C:\Users\chenho\Documents\EY\EY Private\Self-defined-Functions') # Add a directory into sys path
sys.path.append(
r'G:\Data Science\Self-defined Functions') # Add a directory into sys path
import SupportingFunctions as SF
import EDD as EDD
import woe_hc as WOE
import MachineLearningModels as MLM
#### data loading ####
dir_data = r"G:\Data Science\Kaggle Competition\Titanic Sinking\data"
dir_output = r"G:\Data Science\Kaggle Competition\Titanic Sinking\output"
df_train = SF.load_csv(dir_data + "\\train.csv")
df_test = SF.load_csv(dir_data + "\\test.csv")
df_edd_num, df_edd_cat = EDD.EDD(df_train,
ls_force_categorical=['Survived', 'Pclass'])
df = df_train.copy()
#### data exploration ####
df['Age'].max()
sns.distplot(df['Age'].fillna(99))
sns.distplot(df['Fare'])
#### Feature Engineering ####
df['Title'] = df.Name.str.extract(' ([A-Za-z]+)\.', expand=False)
df['Title'] = df['Title'].replace([
#####################################
# Two objects with one feature each #
#####################################
rose = PO.PhysicalObject("flower", [OF.ObjectFeature("red", "color")], "rose")
sunflower = PO.PhysicalObject("flower", [OF.ObjectFeature("yellow", "color")],
"sunflower")
World = VW.VisualWorld([rose, sunflower])
# prior over the visual world
MyCGPrior = BF.Belief(2, [rose.Id, sunflower.Id], [0.5, 0.5])
# prior over the speaker's production biases. In this case, only color matters
# Use the Beta build in supporting functions
BiasPriors = [SF.BuildBeta(5, 3, "color", 0.5)]
MyListener = CL.ComplexListener(World, MyCGPrior, BiasPriors)
Utterance_flower = UT.Utterance("flower") # Ambiguous
Utterance_yellowflower = UT.Utterance(
"flower", [OF.ObjectFeature("yellow", "color")]) # Ambiguous
MyListener.Infer(Utterance_flower)
res = MyListener.ComputePosterior()
#####################################
# Two objects with two features each #
#####################################
rose = PO.PhysicalObject(
ColorPriorParams = [0.43, 0.82]
SizePriorParams = [0.28, 1.18]
CategoryPriorParams = [0.29, 0.22]
# Build priors depending on which corner you're questioning
TLSpot = [0.5, 1, 1, 1]
TRSpot = [1, 0.5, 1, 1]
BLSpot = [1, 1, 0.5, 1]
BRSpot = [1, 1, 1, 0.5]
CGPrior = BF.Belief(4, [
COL_A_A.objects[0].Id, COL_A_A.objects[1].Id, COL_A_A.objects[2].Id,
COL_A_A.objects[3].Id
], TRSpot)
ProductionPriors = [
SF.BuildBeta(ColorPriorParams[0], ColorPriorParams[1], "color")
]
MyListener = ComplexListener(COL_A_A, CGPrior, ProductionPriors, MyFilter,
SpeakerRationalityNoise_Color)
MyListener.Infer(COL_A_A_Utterance)
Result = MyListener.ComputePosterior()
SF.PrintCSV(Result, "COL-A-1", True)
CGPrior = BF.Belief(4, [
COL_A_B.objects[0].Id, COL_A_B.objects[1].Id, COL_A_B.objects[2].Id,
COL_A_B.objects[3].Id
], BRSpot)
ProductionPriors = [
SF.BuildBeta(ColorPriorParams[0], ColorPriorParams[1], "color")
]
MyListener = ComplexListener(COL_A_B, CGPrior, ProductionPriors, MyFilter,
if "Mumbai" in name:
teamSF.append("MI")
if "Gujarat" in name:
teamSF.append("GL")
if "Bangalore" in name:
teamSF.append("RCB")
if "Kolkata" in name:
teamSF.append("KKR")
if "Delhi" in name:
teamSF.append("DD")
if "Rajasthan" in name:
teamSF.append("RR")
if "Kochi" in name:
teamSF.append("Kochi")
teamNos, reverseteamNos = sf.assign_number(teamSF)
print(teamNos)
venueNos, reversevenueNos = sf.assign_number(venue)
#print(venueNos)
tossNos, reversetossNos = sf.assign_number(toss)
#print(tossNos)
batsmanlist, batsmanidlist = sf.assign_number(batsman)
bowlerlist, bowleridlist = sf.assign_number(bowler)
playerlist, playeridlist = sf.assign_number(player)
with open('ipl/reverseteamCodes.json', 'w') as outfile:
json.dump(reverseteamNos, outfile)
with open('ipl/tossCodes.json', 'w') as outfile:
json.dump(tossNos, outfile)
with open('ipl/venueCodes.json', 'w') as outfile:
json.dump(venueNos, outfile)
with open('ipl/batsman.json', 'w') as outfile:
# print(train_set.train_data[6])
# print(train_set.train_labels[6].item())
# Intitilise Convolutional Sparse Coder CSC
CSC = SL_CSC_IHT_Joint(stride, dp_channels, atom_r, atom_c, numb_atom, k,
alpha).to(device)
# Define optimisation parameters
CSC_parameters = [{'params': CSC.D.parameters()}]
# Define training settings/ options
cost_function = nn.MSELoss(size_average=True)
# # Train Convolutional Sparse Coder
CSC = sf.train_SL_CSC(CSC, train_loader, test_loader, num_epochs, T_DIC,
cost_function, CSC_parameters, learning_rate, momentum,
weight_decay, batch_size, p, model_filename)
print("Training seqeunce finished")
filter_dims = list(np.shape(CSC.D_trans.weight.data.cpu().numpy()))
if using_azure == False:
# Get CSC ready to process a few inputs
CSC.batch_size = 100
CSC.mask = torch.ones(CSC.batch_size, filter_dims[0], 1, 1)
test_Y = Variable(
torch.unsqueeze(test_set.test_data[:CSC.batch_size], dim=1)
).type(
torch.FloatTensor
) / 255. # shape from (2000, 28, 28) to (2000, 1, 28, 28), value in range(0,1)
# Calculate the latent representation
test_X, SC_error_percent, numb_SC_iterations, filters_selected = CSC.forward(
def save_epub(self):
sf.pack_epub(self.filename)