def plot_genetic_algorithm_time():
timer = Timer()
experiments = []
time = []
total_time = 0
# Build Neural Network
genetic_algorithm = GeneticFixedTopology(100, 1000)
# 20 runs of 1000 generations each
for i in range(20):
experiments.append(i)
timer.start()
genetic_algorithm.run()
this_time = timer.stop()
time.append(this_time)
genetic_algorithm = GeneticFixedTopology(100, 1000)
total_time += this_time
mean_time = total_time / len(experiments)
# Plot
plt.figure()
plt.title("Time Taken in 1000 Generations", fontsize=20)
plt.xlabel('experimento')
plt.ylabel('tiempo (segundos)')
plt.scatter(experiments, time, color='blue')
plt.axhline(y=mean_time, color='r', linestyle='-')
plt.show()
def __init__(self, args, num_input, mode='both', timer: Timer = None):
self.args = args
self.timer = Timer() if timer is None else timer
if args.all_trees and False:
self.embedding_size_to_mlp = args.embedding_size # * args.num_trees_for_embedd
else:
self.embedding_size_to_mlp = args.embedding_size * args.num_trees_for_embedding
if mode is 'raw_only':
num_features = num_input
elif mode is 'emb_only':
num_features = self.embedding_size_to_mlp
# num_features = args.embedding_size
elif mode is 'both':
num_features = self.embedding_size_to_mlp + num_input
# num_features = args.embedding_size + num_input
else:
raise Exception(
"unidentified mode. possible={'raw_only', 'emb_only', 'both'}")
if args.load_mlp:
model_name = "{:s}_{:d}_{:d}.chkpt".format(
self.args.mlp_model_name, self.args.max_depth,
self.args.num_trees_for_embedding)
self.model = torch.load(model_name)['model']
else:
self.model = MLP(args, num_features)
self.mode = mode
self.loss = torch.nn.BCEWithLogitsLoss(reduction='mean')
total_params = sum(x.data.nelement() for x in self.model.parameters())
print("Model total number of parameters: {}".format(total_params))
class XGBTreeParser:
def __init__(self, booster, num_input, timer: Timer = None):
self.timer = Timer() if timer is None else timer
dump = booster.get_dump(with_stats=True)
self.trees = [DecisionTree(tree, num_input) for tree in dump]
self.leaf_to_index = [tree.leaf_to_index for tree in self.trees]
self.num_nodes = [
len(leaf_to_index) for leaf_to_index in self.leaf_to_index
]
self.max_length = max(self.num_nodes)
self.num_trees = len(self.trees)
self.timer.toc("init done. Max length = " + str(self.max_length))
def _parse_predict_leaf(self, pred_leaves):
def func(leaf_index, leaf):
return self.leaf_to_index[leaf_index][leaf]
a = np.vectorize(func)
indexes = np.arange(0, self.num_trees).reshape(
1, -1) # .repeat(len(pred_leaves), 0)
return a(indexes, pred_leaves)
# Input pred_leaves. shape [num_sample, num trees]
def get_one_hot_version(self, pred_leaves):
transpose = np.transpose(self._parse_predict_leaf(pred_leaves))
return np.concatenate([
np.eye(self.num_nodes[i])[tree] for i, tree in enumerate(transpose)
],
axis=1)
def plot_time_vs_epochs(train_data, test_data):
timer = Timer()
number_epochs = []
time = []
total_time = 0
# Build Neural Network
neural_network = build_network()
# 200 runs of 100 epochs each
for i in range(100):
number_epochs.append(i)
timer.start()
for j in range(1000):
for data in test_data:
neural_network.feed(data)
this_time = timer.stop()
time.append(this_time)
total_time += this_time
mean_time = total_time / len(number_epochs)
# Plot
plt.figure()
plt.title("Time Taken in 1000 Epochs", fontsize=20)
plt.xlabel('Number of Experiment')
plt.ylabel('Time (seconds)')
plt.scatter(number_epochs, time, color='blue')
plt.axhline(y=mean_time, color='r', linestyle='-')
plt.show()
def __init__(self, booster, num_input, timer: Timer = None):
self.timer = Timer() if timer is None else timer
dump = booster.get_dump(with_stats=True)
self.trees = [DecisionTree(tree, num_input) for tree in dump]
self.leaf_to_index = [tree.leaf_to_index for tree in self.trees]
self.num_nodes = [
len(leaf_to_index) for leaf_to_index in self.leaf_to_index
]
self.max_length = max(self.num_nodes)
self.num_trees = len(self.trees)
self.timer.toc("init done. Max length = " + str(self.max_length))
def __correlateBlock(self,w):
N = len(w[0,:]);
n = len(w[:,0]);
wout = zeros((n,N));
T = Timer("Convolving signal with Block..",0,N)
for j in range(0,N):
block = self.__block(self._t,self._t[j],self._pa);
for i in range(0,n):
wout[i,j] = sum(block*w[i,:]);
T.looptime(j)
return wout;
def main1():
instruction1 = BasicInstruction()
instruction3 = Priority_Instruction()
logger = Logger("../resource/log.txt")
program = Program('a')
program.addInstruction(instruction1)
program.addInstruction(instruction1)
program.addInstruction(instruction1)
program.addInstruction(instruction1)
program.addInstruction(instruction1)
program.addInstruction(instruction1)
programb = Program('b')
programb.addInstruction(instruction3)
programc = Program('c')
programc.addInstruction(instruction1)
programc.addInstruction(instruction1)
programc.addInstruction(instruction1)
timer = Timer(2)
memory = Memory()
memory.buildMemory(9)
frame1 = Frame(memory,0,9)
mmu = MMU()
mmu.addEmptyFrame(frame1)
cpu = CPU(None,mmu,None,timer,logger)
scheduler = Scheduler(PFIFO())
ioqueue = IOQueue(scheduler,logger)
disk = Disk(None)
kernel = Kernel(cpu,ioqueue,scheduler,mmu,disk,logger)
disk.setKernel(kernel)
disk.save(program)
disk.save(programb)
disk.save(programc)
cpu.setKernel(kernel)
kernel.executeProgram('a')
class Health:
def __init__(self, health, cooldown_seconds):
from src.Timer import Timer
self.health = health
self.cooldown_seconds = cooldown_seconds
self.time_elapsed_since_hit = self.cooldown_seconds
self.timer = Timer()
def deal_damage(self, damage):
t = self.timer.get_time()
if t > self.cooldown_seconds * 1000:
self.health = self.health - damage
if self.health < 0:
self.health = 0
self.time_elapsed_since_hit = 0
self.timer.reset()
def main0():
instruction1 = BasicInstruction()
instruction2 = IO_Instruction()
instruction3 = Priority_Instruction()
program = Program('a')
program.addInstruction(instruction1)
program.addInstruction(instruction1)
program.addInstruction(instruction1)
programb = Program('b')
programb.addInstruction(instruction3)
programc = Program('c')
programc.addInstruction(instruction1)
programc.addInstruction(instruction1)
programc.addInstruction(instruction1)
timer = Timer(2)
memory = Memory()
memory.buildMemory(9)
frame1 = Frame(memory,0,9)
mmu = MMU()
mmu.addEmptyFrame(frame1)
cpu = CPU(None,mmu,None,timer)
scheduler = Scheduler(PFIFO())
ioqueue = IOQueue(scheduler)
disk = Disk(None)
kernel = Kernel(cpu,ioqueue,scheduler,mmu,disk)
disk.setKernel(kernel)
cpu.setKernel(kernel)
kernel.saveProgram(program)
kernel.saveProgram(programb)
kernel.saveProgram(programc)
kernel.start()
cpu.start()
ioqueue.start()
def __init__(self, player):
from src.Timer import Timer
from src.Sprite import Sprite
self.valid_blink_points = []
self.dot_spr = Sprite(ImageEnum.BLINK_DOT)
self.dot_spr.bounds = (0, 0, 4, 4)
self.can_blink = False
self.player = player
self.blink_frames = 3
self.is_blinking = False
self.frame_displacement = (None, None)
self.frames_passed = None
self.timer = Timer()
self.cooldown_ms = 2500
self.cooldown_passed = False
self.state = BlinkState.CAN_BLINK
def DEM(M,x0,u,y,p,Pw,Pz,alpha_x,alpha_th,embed=0,gembed=1,maxit=1000,numerical=True,mfts=False,tol=1e-6,conit=100):
# Allocate memory
J = zeros(( 1, maxit));
th = zeros((M.nth, maxit));
th[:,0] = M.th;
# Set-up loop
i = 0;
j = 0;
T = Timer('Dynamic Expectation Maximization',i,maxit-1,maxit-1);
# Run loop
while i < maxit-1 and j < conit:
# Update cost, covariance and parameter estimates
if numerical:
J[:,i],th[:,i+1] = NDEMstep(M,x0,u,y,p,Pw,Pz,alpha_x,alpha_th,embed,mfts)[0:2];
else:
J[:,i],th[:,i+1] = ADEMstep(M,x0,u,y,p,Pw,Pz,alpha_x,alpha_th,embed,gembed,mfts)[0:2]
M.th = th[:,i+1];
# Stopping criterion: J < <tol> in last <conit> iterations
if j == 0:
if abs(J[:,i]-J[:,i-1]) < tol:
j = 1;
else:
if abs(J[:,i]-J[:,i-1]) < tol:
j +=1;
else:
j = 0;
i +=1;
T.looptime(i);
# Get final hidden state and output estimations
xp,xu,yp,yu = FEF(M,x0,u,y,p,alpha_x,embed,mfts);
# If converged before maxit, then fill remainer of matrices with last value
for j in range(i-1,maxit):
J[:,j] = J[:,i-1];
th[:,j] = th[:,i];
dat = pedata('DEM',M,J,[],[],[],th,xp,xu,yp,yu);
return dat;
def __correlateGaussian(self,w):
N = len(w[0,:]);
n = len(w[:,0]);
wout = zeros((n,N));
tol = 1e-6;
# find for which coordinate the gaussian < tol
gaus = self.__gaussian(self._t,0,self._pa);
k=0
while gaus[k] > tol:
k+=1;
gaus = zeros(2*k);
T = Timer("Convolving signal with Gaussian..",0,N)
for j in range(0,N):
kmin = max([0,j-k]);
kmax = min([N-1,j+k]);
if j <= k or j >= N-k:
gaus = self.__gaussian(self._t[kmin:kmax],self._t[j],self._pa);
else:
gaus[:] = self.__gaussian(self._t[kmin:kmax],self._t[j],self._pa);
for i in range(0,n):
wout[i,j] = sum(gaus*w[i,kmin:kmax]);
T.looptime(j)
return wout;
def __init__(self):
print("Bienvenue à BruteForceLogin.py!")
args_dictionary = self.get_args()
message_administrator = MessageAdministrator()
timer = Timer()
form_name_finder = FormNameFinder()
dictionary_reader = DictionaryReader(args_dictionary["dict"])
browser_service = BrowserService(args_dictionary["url"])
brute_force_login = BruteForceLogin(args_dictionary,
message_administrator, timer,
form_name_finder,
dictionary_reader, browser_service)
brute_force_login.execute()
def main():
instruction1 = BasicInstruction()
instruction2 = IO_Instruction()
instruction3 = Priority_Instruction()
program = Program('a')
program.addInstruction(instruction1)
program.addInstruction(instruction1)
program.addInstruction(instruction1)
program.addInstruction(instruction1)
program.addInstruction(instruction1)
programd = Program('d')
programd.addInstruction(instruction1)
programd.addInstruction(instruction1)
programd.addInstruction(instruction1)
programd.addInstruction(instruction1)
programd.addInstruction(instruction1)
programb = Program('b')
programb.addInstruction(instruction3)
programc = Program('c')
programc.addInstruction(instruction1)
programc.addInstruction(instruction1)
programc.addInstruction(instruction1)
timer = Timer(2)
memory = Memory()
memory.buildMemory(9)
frame1 = Frame(memory,0,9)
logger = Logger("/home/matlock/Escritorio/Sistemas Operativos/OSProyect/resource/log.txt")
mmu = MMU(logger)
mmu.addEmptyFrame(frame1)
cpu = CPU(None,mmu,None,timer,logger)
scheduler = Scheduler(PFIFO())
ioqueue = IOQueue(scheduler,logger)
disk = Disk(None)
kernel = Kernel(cpu,ioqueue,scheduler,mmu,disk,logger)
disk.setKernel(kernel)
cpu.setKernel(kernel)
x = []
x.append(program)
x.append(programb)
x.append(programc)
x.append(programd)
kernel.saveOnDisk(x)
kernel.executeProgram('a')
def __init__(self,
args,
num_nodes,
svd_name='xgb-svd',
timer: Timer = None):
self.timer = Timer() if timer is None else timer
self.args = args
self.num_nodes = num_nodes
self.max_depth = np.ceil(np.log(max(num_nodes))).astype(int)
self.svd_name = svd_name
self.cumsum = np.cumsum([0] + self.num_nodes)
self.embedding_size = args.embedding_size
self.embeddings = None
if args.load:
model_name = "{:s}_{:d}_{:d}".format(svd_name, self.max_depth,
len(num_nodes))
with open(model_name, "rb") as f:
self.svds = pickle.load(f)
self._get_weights()
else:
self.svds = TruncatedSVD(n_components=self.embedding_size)
class IEEESplitter:
def __init__(self, args, load_raw=False):
self.train_parsed_file = '../data/data/ieee/train.csv'
self.valid_parsed_file = '../data/data/ieee/valid.csv'
self.test_parsed_file = '../data/data/ieee/test.csv'
self.args = args
self.id_cat_col = ["id_" + str(i) for i in range(12, 39)] + [
"DeviceType", "DeviceInfo", "id_33_1", "id_31_1", "id_30_1"
]
self.trans_cat_col = ["M" + str(i) for i in range(1, 10)] + ["card" + str(i) for i in range(1, 7)] + \
["ProductCD", "addr1", "addr2", "P_emaildomain", "R_emaildomain"]
# self.try_list = ["id_" + str(i) for i in [13, 17]]
self.try_list = ["id_" + str(i) for i in []]
self.id_onehot_encoder = OneHot(0.01)
self.trans_onehot_encoder = OneHot(0.01)
self.id_scaler = Scaler()
self.trans_scaler = Scaler()
self.global_name = []
self.global_df = []
self.emails = {
'gmail': 'google',
'att.net': 'att',
'twc.com': 'spectrum',
'scranton.edu': 'other',
'optonline.net': 'other',
'hotmail.co.uk': 'microsoft',
'comcast.net': 'other',
'yahoo.com.mx': 'yahoo',
'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo',
'charter.net': 'spectrum',
'live.com': 'microsoft',
'aim.com': 'aol',
'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink',
'gmail.com': 'google',
'me.com': 'apple',
'earthlink.net': 'other',
'gmx.de': 'other',
'web.de': 'other',
'cfl.rr.com': 'other',
'hotmail.com': 'microsoft',
'protonmail.com': 'other',
'hotmail.fr': 'microsoft',
'windstream.net': 'other',
'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo',
'servicios-ta.com': 'other',
'netzero.net': 'other',
'suddenlink.net': 'other',
'roadrunner.com': 'other',
'sc.rr.com': 'other',
'live.fr': 'microsoft',
'verizon.net': 'yahoo',
'msn.com': 'microsoft',
'q.com': 'centurylink',
'prodigy.net.mx': 'att',
'frontier.com': 'yahoo',
'anonymous.com': 'other',
'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att',
'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo',
'outlook.com': 'microsoft',
'mail.com': 'other',
'bellsouth.net': 'other',
'embarqmail.com': 'centurylink',
'cableone.net': 'other',
'hotmail.es': 'microsoft',
'mac.com': 'apple',
'yahoo.co.uk': 'yahoo',
'netzero.com': 'other',
'yahoo.com': 'yahoo',
'live.com.mx': 'microsoft',
'ptd.net': 'other',
'cox.net': 'other',
'aol.com': 'aol',
'juno.com': 'other',
'icloud.com': 'apple'
}
self.drop_col = [
'TransactionDT', 'V300', 'V309', 'V111', 'C3', 'V124', 'V106',
'V125', 'V315', 'V134', 'V102', 'V123', 'V316', 'V113', 'V136',
'V305', 'V110', 'V299', 'V289', 'V286', 'V318', 'V103', 'V304',
'V116', 'V298', 'V284', 'V293', 'V137', 'V295', 'V301', 'V104',
'V311', 'V115', 'V109', 'V119', 'V321', 'V114', 'V133', 'V122',
'V319', 'V105', 'V112', 'V118', 'V117', 'V121', 'V108', 'V135',
'V320', 'V303', 'V297', 'V120'
]
self.us_emails = ['gmail', 'net', 'edu']
self.timer = Timer()
if load_raw:
df_trans = pd.read_csv("../data/data/ieee/train_transaction.csv"
).set_index("TransactionID")
df_id_raw = pd.read_csv("../data/data/ieee/train_identity.csv"
).set_index("TransactionID")
dt_trans = pd.read_csv("../data/data/ieee/test_transaction.csv"
).set_index("TransactionID")
dt_id = pd.read_csv("../data/data/ieee/test_identity.csv"
).set_index("TransactionID")
self.timer.toc("read done")
self.test_id = list(dt_trans.index)
df_trans, dv_trans = train_test_split(
df_trans,
random_state=args.random_state,
train_size=0.8,
test_size=0.2)
df_id = df_id_raw.loc[
df_trans.index.intersection(df_id_raw.index), :]
dv_id = df_id_raw.loc[
dv_trans.index.intersection(df_id_raw.index), :]
self.timer.toc("split done")
self.feature_engineering_id(df_id)
self.feature_engineering_id(dv_id)
self.feature_engineering_id(dt_id)
self.timer.toc("process id done")
self.feature_engineering_trans(df_trans)
self.feature_engineering_trans(dv_trans)
self.feature_engineering_trans(dt_trans)
self.timer.toc("process trans done")
self.global_count(df_id, df_trans)
self.timer.toc("global_count done")
df_trans, ytrain = self.split_x_y(df_trans)
dv_trans, yvalid = self.split_x_y(dv_trans)
dt_trans, ytest = self.split_x_y(dt_trans)
df_global = self.get_derived(df_id)
self.timer.toc("get train derived done")
dv_global = self.get_derived(dv_id)
self.timer.toc("get valid derived done")
dt_global = self.get_derived(dt_id)
self.timer.toc("get test derived done")
self.init_onehot(df_id, df_trans)
self.init_scaler(df_id.drop(self.id_cat_col, axis=1),
df_trans.drop(self.trans_cat_col, axis=1))
Xtrain = self.transform(df_id, df_trans, df_global)
self.timer.toc("transform train done")
Xvalid = self.transform(dv_id, dv_trans, dv_global)
self.timer.toc("transform valid done")
Xtest = self.transform(dt_id, dt_trans, dt_global)
self.timer.toc("transform test done")
pd.concat([Xtrain, ytrain], axis=1).to_csv(self.train_parsed_file,
float_format='%.8f',
index=True)
self.timer.toc("write train done")
pd.concat([Xvalid, yvalid], axis=1).to_csv(self.valid_parsed_file,
float_format='%.8f',
index=True)
self.timer.toc("write valid done")
Xtest.to_csv(self.test_parsed_file,
float_format='%.8f',
index=True)
self.timer.toc("write test done")
raise Exception("STOP HERE!")
else:
self.train = self.load(self.train_parsed_file)
self.timer.toc("load train done")
self.valid = self.load(self.valid_parsed_file)
self.timer.toc("load valid done")
self.test = self.load(self.test_parsed_file, test=True)
self.timer.toc("load test done")
self.num_input = self.train[0].shape[1]
# sanity check
print(np.unique(self.train[1], return_counts=True))
print(np.unique(self.valid[1], return_counts=True))
# self.train = Xtrain, ytrain.values
# self.valid = Xvalid, yvalid.values
# self.test = Xtest, ytest.values
# self.num_input = Xtrain.shape[1]
def load(self, file, test=False):
df = pd.read_csv(file)
if not test:
X = df.drop(['TransactionID', 'isFraud'], axis=1)
y = df['isFraud']
else:
X = df.drop('TransactionID', axis=1)
y = pd.Series(np.zeros(len(df)))
y.iloc[:10] = 1
self.test_id = df['TransactionID']
return X.values, y.values
def transform(self, df_id, df_trans, df_global):
id_num, id_cat = self.id_onehot_encoder.transform(df_id)
id_num = self.id_scaler.transform(id_num)
id = pd.concat([id_num, id_cat], axis=1)
# id = pd.concat([df_global, id_num, id_cat], axis=1)
trans_num, trans_cat = self.trans_onehot_encoder.transform(df_trans)
trans_num = self.trans_scaler.transform(trans_num)
trans = pd.concat([trans_num, trans_cat], axis=1)
return trans.merge(id, how='left', left_index=True, right_index=True)
def init_onehot(self, df_id, df_trans):
self.id_onehot_encoder.fit(df_id, self.id_cat_col)
self.trans_onehot_encoder.fit(df_trans, self.trans_cat_col)
def init_scaler(self, df_id, df_trans):
self.id_scaler.fit(df_id)
self.trans_scaler.fit(df_trans)
def get_derived(self, df_id):
return
temp = df_id[self.try_list]
df_list = [] # [df_id[['TransactionID']]]
for i in range(len(self.global_name)):
li = self.global_name[i]
column_name = self.global_df[i].columns[0]
merged = temp.merge(self.global_df[i].reset_index(),
how='left',
on=li)
merged.index = temp.index
df_list.append(merged[[column_name]])
return pd.concat(df_list, axis=1)
@staticmethod
def split_x_y(df):
if 'isFraud' in df.columns:
return df.drop('isFraud', axis=1), df['isFraud']
else:
y = pd.Series(np.zeros(len(df)), index=df.index)
y.iloc[:10] = 1
return df, y
@staticmethod
def parse_device_info(y):
if type(y) is not str:
return np.nan
x = y.lower()
if x.startswith("rv"):
return "rv"
elif x.startswith("sm-"):
return "sm"
elif x.startswith("trident"):
return "trident"
elif x.startswith("moto"):
return "moto"
elif x.startswith("lg"):
return "lg"
elif x.startswith("samsung"):
return "sm"
elif x.startswith("windows"):
return "windows"
elif x.startswith("ios"):
return "ios"
elif x.startswith("macos"):
return "macos"
else:
return np.nan
@staticmethod
def feature_engineering_id(dg):
dg["id_33_1"] = dg["id_33"].apply(lambda x: int(x.split("x")[0])
if type(x) is str else np.nan)
dg["id_33"] = dg["id_33"].apply(lambda x: int(x.split("x")[1])
if type(x) is str else np.nan)
dg["id_31_1"] = dg["id_31"].apply(lambda x: x.split(" ")[0]
if type(x) is str else np.nan)
dg["id_31"] = dg["id_31"].apply(lambda x: x.split(" ")[1] if type(
x) is str and len(x.split(" ")) > 1 else np.nan)
dg["id_30_1"] = dg["id_30"].apply(lambda x: x.split(" ")[0]
if type(x) is str else np.nan)
dg["id_30"] = dg["id_30"].apply(lambda x: x.split(" ")[1] if type(
x) is str and len(x.split(" ")) > 1 else np.nan)
dg["DeviceInfo"] = dg["DeviceInfo"].apply(
IEEESplitter.parse_device_info)
dg["nulls_id"] = dg.isna().sum(axis=1)
def feature_engineering_trans(self, dg):
def get_suffix(x):
suffix = str(x).split(".")[-1]
return x if str(x) not in self.us_emails else 'us'
dg["nulls_trans"] = dg.isna().sum(axis=1)
for col in ["P_emaildomain", "R_emaildomain"]:
dg[col + "_bin"] = dg[col].map(self.emails)
dg[col + "_suffix"] = dg[col].apply(get_suffix)
dg.drop(self.drop_col, axis=1, inplace=True)
def global_count(self, df_id, df_trans):
try_lists = IEEESplitter.powerset(self.try_list)
df_temp = df_id.merge(df_trans[['isFraud']],
how='left',
left_index=True,
right_index=True)
df_temp = df_temp[['isFraud'] + self.try_list]
for selected in try_lists:
selected = list(selected)
groupby = df_temp[['isFraud'] + selected].groupby(selected).mean()
groupby.columns = ["global_" + "_".join(selected)]
self.global_name.append(selected)
self.global_df.append(groupby)
@staticmethod
def powerset(li):
l = []
from itertools import combinations
for i in range(1, len(li) + 1):
l = l + list(combinations(li, i))
return l
def export(self, inference, out_csv):
df = pd.DataFrame()
df['TransactionID'] = self.test_id
df['isFraud'] = inference
df.to_csv("../data/data/ieee/" + out_csv, index=False)
def EM(M,x0,u,y,Q,R,alpha=1,maxit=1000,numerical=True,tol=1e-6,conit=100,gembed=1):
"""
Expectation Maximization
INPUTS
M Data-structure of Model class
x0 Initial hidden state - 1-dimensional array_like (list works)
u Input sequence nu x N numpy_array
y Output sequence ny x N numpy_array
Q State-noise covariance - nx x nx numpy_array
R Output-noise covariance - ny x ny numpy_array
alpha Updating gain - scalar float, int - (opt. def=1)
maxit Max. number of iterations - scalar int - (opt. def=1000)
numerical Use numerical gradients - boolean - (opt. def=True)
tol Error tolerance for stopping cond. - float - (opt. def=True)
conit Number of converence iterations. - fint - (opt. def=100)
gembed Gradient-embedding order (if algebraical gradient )
- scalar int - (opt. def=1)
OUTPUTS
dat Data-structure of pedata class containing
"""
# Allocate memory
J = zeros(( 1, maxit));
K = zeros(( M.nx, M.ny, maxit));
P = zeros(( M.nx, M.nx, maxit));
S = zeros(( M.ny, M.ny, maxit));
th = zeros((M.nth, maxit));
th[:,0] = M.th;
# Set-up loop
i = 0;
j = 0;
T = Timer('Expectation Maximization',i,maxit-1,maxit-1);
# Run loop
while i < maxit-1 and j < conit:
# Update cost, covariance and parameter estimates
if numerical:
J[:,i],K[:,:,i],P[:,:,i],S[:,:,i],th[:,i+1] \
= NEMstep(M,x0,u,y,Q,R,alpha)[0:5];
else:
J[:,i],K[:,:,i],P[:,:,i],S[:,:,i],th[:,i+1] \
= AEMstep(M,x0,u,y,Q,R,alpha,gembed)[0:5];
M.th = th[:,i+1];
# Stopping criterion: J < <tol> in last <conit> iterations
if j == 0:
if abs(J[:,i]-J[:,i-1]) < tol:
j = 1;
else:
if abs(J[:,i]-J[:,i-1]) < tol:
j +=1;
else:
j = 0;
i +=1;
T.looptime(i);
# Get final hidden state and output estimations
xp,xu,yp,yu = M.filt(K[:,:,i],x0,u,y);
# If converged before maxit, then fill remainer of matrices with last value
for j in range(i-1,maxit):
J[:,j] = J[:,i-1]
K[:,:,j] = K[:,:,i-1]
P[:,:,j] = P[:,:,i-1]
S[:,:,j] = S[:,:,i-1]
th[:,j] = th[:,i]
dat = pedata('EM',M,J,K,P,S,th,xp,xu,yp,yu);
return dat;
def __init__(self, embedding_size, timer: Timer = None):
self.timer = Timer() if timer is None else timer
parser.add_argument('--all_trees', type=bool, default=True)
parser.add_argument('--mlp_num_epoch', type=int, default=1000)
parser.add_argument('--mlp_lr', type=float, default=3e-6, help='learning rate')
parser.add_argument('--mlp_batch_size', type=int, default=64)
parser.add_argument('--mlp_dropout', type=float, default=0.5)
parser.add_argument('--mlp_weight_decay', type=float, default=0)
parser.add_argument('--n_latent', type=int, default=100)
#################### Finalizing args
args = parser.parse_args()
if args.random_state is not None:
torch.manual_seed(args.random_state)
print("args = ", args)
timer = Timer()
def main():
# split
splitter = IEEESplitter(args)
# splitter = SantanderSplitter("../data/santander/train.csv", args)
# train xgb
trainer = XGBTrainer(splitter, args, timer)
pred = trainer.predict(splitter.test[0])
splitter.export(pred, "xgb_pred.csv")
# train embedding
# embs = train_xgb_emb(trainer)
class MLPTrainer:
def __init__(self, args, num_input, mode='both', timer: Timer = None):
self.args = args
self.timer = Timer() if timer is None else timer
if args.all_trees and False:
self.embedding_size_to_mlp = args.embedding_size # * args.num_trees_for_embedd
else:
self.embedding_size_to_mlp = args.embedding_size * args.num_trees_for_embedding
if mode is 'raw_only':
num_features = num_input
elif mode is 'emb_only':
num_features = self.embedding_size_to_mlp
# num_features = args.embedding_size
elif mode is 'both':
num_features = self.embedding_size_to_mlp + num_input
# num_features = args.embedding_size + num_input
else:
raise Exception(
"unidentified mode. possible={'raw_only', 'emb_only', 'both'}")
if args.load_mlp:
model_name = "{:s}_{:d}_{:d}.chkpt".format(
self.args.mlp_model_name, self.args.max_depth,
self.args.num_trees_for_embedding)
self.model = torch.load(model_name)['model']
else:
self.model = MLP(args, num_features)
self.mode = mode
self.loss = torch.nn.BCEWithLogitsLoss(reduction='mean')
total_params = sum(x.data.nelement() for x in self.model.parameters())
print("Model total number of parameters: {}".format(total_params))
def trainIters(self, train, valid):
early_stopper = EarlyStopper(3, 'moving', reverse=True)
max_auc = 0
# ADAM opts
opt = optim.Adam(self.model.parameters(),
lr=self.args.mlp_lr,
weight_decay=self.args.mlp_weight_decay)
################ Training epoch
self.model.cuda()
for epoch in range(1, self.args.mlp_num_epoch + 1):
self.model.train()
train_losses = self.train_model(opt, train)
valid_losses, results, ground_truths = self.valid_model(valid)
valid_auc = self.evaluate(results,
ground_truths,
print_result=False)
self.timer.toc(
"epoch {:4d} - train loss: {:10.6f} valid loss: {:10.6f} valid auc: {:10.6f}"
.format(epoch, np.mean(train_losses), np.mean(valid_losses),
valid_auc))
checkpoint = {'model': self.model, 'args': self.args}
model_name = "{:s}_{:d}_{:d}.chkpt".format(
self.args.mlp_model_name, self.args.max_depth,
self.args.num_trees_for_embedding)
if valid_auc > max_auc:
max_auc = valid_auc
torch.save(checkpoint, model_name)
if early_stopper.record(valid_auc):
self.model = torch.load(model_name)['model']
return
def valid_model(self, valid):
valid_losses = []
results = []
ground_truths = []
self.model.eval()
for batch, x in enumerate(valid):
out = self.model(x[0].cuda())
loss = self.loss(out, x[1].cuda())
valid_losses.append(loss.item())
results.append(torch.sigmoid(out))
ground_truths.append(x[1])
# x = x.cpu()
inference_result = torch.cat(results, dim=0).detach().cpu().numpy()
ground_truth = torch.cat(ground_truths, dim=0).detach().cpu().numpy()
return valid_losses, inference_result, ground_truth
def train_model(self, opt, train):
train_losses = []
for batch, x in enumerate(train):
opt.zero_grad()
out = self.model(x[0].cuda())
loss = self.loss(out, x[1].cuda())
loss.backward()
opt.step()
# x = x.cpu()
train_losses.append(loss.item())
return train_losses
def inference(self, test):
self.model.cuda()
self.model.eval()
results = []
ground_truths = []
for batch, x in enumerate(test):
results.append(
torch.sigmoid(self.model(x[0].cuda())).detach().cpu().numpy())
ground_truths.append(x[1].detach().cpu().numpy())
return np.concatenate(results, axis=0), np.concatenate(ground_truths,
axis=0)
# return torch.cat(results, dim=0).detach().cpu().numpy(), torch.cat(ground_truths, dim=0).detach().cpu().numpy()
@staticmethod
def evaluate(pred, ground_truth, print_result=True):
auc = roc_auc_score(ground_truth, pred)
if print_result:
print("AUC = ", auc)
print("error = ",
1 - np.sum(np.round(pred) == ground_truth) / len(pred))
return auc
def get_loader(self, raw, emb, shuffle=True):
X, y = raw
if self.mode is 'both':
norm = np.linalg.norm(emb, axis=-1, keepdims=True)
emb = emb / (norm + 1e-8)
# if not self.args.all_trees:
emb = emb.reshape(-1, self.embedding_size_to_mlp)
# emb = np.mean(emb, axis=1)
X = np.concatenate([X, emb], axis=1)
elif self.mode is 'raw_only':
pass
elif self.mode is 'emb_only':
norm = np.linalg.norm(emb, axis=-1, keepdims=True)
emb = emb / (norm + 1e-8)
# if not self.args.all_trees:
X = emb.reshape(-1, self.embedding_size_to_mlp)
# else:
# X = emb
# X = np.mean(emb, axis=1)
else:
raise Exception(
"unidentified mode. possible={'raw_only', 'emb_only', 'both'}")
dataset = MLPDataset(np.nan_to_num(X, copy=False), y)
return DataLoader(dataset,
batch_size=self.args.mlp_batch_size,
shuffle=shuffle)
def run(self, raws, embs):
train_loader = self.get_loader(raws[0], embs[0])
self.timer.toc("train loader done")
valid_loader = self.get_loader(raws[1], embs[1])
self.timer.toc("valid loader done")
test_loader = self.get_loader(raws[2], embs[2], shuffle=False)
self.timer.toc("test loader done")
if not self.args.load_mlp:
self.trainIters(train_loader, valid_loader)
train_pred, train_true = self.inference(train_loader)
self.timer.toc("train inference done")
valid_pred, valid_true = self.inference(valid_loader)
self.timer.toc("valid inference done")
test_pred, test_true = self.inference(test_loader)
self.timer.toc("test inference done")
print('train')
self.evaluate(train_pred, train_true)
print('valid')
self.evaluate(valid_pred, valid_true)
print('test')
self.evaluate(test_pred, test_true)
return test_pred
def __getConfig(self, argv):
try:
opts, args = getopt.getopt(
argv, "p:f:m:o:d:",
["package=", "apkfile=", "monkeytime=", "output=", "device="])
except getopt.GetoptError:
self.__printUseMethod()
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
self.__printUseMethod()
sys.exit()
elif opt in ("-p", "--package"):
self.pkgname = arg
elif opt in ("-f", "--apkfile"):
self.apkFilePath = arg
elif opt in ("-m", "--monkeytime"):
self.monkeyTime = int(arg)
self.monkeyMode = True
elif opt in ("-o", "--output"):
self.recordOutPath = arg
if not os.path.exists(self.recordOutPath):
os.makedirs(self.recordOutPath)
elif opt in ("-d", "--device"):
self.udid = arg
else:
print("err in args")
self.__printUseMethod()
sys.exit(1)
if self.pkgname == "" or self.pkgname is None:
print("package name is necessary")
self.__printUseMethod()
sys.exit(1)
# 未指定设备取第一个
if self.udid == "":
print("no specified device!")
self.udid = [
line.split('\t')[0]
for line in os.popen("adb devices", 'r', 1).read().split('\n')
if len(line) != 0 and line.find('\tdevice') != -1
][0]
if self.udid == "":
print("no available devices!")
sys.exit(1)
# 初始化Logger
self.logger = initLogger(loggerName="UIDump_%s" % self.pkgname,
outputPath=os.path.join(
LOG_OUTPUT_PATH,
"UIDump_%s.log" % self.pkgname),
udid=self.udid)
# 初始化 uiautomator
try:
self.device = DeviceConnect(self.logger, self.udid)
except Exception as e:
self.runStatus = RunStatus.UI2_INIT_ERR
return
# Monkey模式 设置计时器
if self.monkeyMode:
self.timer = Timer(logger=self.logger,
duration=self.monkeyTime,
device=self.device)
# APK_FILE不为空，表示需要从指定路径安装app
if self.apkFilePath is not "":
self.runStatus = self.device.installApk(self.pkgname,
self.apkFilePath)
pass
class UIDump:
def __init__(self, argv):
self.udid = ""
self.pkgname = ""
self.dumpInterval = DUMP_INTERVAL
self.apkFilePath = ""
self.monkeyMode = False
self.monkeyTime = MONKEY_TIME
self.recordOutPath = RECORD_OUTPUT_PATH
self.device = None
self.timer = None
self.logger = None
self.runStatus = RunStatus.SUCCESS
self.__getConfig(argv)
pass
def __getConfig(self, argv):
try:
opts, args = getopt.getopt(
argv, "p:f:m:o:d:",
["package=", "apkfile=", "monkeytime=", "output=", "device="])
except getopt.GetoptError:
self.__printUseMethod()
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
self.__printUseMethod()
sys.exit()
elif opt in ("-p", "--package"):
self.pkgname = arg
elif opt in ("-f", "--apkfile"):
self.apkFilePath = arg
elif opt in ("-m", "--monkeytime"):
self.monkeyTime = int(arg)
self.monkeyMode = True
elif opt in ("-o", "--output"):
self.recordOutPath = arg
if not os.path.exists(self.recordOutPath):
os.makedirs(self.recordOutPath)
elif opt in ("-d", "--device"):
self.udid = arg
else:
print("err in args")
self.__printUseMethod()
sys.exit(1)
if self.pkgname == "" or self.pkgname is None:
print("package name is necessary")
self.__printUseMethod()
sys.exit(1)
# 未指定设备取第一个
if self.udid == "":
print("no specified device!")
self.udid = [
line.split('\t')[0]
for line in os.popen("adb devices", 'r', 1).read().split('\n')
if len(line) != 0 and line.find('\tdevice') != -1
][0]
if self.udid == "":
print("no available devices!")
sys.exit(1)
# 初始化Logger
self.logger = initLogger(loggerName="UIDump_%s" % self.pkgname,
outputPath=os.path.join(
LOG_OUTPUT_PATH,
"UIDump_%s.log" % self.pkgname),
udid=self.udid)
# 初始化 uiautomator
try:
self.device = DeviceConnect(self.logger, self.udid)
except Exception as e:
self.runStatus = RunStatus.UI2_INIT_ERR
return
# Monkey模式 设置计时器
if self.monkeyMode:
self.timer = Timer(logger=self.logger,
duration=self.monkeyTime,
device=self.device)
# APK_FILE不为空，表示需要从指定路径安装app
if self.apkFilePath is not "":
self.runStatus = self.device.installApk(self.pkgname,
self.apkFilePath)
pass
def __printUseMethod(self):
print("UIDump.py -p <app-package-name> [-t] <dump-interval> "
"[-f] <apk-file-path> [-m] <monkey-run-time> [-o] <output-path>")
print("arguments : ")
print(
"-p --package\tinput app is necessary, such as \"-p com.tencent.mm\""
)
print(
"-f --apkfile\tapk file path, if the app isn't installed, "
"you can specify the apk file path, such as '/home/user/a.apk' or 'http://127.0.0.1:8000/user/a.apk"
)
print(
"-m --monkeytime\t monkey run time, if it isn't specified, you can dump through manual operation"
)
print("-o --output\t output path, default is ./output/record")
pass
def startUIDump(self):
if not isSuccess(self.runStatus):
self.device.uninstallApk(self.pkgname)
return self.runStatus
self.logger.info(
"start record mode, the package is %s and dump interval is %d" %
(self.pkgname, self.dumpInterval))
timestamp = time.strftime('%Y%m%d%H%M', time.localtime())
self.logger.info("log start at %s" % timestamp)
try:
self.runStatus = self.startRecord(timestamp)
except Exception as e:
traceback.print_exc()
self.logger.error("unknown err in dump %s, Reason: %s" %
(self.pkgname, e))
self.runStatus = RunStatus.ERROR
if self.apkFilePath is not "":
self.device.uninstallApk(self.pkgname)
timestamp = time.strftime('%Y%m%d%H%M', time.localtime())
self.logger.info("log end at %s\r\n" % timestamp)
return self.runStatus
def startRecord(self, timestamp):
if self.pkgname == "":
self.logger.error("no input package name")
return RunStatus.ERROR
if self.pkgname not in self.device.getInstalledApps():
self.logger.error("%s is not installed" % self.pkgname)
return RunStatus.APK_INSTALL_ERR
outputpath = os.path.join(self.recordOutPath,
self.pkgname + "_" + timestamp)
if not os.path.exists(outputpath):
os.makedirs(outputpath)
# 初始化frida
# ch = CallerHook(self.pkgname, outputpath)
time.sleep(1)
# 设置回调事件
try:
# self.device.startWatchers()
pass
except Exception:
import shutil
shutil.rmtree(outputpath)
self.logger.error("err in start watcher")
return RunStatus.UI2_WATCHER_ERR
if self.timer is None:
# 没设置Timer, 人工跑APP还是用home键退出脚本
stopcondition = self.device.getCurrentApp()
while stopcondition is "":
stopcondition = self.device.getCurrentApp()
self.timer = Timer(stopcondition=stopcondition,
device=self.device,
logger=self.logger)
self.device.pressHome()
dumpcount = 1
time.sleep(1)
# 目前利用frida孵化进程有bug，用Monkey起
self.runStatus = self.device.startApp(self.pkgname)
if not isSuccess(self.runStatus):
import shutil
shutil.rmtree(outputpath)
self.logger.error("err in start app")
return self.runStatus
# ch.start_hook(os.path.join("OneForAllHook", "_agent.js"), str(self.udid))
time.sleep(5) # 有时候app界面还没加载出来，等5s
# 如果设置了MONKEY_TIME，启动monkey
monkey = None
if self.monkeyMode and self.device.getAppInstallStatus():
monkey = Monkey(logger=self.logger,
udid=self.udid,
pkgname=self.pkgname,
timeInterval=MONKEY_TIME_INTERVAL,
outdir=outputpath)
self.runStatus = monkey.startMonkey()
if not isSuccess(self.runStatus):
import shutil
shutil.rmtree(outputpath)
self.logger.error("err in start monkey")
return self.runStatus
# 启动计时器
starttime = self.timer.start()
# 异常重启次数，有些app确实起不起来，最多重启3次
errRestartCount = 0
# 等待启动之后再轮询判断是否已经退出
while True:
if not self.device.isAppRun(self.pkgname):
self.logger.info("app %s is not running " % self.pkgname)
if monkey is not None:
execStatus, self.runStatus = monkey.stopMonkey()
if execStatus:
monkey = None
if not isSuccess(self.runStatus):
break
# 如果app异常退出，且计时未结束重启app
# getAppInstallStatus 避免当前app因为apk问题导致反复重启
if self.monkeyMode and not self.timer.isFinish(
) and self.device.getAppInstallStatus(
) and errRestartCount < 3:
errRestartCount += 1
self.logger.warning(
"Abnormal termination in app running, restart, count = %d"
% errRestartCount)
self.device.closeWatchers()
try:
self.device.startWatchers()
except Exception:
self.runStatus = RunStatus.UI2_WATCHER_ERR
break
self.runStatus = self.device.startApp(self.pkgname)
if not isSuccess(self.runStatus):
break
# ch.start_hook(os.path.join("OneForAllHook", "_agent.js"), str(self.udid))
time.sleep(5)
monkey = Monkey(logger=self.logger,
udid=self.udid,
pkgname=self.pkgname,
timeInterval=MONKEY_TIME_INTERVAL,
outdir=outputpath)
self.runStatus = monkey.startMonkey()
if not isSuccess(self.runStatus):
break
continue
self.device.stopApp(self.pkgname)
self.logger.info(
self.pkgname +
" is canceled, stop record, with restart count = %d" %
errRestartCount)
# ch.stop_hook()
self.device.pressHome()
break
self.logger.info("dump %s UI" % str(dumpcount))
self.runStatus = self.device.dumpUI(outputpath, dumpcount)
if not isSuccess(self.runStatus):
break
dumpcount += 1
time.sleep(self.dumpInterval)
if self.timer.isFinish():
if monkey is not None:
execStatus, self.runStatus = monkey.stopMonkey()
if execStatus:
monkey = None
if not isSuccess(self.runStatus):
break
self.device.stopApp(self.pkgname)
self.device.pressHome()
break
self.device.closeWatchers()
time.sleep(5)
if monkey is not None:
_, self.runStatus = monkey.stopMonkey()
if not self.device.getAppInstallStatus():
import shutil
shutil.rmtree(outputpath)
self.logger.warning("err in apk, pass the case")
return RunStatus.APK_INSTALL_ERR
elif errRestartCount >= 3:
import shutil
shutil.rmtree(outputpath)
self.logger.warning("app restart more than 3 times, pass the case")
return RunStatus.APK_INSTALL_ERR
elif not isSuccess(self.runStatus):
import shutil
shutil.rmtree(outputpath)
self.logger.error("err in restart record")
return self.runStatus
else:
self.device.saveLog(outputpath, starttime)
self.logger.info("the output saved in " + outputpath)
return self.runStatus.SUCCESS
def startRecord(self, timestamp):
if self.pkgname == "":
self.logger.error("no input package name")
return RunStatus.ERROR
if self.pkgname not in self.device.getInstalledApps():
self.logger.error("%s is not installed" % self.pkgname)
return RunStatus.APK_INSTALL_ERR
outputpath = os.path.join(self.recordOutPath,
self.pkgname + "_" + timestamp)
if not os.path.exists(outputpath):
os.makedirs(outputpath)
# 初始化frida
# ch = CallerHook(self.pkgname, outputpath)
time.sleep(1)
# 设置回调事件
try:
# self.device.startWatchers()
pass
except Exception:
import shutil
shutil.rmtree(outputpath)
self.logger.error("err in start watcher")
return RunStatus.UI2_WATCHER_ERR
if self.timer is None:
# 没设置Timer, 人工跑APP还是用home键退出脚本
stopcondition = self.device.getCurrentApp()
while stopcondition is "":
stopcondition = self.device.getCurrentApp()
self.timer = Timer(stopcondition=stopcondition,
device=self.device,
logger=self.logger)
self.device.pressHome()
dumpcount = 1
time.sleep(1)
# 目前利用frida孵化进程有bug，用Monkey起
self.runStatus = self.device.startApp(self.pkgname)
if not isSuccess(self.runStatus):
import shutil
shutil.rmtree(outputpath)
self.logger.error("err in start app")
return self.runStatus
# ch.start_hook(os.path.join("OneForAllHook", "_agent.js"), str(self.udid))
time.sleep(5) # 有时候app界面还没加载出来，等5s
# 如果设置了MONKEY_TIME，启动monkey
monkey = None
if self.monkeyMode and self.device.getAppInstallStatus():
monkey = Monkey(logger=self.logger,
udid=self.udid,
pkgname=self.pkgname,
timeInterval=MONKEY_TIME_INTERVAL,
outdir=outputpath)
self.runStatus = monkey.startMonkey()
if not isSuccess(self.runStatus):
import shutil
shutil.rmtree(outputpath)
self.logger.error("err in start monkey")
return self.runStatus
# 启动计时器
starttime = self.timer.start()
# 异常重启次数，有些app确实起不起来，最多重启3次
errRestartCount = 0
# 等待启动之后再轮询判断是否已经退出
while True:
if not self.device.isAppRun(self.pkgname):
self.logger.info("app %s is not running " % self.pkgname)
if monkey is not None:
execStatus, self.runStatus = monkey.stopMonkey()
if execStatus:
monkey = None
if not isSuccess(self.runStatus):
break
# 如果app异常退出，且计时未结束重启app
# getAppInstallStatus 避免当前app因为apk问题导致反复重启
if self.monkeyMode and not self.timer.isFinish(
) and self.device.getAppInstallStatus(
) and errRestartCount < 3:
errRestartCount += 1
self.logger.warning(
"Abnormal termination in app running, restart, count = %d"
% errRestartCount)
self.device.closeWatchers()
try:
self.device.startWatchers()
except Exception:
self.runStatus = RunStatus.UI2_WATCHER_ERR
break
self.runStatus = self.device.startApp(self.pkgname)
if not isSuccess(self.runStatus):
break
# ch.start_hook(os.path.join("OneForAllHook", "_agent.js"), str(self.udid))
time.sleep(5)
monkey = Monkey(logger=self.logger,
udid=self.udid,
pkgname=self.pkgname,
timeInterval=MONKEY_TIME_INTERVAL,
outdir=outputpath)
self.runStatus = monkey.startMonkey()
if not isSuccess(self.runStatus):
break
continue
self.device.stopApp(self.pkgname)
self.logger.info(
self.pkgname +
" is canceled, stop record, with restart count = %d" %
errRestartCount)
# ch.stop_hook()
self.device.pressHome()
break
self.logger.info("dump %s UI" % str(dumpcount))
self.runStatus = self.device.dumpUI(outputpath, dumpcount)
if not isSuccess(self.runStatus):
break
dumpcount += 1
time.sleep(self.dumpInterval)
if self.timer.isFinish():
if monkey is not None:
execStatus, self.runStatus = monkey.stopMonkey()
if execStatus:
monkey = None
if not isSuccess(self.runStatus):
break
self.device.stopApp(self.pkgname)
self.device.pressHome()
break
self.device.closeWatchers()
time.sleep(5)
if monkey is not None:
_, self.runStatus = monkey.stopMonkey()
if not self.device.getAppInstallStatus():
import shutil
shutil.rmtree(outputpath)
self.logger.warning("err in apk, pass the case")
return RunStatus.APK_INSTALL_ERR
elif errRestartCount >= 3:
import shutil
shutil.rmtree(outputpath)
self.logger.warning("app restart more than 3 times, pass the case")
return RunStatus.APK_INSTALL_ERR
elif not isSuccess(self.runStatus):
import shutil
shutil.rmtree(outputpath)
self.logger.error("err in restart record")
return self.runStatus
else:
self.device.saveLog(outputpath, starttime)
self.logger.info("the output saved in " + outputpath)
return self.runStatus.SUCCESS
from src.Timer import Timer
from src.UsbSwitcher import UsbSwitcher
from src.service.CheckService import CheckService
checkService = CheckService()
if (checkService.check()):
switcher = UsbSwitcher()
switcher.switch()
timer = Timer()
timer.timer()
switcher.switch()
class Blink_Component:
def __init__(self, player):
from src.Timer import Timer
from src.Sprite import Sprite
self.valid_blink_points = []
self.dot_spr = Sprite(ImageEnum.BLINK_DOT)
self.dot_spr.bounds = (0, 0, 4, 4)
self.can_blink = False
self.player = player
self.blink_frames = 3
self.is_blinking = False
self.frame_displacement = (None, None)
self.frames_passed = None
self.timer = Timer()
self.cooldown_ms = 2500
self.cooldown_passed = False
self.state = BlinkState.CAN_BLINK
def get_actual_mouse_pos(self):
from src.WorldConstants import CONST_CAMERA_PLAYER_OFFSET_X, CONST_CAMERA_PLAYER_OFFSET_Y
import pygame
x, y = pygame.mouse.get_pos()
player_x, player_y = self.player.sprite.sprite_rect().topleft
# x_origin = 0
# y_origin = 0
x_origin = player_x - CONST_CAMERA_PLAYER_OFFSET_X
y_origin = player_y - CONST_CAMERA_PLAYER_OFFSET_Y
mouse_x = x + x_origin
mouse_y = y + y_origin
return mouse_x, mouse_y
def fill_valid_blink_points(self):
player_x, player_y = self.player.sprite.sprite_rect().center
mouse_x, mouse_y = self.get_actual_mouse_pos()
if mouse_x == player_x:
mouse_x += 1
m = (player_y - mouse_y) / (player_x - mouse_x)
c = player_y - (m * player_x)
#self.valid_blink_points = []
line = Line(player_x, player_y, mouse_x, mouse_y)
self.valid_blink_points = line.get_valid_points(
self.player.level, 1, self.player.level.colliders)
# step = 0
#
# if player_x <= mouse_x:
# step = 1
# elif mouse_x < player_x:
# step = -1
#
# for x in range(player_x, mouse_x, step):
# y = m*x + c
# if self.player.level.point_in_wall((x, y)) or self.player.level.point_in_collider((x, y)):
# break
# self.valid_blink_points.append((x, y))
def draw(self, screen, camera):
if self.state == BlinkState.SHOWING_LINE:
self.fill_valid_blink_points()
for x, y in self.valid_blink_points:
self.dot_spr.set_location((x, y))
self.dot_spr.draw(screen, camera)
def handle_event(self, event):
pass
def update(self, deltaTime):
keys = pygame.key.get_pressed()
mice = pygame.mouse.get_pressed()
left_pressed = bool(mice[0])
from src.WorldConstants import BLINK_KEY
if False:
pass
elif self.state == BlinkState.CAN_BLINK:
if keys[BLINK_KEY]:
self.state = BlinkState.SHOWING_LINE
elif self.state == BlinkState.SHOWING_LINE:
if not keys[BLINK_KEY]:
self.state = BlinkState.CAN_BLINK
elif left_pressed:
self.state = BlinkState.BLINKING
play_sound(SoundEnum.BLINK)
elif self.state == BlinkState.BLINKING:
is_first_blink_frame = self.frames_passed == None
if is_first_blink_frame:
self.frames_passed = 0
player_x, player_y = self.player.sprite.sprite_rect().center
valid_x, valid_y = self.valid_blink_points[-1]
d_x = (valid_x - player_x)
d_y = (valid_y - player_y)
self.frame_displacement = (d_x / self.blink_frames,
d_y / self.blink_frames)
self.player.moving_component.move(self.frame_displacement)
self.frames_passed = self.frames_passed + 1
elif not is_first_blink_frame:
if self.frames_passed < self.blink_frames:
self.player.moving_component.move(self.frame_displacement)
self.frames_passed = self.frames_passed + 1
elif self.frames_passed >= self.blink_frames:
self.frames_passed = None
self.frame_displacement = (None, None)
self.timer.reset()
self.state = BlinkState.COOLING_DOWN
elif self.state == BlinkState.COOLING_DOWN:
cooldown_passed = self.timer.get_time() > self.cooldown_ms
if cooldown_passed:
self.state = BlinkState.CAN_BLINK
def __init__(self, health, cooldown_seconds):
from src.Timer import Timer
self.health = health
self.cooldown_seconds = cooldown_seconds
self.time_elapsed_since_hit = self.cooldown_seconds
self.timer = Timer()
def __init__(self, args, max_length, timer: Timer = None):
self.timer = Timer() if timer is None else timer
self.args = args
self.model: XGBEmbedding = XGBEmbedding(args.num_trees_for_embedding, max_length, args.embedding_size)
total_params = sum(x.data.nelement() for x in self.model.parameters())
print("Model total number of parameters: {}".format(total_params))
from src.Timer import Timer
if __name__ == '__main__':
t = Timer(12)
t.begin()
class XGBEmbeddingTrainer:
def __init__(self, args, max_length, timer: Timer = None):
self.timer = Timer() if timer is None else timer
self.args = args
self.model: XGBEmbedding = XGBEmbedding(args.num_trees_for_embedding, max_length, args.embedding_size)
total_params = sum(x.data.nelement() for x in self.model.parameters())
print("Model total number of parameters: {}".format(total_params))
def trainIters(self, train, valid):
# ADAM opts
opt = optim.Adam(self.model.parameters(), lr=self.args.lr, weight_decay=self.args.weight_decay)
early_stopper = EarlyStopper(3, 'moving')
min_loss = 1e30
################ Training epoch
self.model.cuda()
for epoch in range(1, self.args.num_epoch + 1):
self.model.train()
train_losses = self.train_model(opt, train)
valid_losses = self.valid_model(valid)
valid_mean_loss = np.mean(valid_losses)
self.timer.toc("epoch {:4d} - train loss: {:10.6f} valid loss: {:10.6f}".format(epoch, np.mean(train_losses),
valid_mean_loss))
checkpoint = {'model': self.model, 'args': self.args}
model_name = "{:s}_{:d}_{:d}.chkpt".format(self.args.model_name, self.args.max_depth,
self.args.num_trees_for_embedding)
if valid_mean_loss < min_loss:
min_loss = valid_mean_loss
torch.save(checkpoint, model_name)
if early_stopper.record(valid_mean_loss):
self.model = torch.load(model_name)['model']
return
def valid_model(self, valid):
valid_losses = []
self.model.eval()
for batch, x in enumerate(valid):
x[0] = x[0].cuda()
loss = torch.mean(self.model(x[0]))
valid_losses.append(loss.item())
# x = x.cpu()
return valid_losses
def train_model(self, opt, train):
train_losses = []
for batch, x in enumerate(train):
opt.zero_grad()
x[0] = x[0].cuda()
loss = torch.mean(self.model(x[0]))
loss.backward()
opt.step()
# x = x.cpu()
train_losses.append(loss.item())
return train_losses
def inference(self, test):
self.model.cuda()
self.model.eval()
results = []
for batch, x in enumerate(test):
x[0] = x[0].cuda()
results.append(self.model.inference(x[0]))
return torch.cat(results, dim=0).detach().cpu().numpy()
def init_model(self, train, valid):
if self.args.load is False:
self.trainIters(train, valid)
else:
model_name = "{:s}_{:d}_{:d}.chkpt".format(self.args.model_name, self.args.max_depth,
self.args.num_trees_for_embedding)
self.model = torch.load(model_name)['model']
self.valid_model(valid)
def get_embedding(self, loaders, trees):
XGBEmbeddingEvaluator(self.model.get_weights().detach().cpu().numpy(), trees, print_eval=self.args.print_eval)
emb = []
for loader in loaders:
emb.append(self.inference(loader))
return emb
def __init__(self, args, load_raw=False):
self.train_parsed_file = '../data/data/ieee/train.csv'
self.valid_parsed_file = '../data/data/ieee/valid.csv'
self.test_parsed_file = '../data/data/ieee/test.csv'
self.args = args
self.id_cat_col = ["id_" + str(i) for i in range(12, 39)] + [
"DeviceType", "DeviceInfo", "id_33_1", "id_31_1", "id_30_1"
]
self.trans_cat_col = ["M" + str(i) for i in range(1, 10)] + ["card" + str(i) for i in range(1, 7)] + \
["ProductCD", "addr1", "addr2", "P_emaildomain", "R_emaildomain"]
# self.try_list = ["id_" + str(i) for i in [13, 17]]
self.try_list = ["id_" + str(i) for i in []]
self.id_onehot_encoder = OneHot(0.01)
self.trans_onehot_encoder = OneHot(0.01)
self.id_scaler = Scaler()
self.trans_scaler = Scaler()
self.global_name = []
self.global_df = []
self.emails = {
'gmail': 'google',
'att.net': 'att',
'twc.com': 'spectrum',
'scranton.edu': 'other',
'optonline.net': 'other',
'hotmail.co.uk': 'microsoft',
'comcast.net': 'other',
'yahoo.com.mx': 'yahoo',
'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo',
'charter.net': 'spectrum',
'live.com': 'microsoft',
'aim.com': 'aol',
'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink',
'gmail.com': 'google',
'me.com': 'apple',
'earthlink.net': 'other',
'gmx.de': 'other',
'web.de': 'other',
'cfl.rr.com': 'other',
'hotmail.com': 'microsoft',
'protonmail.com': 'other',
'hotmail.fr': 'microsoft',
'windstream.net': 'other',
'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo',
'servicios-ta.com': 'other',
'netzero.net': 'other',
'suddenlink.net': 'other',
'roadrunner.com': 'other',
'sc.rr.com': 'other',
'live.fr': 'microsoft',
'verizon.net': 'yahoo',
'msn.com': 'microsoft',
'q.com': 'centurylink',
'prodigy.net.mx': 'att',
'frontier.com': 'yahoo',
'anonymous.com': 'other',
'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att',
'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo',
'outlook.com': 'microsoft',
'mail.com': 'other',
'bellsouth.net': 'other',
'embarqmail.com': 'centurylink',
'cableone.net': 'other',
'hotmail.es': 'microsoft',
'mac.com': 'apple',
'yahoo.co.uk': 'yahoo',
'netzero.com': 'other',
'yahoo.com': 'yahoo',
'live.com.mx': 'microsoft',
'ptd.net': 'other',
'cox.net': 'other',
'aol.com': 'aol',
'juno.com': 'other',
'icloud.com': 'apple'
}
self.drop_col = [
'TransactionDT', 'V300', 'V309', 'V111', 'C3', 'V124', 'V106',
'V125', 'V315', 'V134', 'V102', 'V123', 'V316', 'V113', 'V136',
'V305', 'V110', 'V299', 'V289', 'V286', 'V318', 'V103', 'V304',
'V116', 'V298', 'V284', 'V293', 'V137', 'V295', 'V301', 'V104',
'V311', 'V115', 'V109', 'V119', 'V321', 'V114', 'V133', 'V122',
'V319', 'V105', 'V112', 'V118', 'V117', 'V121', 'V108', 'V135',
'V320', 'V303', 'V297', 'V120'
]
self.us_emails = ['gmail', 'net', 'edu']
self.timer = Timer()
if load_raw:
df_trans = pd.read_csv("../data/data/ieee/train_transaction.csv"
).set_index("TransactionID")
df_id_raw = pd.read_csv("../data/data/ieee/train_identity.csv"
).set_index("TransactionID")
dt_trans = pd.read_csv("../data/data/ieee/test_transaction.csv"
).set_index("TransactionID")
dt_id = pd.read_csv("../data/data/ieee/test_identity.csv"
).set_index("TransactionID")
self.timer.toc("read done")
self.test_id = list(dt_trans.index)
df_trans, dv_trans = train_test_split(
df_trans,
random_state=args.random_state,
train_size=0.8,
test_size=0.2)
df_id = df_id_raw.loc[
df_trans.index.intersection(df_id_raw.index), :]
dv_id = df_id_raw.loc[
dv_trans.index.intersection(df_id_raw.index), :]
self.timer.toc("split done")
self.feature_engineering_id(df_id)
self.feature_engineering_id(dv_id)
self.feature_engineering_id(dt_id)
self.timer.toc("process id done")
self.feature_engineering_trans(df_trans)
self.feature_engineering_trans(dv_trans)
self.feature_engineering_trans(dt_trans)
self.timer.toc("process trans done")
self.global_count(df_id, df_trans)
self.timer.toc("global_count done")
df_trans, ytrain = self.split_x_y(df_trans)
dv_trans, yvalid = self.split_x_y(dv_trans)
dt_trans, ytest = self.split_x_y(dt_trans)
df_global = self.get_derived(df_id)
self.timer.toc("get train derived done")
dv_global = self.get_derived(dv_id)
self.timer.toc("get valid derived done")
dt_global = self.get_derived(dt_id)
self.timer.toc("get test derived done")
self.init_onehot(df_id, df_trans)
self.init_scaler(df_id.drop(self.id_cat_col, axis=1),
df_trans.drop(self.trans_cat_col, axis=1))
Xtrain = self.transform(df_id, df_trans, df_global)
self.timer.toc("transform train done")
Xvalid = self.transform(dv_id, dv_trans, dv_global)
self.timer.toc("transform valid done")
Xtest = self.transform(dt_id, dt_trans, dt_global)
self.timer.toc("transform test done")
pd.concat([Xtrain, ytrain], axis=1).to_csv(self.train_parsed_file,
float_format='%.8f',
index=True)
self.timer.toc("write train done")
pd.concat([Xvalid, yvalid], axis=1).to_csv(self.valid_parsed_file,
float_format='%.8f',
index=True)
self.timer.toc("write valid done")
Xtest.to_csv(self.test_parsed_file,
float_format='%.8f',
index=True)
self.timer.toc("write test done")
raise Exception("STOP HERE!")
else:
self.train = self.load(self.train_parsed_file)
self.timer.toc("load train done")
self.valid = self.load(self.valid_parsed_file)
self.timer.toc("load valid done")
self.test = self.load(self.test_parsed_file, test=True)
self.timer.toc("load test done")
self.num_input = self.train[0].shape[1]
# sanity check
print(np.unique(self.train[1], return_counts=True))
print(np.unique(self.valid[1], return_counts=True))