def set_max_seq_len(self):
if self.max_length > MAX_SEQ_LENGTH:
self.max_length = MAX_SEQ_LENGTH
info("update max_length {}".format(self.max_length))
if self.seq_len_std > MAX_TOLERANT_STD:
self.max_length = MAX_SEQ_LENGTH
info("update max_length {}".format(self.max_length))
def run_first_stage_model(self, train_preprocessed_data, train_diff_y):
if self.switch_new_model and not self.cur_model_train_start: # 如果切换模型，且当前模型没有开始训练
self._clear_train_space()
self._init_nn_train_process()
self.model = self.model_manager.select_classifier(model_name=self.model_lib[self.model_id],
feature_mode=self.feature_lib[self.feature_id],
data_feature=self.feature_generator.data_feature)
info(color_msg("start new nn model training!"))
if self.model_lib[self.model_id] == "text_cnn":
if self.imbalance_level == 2 or self.metadata["class_num"] >= 5:
self.callbacks = []
else:
self.callbacks = [self.callbacks_[0]]
else:
self.callbacks = [self.callbacks_[1]]
self._train_nn_process(train_preprocessed_data, train_diff_y)
if self.train_model_id >= 1: # 训练了至少2个模型
self._ensemble_multi_models()
else:
self._ensemble_multi_models()
# 达到结束的条件
if self.evaluator.decide_stop(train_epoch=self.train_epoch):
self._reset_train_status()
def recompileApk(workDir, decompileDir, game, channel, destApkName):
targetApk = workDir + "/output.apk"
log_utils.debug("now to recompileApk for %s", destApkName)
ret = apk_utils.recompileApk(decompileDir, targetApk)
if ret:
return 1
apk_utils.copyRootResFiles(targetApk, decompileDir)
if 'signApk' not in channel or channel['signApk'] != '0':
ret = apk_utils.signApk(workDir, game, channel, targetApk)
if ret:
return 1
else:
log_utils.debug("the apk is set to unsigned.")
channelNameStr = channel["name"].replace(' ', '')
# if isPublic:
# #destApkName = channelNameStr + '-' + time.strftime('%Y%m%d%H') + '.apk'
# destApkName = apk_utils.getOutputApkName(game, channel, newPackageName, decompileDir)
# else:
# destApkName = channelNameStr + '-' + time.strftime('%Y%m%d%H') + '-debug.apk'
destApkPath = file_utils.getFullOutputPath(game["appName"],
channel["name"])
destApkPath = os.path.join(destApkPath, destApkName)
ret = apk_utils.alignApk(targetApk, destApkPath)
if ret:
return 1
log_utils.info("channel %s package success to %s.", channel["name"],
destApkName)
return 0
def entry(isPublic, isSelectable, threadNum):
log_utils.info("Curr Python Version::%s", config_utils.get_py_version())
print(u"**********所有游戏**********")
print(u"\t appID \t\t 游戏文件夹 \t\t 游戏名称 \n\n")
games = config_utils.getAllGames()
if games != None and len(games) > 0:
for ch in games:
print(u"\t %s \t\t %s \t\t\t%s" %
(ch['appID'], ch['appName'], ch['appDesc']))
sys.stdout.write(u"请选择一个游戏(输入appID)：")
sys.stdout.flush()
selectedGameID = raw_input()
selectedGameID = str(selectedGameID)
game = getGameByAppID(selectedGameID, games)
log_utils.info("current selected game is %s(%s)", game['appName'],
game['appDesc'])
if isSelectable:
main.main(game, isPublic)
else:
main_thread.main(game, isPublic, threadNum)
def modifyFileContent(sourcefile, oldContent, newContent):
if os.path.isdir(sourcefile):
log_utils.warning("the source %s must be a file not a dir", sourcefile)
return
if not os.path.exists(sourcefile):
log_utils.warning("the source is not exists.path:%s", sourcefile)
return
f = open(sourcefile, 'r+')
data = str(f.read())
f.close()
bRet = False
idx = data.find(oldContent)
while idx != -1:
data = data[:idx] + newContent + data[idx + len(oldContent):]
idx = data.find(oldContent, idx + len(oldContent))
bRet = True
if bRet:
fw = open(sourcefile, 'w')
fw.write(data)
fw.close()
log_utils.info("modify file success.path:%s", sourcefile)
else:
log_utils.warning(
"there is no content matched in file:%s with content:%s",
sourcefile, oldContent)
def _show_runtime_info(self):
info(color_msg("********************************************************"))
info(color_msg("current model_id is {}, model_name is {}".format(self.model_id, self.model_lib[self.model_id])))
info(color_msg(
"current feature_id is {}, feature_name is {}".format(self.feature_id, self.feature_lib[self.feature_id])))
info(color_msg("train_model_id is {}".format(self.train_model_id)))
info(color_msg("********************************************************\n"))
def entry(isPublic, isSelectable, threadNum, appID, target):
log_utils.info("Curr Python Version::%s", config_utils.get_py_version())
print(u"**********所有游戏**********")
print(u"\t appID \t\t 游戏文件夹 \t\t 游戏名称 \n\n")
games = config_utils.getAllGames()
if games != None and len(games) > 0:
for ch in games:
print(u"\t %s \t\t %s \t\t\t%s" % (ch['appID'], ch['appName'], ch['appDesc']))
sys.stdout.write(u"请选择一个游戏(输入appID)：")
sys.stdout.flush()
selectedGameID = str(appID)
game = getGameByAppID(selectedGameID, games)
log_utils.info("current selected game is %s(%s)", game['appName'], game['appDesc'])
if isSelectable:
return main(game, isPublic, target)
else:
return main_thread.main(game, isPublic, threadNum)
def modifyFileContent(sourcefile, oldContent, newContent):
if os.path.isdir(sourcefile):
log_utils.warning("the source %s must be a file not a dir", sourcefile)
return
if not os.path.exists(sourcefile):
log_utils.warning("the source is not exists.path:%s", sourcefile)
return
f = open(sourcefile, 'r+')
data = str(f.read())
f.close()
bRet = False
idx = data.find(oldContent)
while idx != -1:
data = data[:idx] + newContent + data[idx + len(oldContent):]
idx = data.find(oldContent, idx + len(oldContent))
bRet = True
if bRet:
fw = open(sourcefile, 'w')
fw.write(data)
fw.close()
log_utils.info("modify file success.path:%s", sourcefile)
else:
log_utils.warning("there is no content matched in file:%s with content:%s", sourcefile, oldContent)
def update_model_weights(self, model, train_epoch, is_svm=False):
info(
color_msg(
"train_epoch is {}， cur model: model_weight_list is {}\n".
format(train_epoch, len(self.model_weights_list))))
if self.model_weights_list:
if self.cur_val_auc > self.best_auc:
info(color_msg("Update best result!"))
self.best_auc = self.cur_val_auc
self.is_best = True
self.best_call_num = train_epoch
else:
self.is_best = False
model.set_weights(self.model_weights_list[self.best_call_num])
else: # 新增第一个模型权值
self.is_best = True
self.best_auc = self.cur_val_auc
self.best_call_num = train_epoch
if is_svm:
pass
else:
model_weights = model.get_weights()
self.model_weights_list.append(model_weights)
def get_chongding_by_api():
api_url = 'http://htpmsg.jiecaojingxuan.com/msg/current'
# api_url = 'http://192.168.152.58:8080/1'
req = requests.get(url=api_url)
msg = u''
while msg != u"成功":
req = requests.get(url=api_url)
try:
msg = json.loads(req.text)['msg']
except BaseException:
log_utils.debug("所获取JSON异常%s" % req.text)
msg = u''
log_utils.info("获取JSON成功")
event = json.loads(req.text)['data']['event']
question = event['desc']
answerStr = event['options']
answerStr = answerStr.replace("\"", "")
answerStr = answerStr.replace("[", "")
answerStr = answerStr.replace("]", "")
answer = answerStr.split(",")
if question[1] == '.':
question = question[2:]
else:
question = question[3:]
return question, answer
def dealWithIconPath(badge_path=""):
"""外部入口"""
log_utils.info("进入appicon处理程序---开始处理icon")
# 检查资源
tempfile = file_manager.getFullPath("upload/icon")
file_manager.clear(tempfile)
# 1、获取icon的全路径文件夹
iconPath = file_manager.getFullPath(Index.root_icon)
# 2、获取该路径下的所有文件
icons = file_manager.list_files(iconPath, [], ".DS_Store")
if len(icons) == 0:
log_utils.info("还没有上传appicon资源文件，请上传之后操作！")
return 101
else:
# 如果数组不为空，取出第一个元素
# 获取到icon文件
for filename in icons:
if ".png" in filename:
appIconDir = filename
iOSIconSizes = [
("icon-20@2x", 40),
("icon-20@3x", 60),
("icon-29", 29),
("icon-29@2x", 58),
("icon-29@3x", 87),
("icon-40@2x", 80),
("icon-40@3x", 120),
("icon-60@2x", 120),
("icon-60@3x", 180),
("icon-1024", 1024),
("icon-40", 40),
("icon-72", 72),
("icon-29-ipad", 29),
("icon-72@2x", 144),
("icon-76", 76),
("icon-76@2x", 152),
("icon-83.5@2x", 167),
("icon-57", 57),
("icon-57@2x", 114),
("icon-20-ipad", 20),
("icon-20-ipad@2x", 40),
("icon-50", 50),
("icon-50@2x", 100),
("icon-29-ipad@2x", 58),
]
AndroidIconList = [("drawable-ldpi", 36, "icon-36"),
("drawable-mdpi", 48, "icon-48"),
("drawable-hdpi", 72, "icon-72"),
("drawable-xhdpi", 96, "icon-96"),
("drawable-xxhdpi", 144, "icon-144"),
("drawable-xxxhdpi", 192, "icon-192"),
("drawable", 512, "icon-512")]
processingIcons(appIconDir, "ios", iOSIconSizes)
processingIcons(appIconDir, "android", AndroidIconList)
make_archiveWithInfo()
def signApk(appName, channelId, apkfile):
"""
Sign apk
"""
keystore = config_utils.getKeystore(appName, channelId)
log_utils.info("the keystore file is %s", keystore['keystore'])
signApkInternal(apkfile, keystore['keystore'], keystore['password'], keystore['aliaskey'], keystore['aliaspwd'])
def decide_stop(self, train_epoch):
self.update_early_stop_params()
self.val_auc_list.append(self.cur_val_auc)
self.stop_criteria = self.check_early_stop_criteria(train_epoch)
info(
color_msg("Note: stop condition is {}".format(self.stop_criteria),
color='blue'))
return self.stop_criteria
def _nlp_2_corpus_v2(self, x_np):
pool_res_list = list()
with Pool(NCPU) as pool:
for d in x_np:
res = pool.apply_async(self._nlp_2_corpus_item, d)
pool_res_list.append(res)
info("Note: nlp2corpus mode=2, num={}".format(len(pool_res_list)))
return [r.get() for r in pool_res_list]
def lr_decay(self, epoch):
if self.call_num == 1 or self.cur_lr is None:
self.cur_lr = self.model_manager.lr
if self.train_epoch % 3 == 0 and self.train_epoch > 0:
self.cur_lr = 3 * self.cur_lr / 5
self.cur_lr = max(self.cur_lr, 0.0001)
info(color_msg("recompile lr {}".format(self.cur_lr), color="blue"))
lr = self.cur_lr
return lr
def checkResourcesDir():
log_utils.info("正在检查资源路径,请稍后.....")
# 检查备用资源
dirs = [
root_icon, root_splash, root_badge, output_icon, output_splash,
output_badge
]
for item in dirs:
log_utils.info(file_manager.createFilePath(item))
def run_second_stage(self, remaining_time_budget):
if not self.start_second_stage_model:
self._clear_train_space()
self.start_second_stage_model = True
if self.imbalance_level == 2:
self.second_stage_model.split_val = False
if self.second_stage_model.model_id == len(
self.second_stage_model.cand_models
) and self.second_stage_model.data_id == self.second_stage_model.max_data:
self.second_stage_done = True
info("finish second stage!")
return
if self.second_stage_patience >= self.second_stage_patience_max_num:
self.second_stage_model.epoch = 1
self.second_stage_patience = 0
do_clean = True
else:
do_clean = False
if self.second_stage_model.split_val:
self.second_stage_model.train_iter(
(self.data_manager.meta_data_x, self.data_manager.meta_data_y),
eval_dataset=(self.val_x, self.val_y),
remaining_time_budget=remaining_time_budget,
do_clean=do_clean)
else:
self.second_stage_model.train_iter(
(self.data_manager.meta_train_x,
self.data_manager.meta_train_y),
eval_dataset=(self.val_x, self.val_y),
remaining_time_budget=remaining_time_budget,
do_clean=do_clean)
second_auc = self.second_stage_model.best_sco
self.evaluator.best_auc = second_auc
if second_auc == -1 or second_auc == 0.02:
second_auc = 0.0
if second_auc >= self.best_auc * 0.97 and second_auc > 0.0:
self.use_second_stage_model = True
if self.second_stage_model.Xtest is None and self.second_stage_model.FIRSTROUND:
self.second_stage_model.START = True
elif self.second_stage_model.Xtest is None and self.second_stage_model.new_data:
self.second_stage_model.START = False
return
else:
if self.second_stage_model.START == False and self.second_stage_model.FIRSTROUND == False and self.second_stage_model.LASTROUND:
self.second_stage_model.is_best = False
self.second_stage_model.LASTROUND = False
elif self.second_stage_model.START == True: # 如果START模型没有超过当前
self.second_stage_model.START = False
self.use_second_stage_model = False
return
def get_test_numpy(self):
if self.test_tfds is None:
error("Error: test_tfds is None.")
return self.accum_test_x, self.accum_test_y
if len(self.accum_test_x) == 0:
time_test_np_start = time.time()
tfds_test_os_iterator = self.test_tfds.make_one_shot_iterator()
as_timer("tfds_test_ositer")
tfds_test_iter_next = tfds_test_os_iterator.get_next()
time_test_os_iterator_end = time.time()
info(
"note: now take time_test_os_iterator_end cost_time={}s".format(
round(time_test_os_iterator_end - time_test_np_start, 3)
)
)
with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess:
if self.domain == "text":
while True:
try:
example, labels = sess.run(tfds_test_iter_next)
example = np.squeeze(example, (2, 3))
example = np.squeeze(example, axis=-1)
example = example.astype(np.int)
self.accum_test_x.extend(example)
self.accum_test_y.extend(labels)
self.accm_test_cnt += example.shape[0]
# X.append(example)
# Y.append(labels)
except tf.errors.OutOfRangeError:
break
else:
while True:
try:
example, labels = sess.run(tfds_test_iter_next)
# output: Note:time example shape=(86401, 1, 1, 1)
# logger.info("Note:time example shape={}".format(example.shape))
self.accum_test_x.append(example)
self.accum_test_y.append(labels)
self.accm_test_cnt += 1
except tf.errors.OutOfRangeError:
as_timer("tfds_test_run_OOR_{}".format(self.accm_test_cnt))
break
time_test_np_end = time.time()
info(
"note: now take test accm_test_cnt={}, cost_time={}s".format(
self.accm_test_cnt, round(time_test_np_end - time_test_np_start, 3)
)
)
self.accum_test_y = np.array(self.accum_test_y)
return self.accum_test_x
def make_archiveWithInfo():
"""压缩文件到指定文件夹"""
nowtimestr = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
temp_splash_path = file_manager.createFilePath(
file_manager.getFullPath(Index.temp_splash))
new_path = shutil.make_archive(
file_manager.getFullPath(Index.output_splash) + nowtimestr, 'zip',
temp_splash_path)
log_utils.printf()
log_utils.info("打包结束之后的zip路径为:" + new_path)
def handleThirdPlugins(workDir, decompileDir, game, channel, packageName):
pluginsFolder = file_utils.getFullPath('config/plugin')
gamePluginFolder = file_utils.getFullPath('games/'+game['appName']+'/plugin')
plugins = channel.get('third-plugins')
if plugins == None or len(plugins) <= 0:
log_utils.info("the channel %s has no supported plugins.", channel['name'])
return 0
#copy all resources to temp folder.
for plugin in plugins:
pluginName = plugin['name']
pluginSourceFolder = os.path.join(pluginsFolder, pluginName)
if not os.path.exists(pluginSourceFolder):
log_utils.warning("the plugin %s config folder is not exists", pluginName)
continue
pluginTargetFolder = workDir + "/plugins/" + pluginName
file_utils.copy_files(pluginSourceFolder, pluginTargetFolder)
gamePluginSourceFolder = os.path.join(gamePluginFolder, pluginName)
if not os.path.exists(gamePluginSourceFolder):
log_utils.warning("the plugin %s is not configed in the game %s", pluginName, game['appName'])
continue
file_utils.copy_files(gamePluginSourceFolder, pluginTargetFolder)
if not os.path.exists(pluginSourceFolder + "/classes.dex"):
jar2dex(pluginSourceFolder, pluginTargetFolder)
#handle plugins
smaliDir = os.path.join(decompileDir, "smali")
pluginNum = 0
for plugin in plugins:
pluginName = plugin['name']
pluginFolder = workDir + "/plugins/" + pluginName
if not os.path.exists(pluginFolder):
log_utils.warning("the plugin %s temp folder is not exists", pluginName)
continue
pluginDexFile = os.path.join(pluginFolder, "classes.dex")
ret = dex2smali(pluginDexFile, smaliDir, "baksmali.jar")
if ret:
return 1
ret = copyResource(game, channel, packageName, pluginFolder, decompileDir, plugin['operations'], pluginName, plugin)
if ret:
return 1
pluginNum += 1
log_utils.info("Total plugin num:%s;success handle num:%s", str(len(plugins)), str(pluginNum))
def _nlp_2_corpus_v0(self, x_np):
nlp_x_corpus = []
for x in x_np: # each x in X is a list of indices (but as float)
# fixme: x element?
x = x[x != -1]
tokens = [self.nlp_index_to_token_map[int(i)] for i in x]
document = self.nlp_sep.join(tokens)
nlp_x_corpus.append(document)
info("Note: nlp2corpus mode=0, num={}".format(len(nlp_x_corpus)))
return nlp_x_corpus
def _update_multi_model_result(self, pred):
if self.is_best:
self.test_result_list[self.train_model_id] = pred
result = np.mean(self.test_result_list[:self.train_model_id + 1], axis=0)
else:
info(color_msg("do not ensemble cur model!!!!\n"))
if isinstance(self.test_result_list[self.train_model_id], int):
result = self.test_result_list[0]
else:
result = np.mean(self.test_result_list[:self.train_model_id + 1], axis=0)
return result
def _nlp_2_corpus_v4(self, x_np):
with Pool(NCPU) as pool:
# pool_res_list = pool.map(self.nlp_corpus_worker.nlp_2_corpus_item, x_np)
pool_res_list = pool.map(self.nlp_corpus_worker.nlp_2_corpus_item, x_np)
corpus_res = list()
for res in pool_res_list:
corpus_res.extend(corpus_res)
info("Note: nlp2corpus mode=1, num={}".format(len(pool_res_list)))
# return pool_res_list
return corpus_res
def set_max_vocab_size(self, input_x):
avg_punct_cnt = detect_punctuation(input_x)
avg_upper_cnt, avg_digit_cnt = detect_supper_and_digits(input_x)
info(
"avg_punct_cnt is {} and avg_upper_cnt is {} and avg_digit_cnt is {}"
.format(avg_punct_cnt, avg_upper_cnt, avg_digit_cnt))
if avg_punct_cnt <= 0.02:
Max_Vocab_Size = 30000
else:
Max_Vocab_Size = 20000
self.vocab_size = Max_Vocab_Size
def addSplashScreen(workDir, channel, decompileDir):
"""
if the splash attrib is not zero ,then set the splash activity
if the splash_copy_to_unity attrib is set, then copy the splash img to unity res fold ,replace the default splash.png.
"""
if channel['splash'] =='0':
return 0
if channel['splash_copy_to_unity'] == '1':
return copySplashToUnityResFolder(workDir, channel, decompileDir)
splashPath = file_utils.getSplashPath()
smaliPath = splashPath + "/smali"
smaliTargetPath = decompileDir + "/smali"
copyResToApk(smaliPath, smaliTargetPath)
splashLayoutPath = splashPath + "/u8_splash.xml"
splashTargetPath = decompileDir + "/res/layout/u8_splash.xml"
file_utils.copy_file(splashLayoutPath, splashTargetPath)
resPath = workDir + "/sdk/" + channel['name'] + "/splash/" + channel['splash']
resTargetPath = decompileDir + "/res"
copyResToApk(resPath, resTargetPath)
#remove original launcher activity of the game
activityName = removeStartActivity(decompileDir)
#append the launcher activity with the splash activity
appendSplashActivity(decompileDir, channel['splash'])
splashActivityPath = smaliTargetPath + "/com/u8/sdk/SplashActivity.smali"
f = open(splashActivityPath, 'r+')
content = str(f.read())
f.close()
replaceTxt = '{U8SDK_Game_Activity}'
idx = content.find(replaceTxt)
if idx == -1:
log_utils.error("modify splash file failed.the {U8SDK_Game_Activity} not found in SplashActivity.smali")
return 1
content = content[:idx] + activityName + content[(idx + len(replaceTxt)):]
f2 = open(splashActivityPath, 'w')
f2.write(content)
f2.close()
log_utils.info("modify splash file success.")
return 0
def step_decay(self, epoch):
epoch = self.train_epoch // 3
initial_lrate = self.model_manager.lr # 0.016 #0.0035 #
drop = 0.65 # 0.65
epochs_drop = 1.0 # 2.0
if (self.train_epoch) <= 2:
lrate = initial_lrate
else:
lrate = initial_lrate * math.pow(drop, math.floor((1 + epoch) / epochs_drop))
lrate = max(lrate, 0.0001)
info(color_msg("recompile lr {}".format(lrate), color="blue"))
return lrate
def getpath(dir):
# 2、获取该路径下的所有文件
icons = file_manager.list_files(dir, [], ".DS_Store");
if len(icons) == 0:
log_utils.info("还没有上传appicon资源文件，请上传之后操作！")
return 101
else:
# 如果数组不为空，取出第一个元素
# 获取到icon文件
for filename in icons:
if ".png" in filename:
appIconDir = filename
return appIconDir
def get_nlp_train_dataset_full(self):
self.get_train_numpy_full()
# build nlp corpus.
if self.nlp_train_x_corpus is None:
time_nlp_train_2_corpus_start = time.time()
self.nlp_train_x_corpus = self._nlp_2_corpus(self.accum_train_x)
time_nlp_train_2_corpus_end = time.time()
info(
"note: get_nlp_train_dataset, np2corpus train, num={}, cost_time={}s".format(
len(self.nlp_train_x_corpus), round(time_nlp_train_2_corpus_end - time_nlp_train_2_corpus_start, 3)
)
)
return {"x": self.nlp_train_x_corpus, "y": np.array(self.accum_train_y)}
def get_nlp_test_dataset(self):
self.get_test_numpy()
# build nlp corpus.
if self.nlp_test_x_corpus is None:
time_nlp_test_2_corpus_start = time.time()
self.nlp_test_x_corpus = self._nlp_2_corpus(self.accum_test_x)
time_nlp_test_2_corpus_end = time.time()
info(
"note: get_nlp_test_dataset, np2corpus test, num={}, cost_time={}s".format(
len(self.nlp_test_x_corpus), round(time_nlp_test_2_corpus_end - time_nlp_test_2_corpus_start, 3)
)
)
return self.nlp_test_x_corpus, self.accum_test_x
def init_train_tfds(self, train_tfds, train_num):
if self.train_tfds is None or self.train_num == 0:
self.train_num = train_num
if self.domain == "text":
self.train_tfds = train_tfds.padded_batch(
20,
padded_shapes=([None, 1, 1, 1], [None]),
padding_values=(tf.constant(-1, dtype=tf.float32), tf.constant(-1, dtype=tf.float32)),
)
else:
# if cache:
self.train_tfds = train_tfds.cache()
# self.train_tfds = self.train_tfds.map(lambda x, y: (np.array(x, dtype=np.float16), y), num_parallel_calls=4)
info("note: train_tfds cache.")
def check_early_stop_criteria(self, train_epoch):
# db方案早停条件: 出现k次低于最佳auc的情况: 针对模型比较震荡
early_stop_criteria_1 = self.k >= self.patience or train_epoch > self.max_epoch
# upwind 方案早停条件1: 当前评估auc足够高且训练次数足够大，出现一次下降即停：针对模型后期，避免下降
early_stop_criteria_2 = self.cur_val_auc < self.last_val_auc and self.cur_val_auc > 0.96 and train_epoch > self.max_epoch
# upwind 方案早停条件2: 当前训练次数达到阈值，且连续下降次数达到阈值即停：针对模型难收敛/过拟合
early_stop_criteria_3 = train_epoch >= 5 and self.k >= 2
info(
color_msg("stop criteria 1 is {}, 2 is {}, 3 is {}".format(
early_stop_criteria_1, early_stop_criteria_2,
early_stop_criteria_2),
color='blue'))
return (early_stop_criteria_1 or early_stop_criteria_2
or early_stop_criteria_3)
def signApk(workDir, game, channel, apkfile):
"""
Sign apk
"""
keystore = config_utils.getKeystore(game["appName"], channel["id"])
#如果你想每次打包自动生成一个keystore文件，那么可以启用下面这句代码
#keystore = generateKeystore(workDir, game, channel)
log_utils.info("the keystore file is %s", keystore['keystore'])
signApkInternal(apkfile, keystore['keystore'], keystore['password'],
keystore['aliaskey'], keystore['aliaspwd'],
keystore['sigalg'])
def checkApkForU8SDK(workDir, decompileDir):
"""
检查母包中接入U8SDK抽象层是否正确
不正确，则自动修正
"""
ret = 0
log_utils.info("now to check the u8.apk is correct?")
manifestFile = decompileDir + "/AndroidManifest.xml"
manifestFile = file_utils.getFullPath(manifestFile)
ET.register_namespace('android', androidNS)
tree = ET.parse(manifestFile)
root = tree.getroot()
key = '{'+androidNS+'}name'
applicationNode = root.find('application')
name = applicationNode.get(key)
if not name or name != "com.u8.sdk.U8Application":
log_utils.error("the android:name in application element must be 'com.u8.sdk.U8Application'. now change it to com.u8.sdk.U8Application, but maybe something will be wrong .")
applicationNode.set(key, 'com.u8.sdk.U8Application')
tree.write(manifestFile, 'UTF-8')
smaliName = file_utils.getFullPath(decompileDir + "/smali/com/u8/sdk/U8SDK.smali")
if not os.path.exists(smaliName):
log_utils.error("the u8sdk2.jar is not packaged to the u8.apk. now merge it. but maybe something will be wrong .")
u8sdkJarPath = file_utils.getFullPath('config/local/u8sdk2.jar')
if not os.path.exists(u8sdkJarPath):
log_utils.error("the u8sdk2.jar is not in config/local path. correct failed")
return 1
targetPath = file_utils.getFullPath(workDir + "/local")
if not os.path.exists(targetPath):
os.makedirs(targetPath)
file_utils.copy_file(u8sdkJarPath, targetPath+"/u8sdk2.jar")
jar2dex(targetPath, targetPath)
smaliPath = file_utils.getFullPath(decompileDir + "/smali")
ret = dex2smali(targetPath + '/classes.dex', smaliPath)
# if not ret:
# ret = mergeJar(workDir, decompileDir)
log_utils.info("check u8.apk successfully")
return ret
def copyAppRootResources(game, decompileDir):
"""
Copy game root files to apk. the files will be in the root path of apk
"""
resPath = "games/" + game['appName'] + "/root"
resPath = file_utils.getFullPath(resPath)
if not os.path.exists(resPath):
log_utils.info("the game %s has no root folder", game['appName'])
return
targetResPath = file_utils.getFullPath(decompileDir)
copyResToApk(resPath, targetResPath)
return
def copyChannelResources(game, channel, decompileDir):
"""
Copy channel resources to decompile folder. for example icon resources, assets and so on.
"""
resPath = "games/" + game['appName'] + "/channels/" + channel['id']
resPath = file_utils.getFullPath(resPath)
if not os.path.exists(resPath):
log_utils.warning("the channel %s special res path is not exists. %s", channel['id'], resPath)
return 0
targetResPath = file_utils.getFullPath(decompileDir)
copyResToApk(resPath, targetResPath)
log_utils.info("copy channel %s special res to apk success.", channel['name'])
return 0
def run(self):
while True:
if self.queue.empty():
break
channel = self.queue.get()
ret = core.pack(self.game, channel, self.sourcePath, self.isPublic)
if ret:
self.failNum = self.failNum + 1
else:
self.sucNum = self.sucNum + 1
self.queue.task_done()
log_utils.info("Thread-%s:sucNum:%s;failNum:%s",self.threadIndex,str(self.sucNum), str(self.failNum))
return
def doGamePostScript(game, channel, decompileDir, packageName):
scriptDir = file_utils.getFullPath("games/"+game['appName']+"/scripts")
if not os.path.exists(scriptDir):
log_utils.info("the game post script is not exists. if you have some specail logic, you can do it in games/[yourgame]/scripts/post_script.py")
return 0
sdkScript = os.path.join(scriptDir, "post_script.py")
if not os.path.exists(sdkScript):
log_utils.info("the game post script is not exists. if you have some specail logic, you can do it in games/[yourgame]/scripts/post_script.py")
return 0
sys.path.append(scriptDir)
import post_script
log_utils.info("now to execute post_script.py of game %s ", game['appName'])
ret = post_script.execute(game, channel, decompileDir, packageName)
del sys.modules['post_script']
sys.path.remove(scriptDir)
return ret
def main(game, isPublic, target):
appName = game['appName']
channels = config_utils.getAllChannels(appName, isPublic)
if channels is None or len(channels) == 0:
print("没有任何可以打包的渠道")
return 3
selected = []
if target == '*':
selected = channels
else:
for t in target.split(','):
t = t.strip()
matchChannels = [c for c in channels if c['name'].lower() == t.lower()]
if len(matchChannels) > 0:
selected.append(matchChannels[0])
clen = len(selected)
log_utils.info("now hava %s channels to package...", clen)
baseApkPath = file_utils.getFullPath('games/'+game['appName']+'/u8.apk')
log_utils.info("the base apk file is : %s", baseApkPath)
if not os.path.exists(baseApkPath):
log_utils.error('the base apk file is not exists, must named with u8.apk')
return 2
sucNum = 0
failNum = 0
for channel in selected:
ret = core.pack(game, channel, baseApkPath, isPublic)
if ret:
exit(1)
failNum = failNum + 1
else:
sucNum = sucNum + 1
log_utils.info("<< success num:%s; failed num:%s >>", sucNum, failNum)
if failNum > 0:
log_utils.error("<< all done with error >>")
else:
log_utils.info("<< all nice done >>")
return 0
def execFormatCmd(cmd):
cmd = cmd.replace('\\', '/')
cmd = re.sub('/+', '/', cmd)
ret = 0
try:
reload(sys)
sys.setdefaultencoding('utf-8')
if platform.system() == 'Windows':
st = subprocess.STARTUPINFO
st.dwFlags = subprocess.STARTF_USESHOWWINDOW
st.wShowWindow = subprocess.SW_HIDE
# cmd = str(cmd).encode('gbk')
s = subprocess.Popen(cmd, shell=True)
ret = s.wait()
if ret:
s = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
stdoutput, erroutput = s.communicate()
log_utils.error("*******ERROR*******")
log_utils.error(stdoutput)
log_utils.error(erroutput)
log_utils.error("*******************")
cmd = 'error::' + cmd + ' !!!exec Fail!!! '
else:
s = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
stdoutput, erroutput = s.communicate()
log_utils.info(stdoutput)
log_utils.info(erroutput)
cmd = cmd + ' !!!exec success!!! '
log_utils.info(cmd)
except Exception as e:
print(e)
return
return ret
def modifyGameName(channel, decompileDir):
"""
修改当前渠道的游戏名称,如果某个渠道的游戏名称特殊，可以配置gameName来指定。默认就是母包中游戏的名称
"""
log_utils.info("now to modify game name ....")
if 'gameName' not in channel:
log_utils.info("now no game name modify")
return
manifestFile = decompileDir + "/AndroidManifest.xml"
manifestFile = file_utils.getFullPath(manifestFile)
ET.register_namespace('android', androidNS)
tree = ET.parse(manifestFile)
root = tree.getroot()
labelKey = '{'+androidNS+'}label'
applicationNode = root.find('application')
applicationNode.set(labelKey, channel['gameName'])
log_utils.info("the new game name is " + channel['gameName'])
tree.write(manifestFile, 'UTF-8')
def main(game, isPublic, isFullRes = False):
print(u"**********当前配置的渠道号**********")
print(u"\t渠道名 \t\t 渠道号 \t\t 渠道 \n")
appName = game['appName']
channels = config_utils.getAllChannels(appName, isPublic)
if channels is None or len(channels) == 0:
print("没有任何可以打包的渠道")
return
for ch in channels:
name = ch['name']
if len(name) <= 6:
chStr = u"\t%s \t\t\t %s \t\t %s " % (ch['name'], ch['id'], ch['desc'])
elif len(name) > 6 and len(name) <= 13:
chStr = u"\t%s \t\t %s \t\t %s " % (ch['name'], ch['id'], ch['desc'])
else:
chStr = u"\t%s \t %s \t\t %s " % (ch['name'], ch['id'], ch['desc'])
print(chStr)
selected = []
while(True):
sys.stdout.write(u"请选择需要打包的渠道(渠道名),全部输入*,多个用逗号分割：")
sys.stdout.flush()
target = raw_input()
if target == '*':
selected = channels
else:
for t in target.split(','):
t = t.strip()
matchChannels = [c for c in channels if c['name'].lower() == t.lower()]
if len(matchChannels) > 0:
selected.append(matchChannels[0])
if len(selected) == 0:
print(u"\n无效的渠道名，请重新输入！！\n")
else:
break
clen = len(selected)
log_utils.info("now hava %s channels to package...", clen)
baseApkPath = file_utils.getFullPath('games/'+game['appName']+'/u8.apk')
log_utils.info("the base apk file is : %s", baseApkPath)
if not os.path.exists(baseApkPath):
log_utils.error('the base apk file is not exists, must named with u8.apk')
return
sucNum = 0
failNum = 0
for channel in selected:
ret = core.pack(game, channel, baseApkPath, isPublic)
if ret:
failNum = failNum + 1
else:
sucNum = sucNum + 1
log_utils.info("<< success num:%s; failed num:%s >>", sucNum, failNum)
if failNum > 0:
log_utils.error("<< all done with error >>")
else:
log_utils.info("<< all nice done >>")
def pack(game, channel, sourcepath, isPublic):
sourcepath = sourcepath.replace("\\", "/")
if not os.path.exists(sourcepath):
return 1
appID = game["appID"]
appKey = game["appKey"]
appName = game["appName"]
channelId = channel["id"]
channelName = channel["name"]
sdkName = channel["sdk"]
log_utils.info("now to package %s...", channelName)
workDir = "workspace/" + appName + "/" + channelName
workDir = file_utils.getFullPath(workDir)
file_utils.del_file_folder(workDir)
tempApkSource = workDir + "/temp.apk"
file_utils.copy_file(sourcepath, tempApkSource)
decompileDir = workDir + "/decompile"
ret = apk_utils.decompileApk(tempApkSource, decompileDir)
if ret:
return 1
# 检查母包接入是否正确
# ret = apk_utils.checkApkForU8SDK(workDir, decompileDir)
# if ret:
# return 1
# copy sdk code to decompileDir
sdkSourceDir = file_utils.getFullPath("config/sdk/" + sdkName)
smaliDir = decompileDir + "/smali"
sdkDestDir = workDir + "/sdk/" + sdkName
file_utils.copy_files(sdkSourceDir, sdkDestDir)
if not os.path.exists(sdkSourceDir + "/classes.dex"):
apk_utils.jar2dex(sdkSourceDir, sdkDestDir)
sdkDexFile = sdkDestDir + "/classes.dex"
ret = apk_utils.dex2smali(sdkDexFile, smaliDir, "baksmali.jar")
if ret:
return 1
# change xml config and so on
newPackageName = apk_utils.renamePackageName(channel, decompileDir, channel["suffix"], isPublic)
# copy third-plugins resources. note:The third plugins must be operated before the main sdk.
ret = apk_utils.handleThirdPlugins(workDir, decompileDir, game, channel, newPackageName)
if ret:
return 1
# copy main sdk resources.
ret = apk_utils.copyResource(
game, channel, newPackageName, sdkDestDir, decompileDir, channel["operations"], channelName
)
if ret:
return 1
# copy channel special resources
ret = apk_utils.copyChannelResources(game, channel, decompileDir)
if ret:
return 1
# copy game root resources and res resources
apk_utils.copyAppResources(game, decompileDir)
apk_utils.copyAppRootResources(game, decompileDir)
apk_utils.appendChannelIconMark(game, channel, decompileDir)
# generate config files for apk to run.
apk_utils.writeDevelopInfo(appID, appKey, channel, decompileDir)
apk_utils.writePluginInfo(channel, decompileDir)
apk_utils.writeManifestMetaInfo(channel, decompileDir)
# if the main sdk has special logic. execute the special logic script.
ret = apk_utils.doSDKScript(channel, decompileDir, newPackageName, sdkDestDir)
if ret:
return 1
# if the game has some special logic. execute the special logic script.called post_script.py
ret = apk_utils.doGamePostScript(game, channel, decompileDir, newPackageName)
if ret:
return 1
# here to config the splash screen.
ret = apk_utils.addSplashScreen(workDir, channel, decompileDir)
if ret:
return 1
# check cpu supports
apk_utils.checkCpuSupport(game, decompileDir)
# modify game name if channel specified.
apk_utils.modifyGameName(channel, decompileDir)
# generate new R.java
ret = apk_utils.generateNewRFile(newPackageName, decompileDir)
if ret:
return 1
targetApk = workDir + "/output.apk"
ret = apk_utils.recompileApk(decompileDir, targetApk)
if ret:
return 1
apk_utils.copyRootResFiles(targetApk, decompileDir)
ret = apk_utils.signApk(appName, channelId, targetApk)
if ret:
return 1
# destApkName = channelName + '.apk'
channelNameStr = channelName.replace(" ", "")
if isPublic:
destApkName = channelNameStr + "-" + time.strftime("%Y%m%d%H") + ".apk"
else:
destApkName = channelNameStr + "-" + time.strftime("%Y%m%d%H") + "-debug.apk"
destApkPath = file_utils.getFullOutputPath(appName, channelName)
destApkPath = os.path.join(destApkPath, destApkName)
ret = apk_utils.alignApk(targetApk, destApkPath)
if ret:
return 1
# clear workspace
# file_utils.del_file_folder(workDir)
log_utils.info("channel %s package success.", channelName)
return 0
def writeManifestMetaInfo(channel, decompileDir):
manifestFile = decompileDir + "/AndroidManifest.xml"
manifestFile = file_utils.getFullPath(manifestFile)
ET.register_namespace('android', androidNS)
tree = ET.parse(manifestFile)
root = tree.getroot()
key = '{'+androidNS+'}name'
val = '{'+androidNS+'}value'
appNode = root.find('application')
if appNode is None:
return
metaDataList = appNode.findall('meta-data')
if metaDataList != None:
for metaDataNode in metaDataList:
keyName = metaDataNode.attrib[key]
for child in channel['params']:
if keyName == child['name'] and child['bWriteInManifest'] == '1':
log_utils.warning("the meta-data node %s repeated. remove it .", keyName)
appNode.remove(metaDataNode)
if 'third-plugins' in channel and channel['third-plugins'] != None and len(channel['third-plugins']) > 0:
for cPlugin in channel['third-plugins']:
if 'params' in cPlugin and cPlugin['params'] != None and len(cPlugin['params']) > 0:
for child in cPlugin['params']:
if keyName == child['name'] and child['bWriteInManifest'] == '1':
log_utils.warning("the meta-data node %s repeated. remove it .", keyName)
appNode.remove(metaDataNode)
for child in channel['params']:
if child['bWriteInManifest'] != None and child['bWriteInManifest'] == '1':
metaNode = SubElement(appNode, 'meta-data')
metaNode.set(key, child['name'])
metaNode.set(val, child['value'])
if 'third-plugins' in channel and channel['third-plugins'] != None and len(channel['third-plugins']) > 0:
for cPlugin in channel['third-plugins']:
if 'params' in cPlugin and cPlugin['params'] != None and len(cPlugin['params']) > 0:
for child in cPlugin['params']:
if child['bWriteInManifest'] != None and child['bWriteInManifest'] == '1':
metaNode = SubElement(appNode, 'meta-data')
metaNode.set(key, child['name'])
metaNode.set(val, child['value'])
if 'U8_APPLICATION_PROXY_NAME' in channel:
metaNode = SubElement(appNode, 'meta-data')
metaNode.set(key, "U8_APPLICATION_PROXY_NAME")
metaNode.set(val, channel['U8_APPLICATION_PROXY_NAME'])
#log_utils.info(ET.tostring(root,encoding="us-ascii", method="text"))
tree.write(manifestFile, 'UTF-8')
log_utils.info("The manifestFile meta-data write successfully")
def packAllChannels(game, isPublic, threadNum):
basePath = file_utils.getCurrDir()
log_utils.info("Curr Work Dir::%s", basePath)
appName = game['appName']
channels = config_utils.getAllChannels(appName, isPublic)
if channels != None and len(channels) > 0:
clen = len(channels)
log_utils.info("Now Have %s channels to package ", clen)
packagePath = file_utils.getFullPath("u8.apk")
log_utils.info("The base apk file is : %s", packagePath)
if not os.path.exists(packagePath):
log_utils.error("The apk file name must be 'u8.apk'")
return
que = queue.Queue()
for channel in channels:
que.put(channel, True, None)
# start threads to pack
if threadNum <= 0:
threadNum = 1
log_utils.info("Now start %s threads to pack", threadNum)
for i in range(threadNum):
thread = PackerThread(que, packagePath, isPublic, game, i+1)
thread.start()
que.join()
log_utils.info("<< all nice done >>")
else:
log_utils.info("<< no channels to pack >>")
def copyResource(game, channel, packageName, sdkDir, decompileDir , operations, name, pluginInfo = None):
"""
Copy sdk resources to the apk decompile dir
Merge manifest.xml
Merge all res xml if the xml already exists in target apk.
copy all others resources
"""
if operations != None:
for child in operations:
if child['type'] == 'mergeManifest':
manifestFrom = file_utils.getFullPath(os.path.join(sdkDir, child['from']))
manifestFromTemp = manifestFrom
manifestTo = file_utils.getFullPath(os.path.join(decompileDir, child['to']))
if 'orientation' in game:
if game['orientation'] == 'portrait':
manifestFrom = manifestFrom[:-4] + "_portrait.xml"
else:
manifestFrom = manifestFrom[:-4] + "_landscape.xml"
if not os.path.exists(manifestFrom):
manifestFrom = manifestFromTemp
log_utils.info("The sdk manifest file is %s", manifestFrom)
#merge into xml
bRet = mergeManifest(channel, manifestTo, manifestFrom)
if bRet:
log_utils.info("merge manifest file success.")
else:
log_utils.error("merge manifest file failed.")
return 1
elif child['type'] == 'copyRes':
if child['from'] == None or child['to'] == None:
log_utils.error("the sdk config file error. 'copyRes' need 'from' and 'to'.sdk name:%s", name)
return 1
copyFrom = file_utils.getFullPath(os.path.join(sdkDir, child['from']))
copyTo = file_utils.getFullPath(os.path.join(decompileDir, child['to']))
if child['to'] == 'lib':
copyLibs(game, copyFrom, copyTo)
else:
copyResToApk(copyFrom, copyTo)
elif child['type'] == 'script' and pluginInfo != None:
#now only third-plugin support script
if child['from'] == None:
log_utils.error("the sdk config file is error. 'script' need 'from' attrib to specify script.py")
return 1
scriptName = child['from']
log_utils.info("now to execute plugin script. name:%s", scriptName)
doScript(channel, pluginInfo, decompileDir, packageName, sdkDir, scriptName)
return 0
def renamePackageName(channel, decompileDir, newPackageName, isPublic = True):
"""
Rename package name to the new name configed in the channel
"""
manifestFile = decompileDir + "/AndroidManifest.xml"
manifestFile = file_utils.getFullPath(manifestFile)
ET.register_namespace('android', androidNS)
tree = ET.parse(manifestFile)
root = tree.getroot()
package = root.attrib.get('package')
oldPackageName = package
tempPackageName = newPackageName
if not isPublic:
newPackageName = oldPackageName + ".debug"
if tempPackageName != None and len(tempPackageName) > 0:
if tempPackageName[0:1] == '.':
if not isPublic:
newPackageName = oldPackageName + ".debug" + tempPackageName
else:
newPackageName = oldPackageName + tempPackageName
else:
newPackageName = tempPackageName
if newPackageName == None or len(newPackageName) <= 0:
newPackageName = oldPackageName
log_utils.info("the new package name is %s", newPackageName)
#now to check activity or service
appNode = root.find('application')
if appNode != None:
#now to config icon if icon configed.
# if 'icon' in channel and channel['icon'] != None:
# iconKey = '{'+androidNS+'}icon'
# iconVal = '@drawable/' + channel['icon']
# appNode.set(iconKey, iconVal)
activityLst = appNode.findall('activity')
key = '{'+androidNS+'}name'
if activityLst != None and len(activityLst) > 0:
for aNode in activityLst:
activityName = aNode.attrib[key]
if activityName[0:1] == '.':
activityName = oldPackageName + activityName
elif activityName.find('.') == -1:
activityName = oldPackageName + '.' + activityName
aNode.attrib[key] = activityName
serviceLst = appNode.findall('service')
#key = '{'+androidNS+'}name'
if serviceLst != None and len(serviceLst) > 0:
for sNode in serviceLst:
serviceName = sNode.attrib[key]
if serviceName[0:1] == '.':
serviceName = oldPackageName + serviceName
elif serviceName.find('.') == -1:
serviceName = oldPackageName + '.' + serviceName
sNode.attrib[key] = serviceName
receiverLst = appNode.findall('receiver')
#key = '{'+androidNS+'}name'
if receiverLst != None and len(receiverLst) > 0:
for sNode in receiverLst:
receiverName = sNode.attrib[key]
if receiverName[0:1] == '.':
receiverName = oldPackageName + receiverName
elif receiverName.find('.') == -1:
receiverName = oldPackageName + '.' + receiverName
sNode.attrib[key] = receiverName
providerLst = appNode.findall('provider')
#key = '{'+androidNS+'}name'
if providerLst != None and len(providerLst) > 0:
for sNode in providerLst:
providerName = sNode.attrib[key]
if providerName[0:1] == '.':
providerName = oldPackageName + providerName
elif providerName.find('.') == -1:
providerName = oldPackageName + '.' + providerName
sNode.attrib[key] = providerName
root.attrib['package'] = newPackageName
tree.write(manifestFile, 'UTF-8')
package = newPackageName
return package