def handleAutoResDefined(decompileDir, packageName):
resPath = decompileDir + "/res"
files = []
file_utils.list_files(resPath, files, [])
for f in files:
if (not f.endswith(".xml")):
continue
file_utils.modifyFileContent(f, 'http://schemas.android.com/apk/res-auto',
'http://schemas.android.com/apk/lib/' + packageName)
def merge_res(self, targetRes):
if not os.path.exists(self.aarDir):
log_utils.warning("the aar is not unarchived:" + self.aarFile)
return False
resPath = os.path.join(self.aarDir, "res")
if not os.path.exists(resPath):
self.needR = False
log_utils.debug(
"aar res merge completed. there is no res folder in " +
self.aarFile)
return True
resFiles = file_utils.list_files(resPath, [], [])
if len(resFiles) == 0:
self.needR = False
log_utils.debug(
"aar res merge completed. there is no res file in " +
self.aarFile)
return True
resPaths = [resPath, targetRes]
ResourceMerger2.merge(resPaths)
log_utils.debug("res in aar " + self.aarFile + " merged into " +
targetRes)
return True
def copyRootResFiles(apkfile, decompileDir):
apkfile = file_utils.getFullPath(apkfile)
aapt = file_utils.getFullToolPath("aapt")
decompileDir = file_utils.getFullPath(decompileDir)
igoreFiles = ['AndroidManifest.xml','apktool.yml', 'smali', 'res', 'original','lib','build','assets','unknown']
igoreFileFullPaths = []
for ifile in igoreFiles:
fullpath = os.path.join(decompileDir, ifile)
igoreFileFullPaths.append(fullpath)
addFiles = []
addFiles = file_utils.list_files(decompileDir, addFiles, igoreFileFullPaths)
if len(addFiles) <= 0:
return
addCmd = '"%s" add "%s"'
for f in addFiles:
fname = f[(len(decompileDir)+1):]
addCmd = addCmd + ' ' + fname
addCmd = addCmd % (aapt, apkfile)
currPath = os.getcwd()
os.chdir(decompileDir)
file_utils.execFormatCmd(addCmd)
os.chdir(currPath)
def __init__(self, args):
super(CRAFTDataset, self).__init__()
filelist, _, _ = file_utils.list_files(
'/home/ubuntu/Kyumin/Autotation/data/IC13/images')
self.args = args
self.labels = []
self.images = filelist
for img_name in filelist:
# get datapath
label_dir = img_name.replace('Autotation', 'craft').replace(
'images', 'labels').replace('.jpg', '/')
self.labels.append(label_dir)
def __init__(self, args):
filelist, _, _ = file_utils.list_files('./data/train/data')
self.images = []
self.confmaps = []
self.scores_region = []
self.scores_link = []
for filename in filelist:
# get datapath
dataset = os.path.dirname(filename).split(os.sep)[-1]
filenum = os.path.splitext(os.path.basename(filename))
label_dir = './data/train/ground_truth/{}/gt_{}/'.format(
dataset, filenum)
# If not exists, generate ground truth
if not os.path.exists(label_dir):
continue
image = imgproc.loadImage(filename)
score_region = torch.load(label_dir + 'region.pt')
score_link = torch.load(label_dir + 'link.pt')
conf_map = torch.load(label_dir + 'conf.pt')
# resize
img_resized, target_ratio, size_heatmap = imgproc.resize_aspect_ratio(
image,
args.canvas_size,
interpolation=cv2.INTER_LINEAR,
mag_ratio=args.mag_ratio)
# Image Preprocess
x = imgproc.normalizeMeanVariance(img_resized)
x = x.transpose((2, 0, 1)) # [h, w, c] to [c, h, w]
h, w, _ = img_resized.shape
# GT reshape
score_region = cv2.resize(score_region, dsize=(h / 2, w / 2))
score_link = cv2.resize(score_link, dsize=(h / 2, w / 2))
conf_map = cv2.resize(conf_map, dsize=(h / 2, w / 2))
self.scores_region.append(score_region)
self.scores_link.append(score_link)
self.confmaps.append(conf_map)
self.images.append(x)
def crawl():
results = []
queries = []
loop = asyncio.get_event_loop()
for query in list_files(join(basedir, queries_dir),
filter_by_extensions=['json']):
queries.append(query)
for query in queries:
urls = query.get('urls', [])
while len(urls) > 0:
futures = []
urls_slice = urls[:500]
del urls[:500]
for url in urls_slice:
crawler = Crawler(url, query.get("queries", {}))
futures.append(crawler.crawl_async())
results = loop.run_until_complete(asyncio.gather(*futures))
json.dump(results, sys.stdout, indent=2)
loop.close()
def dexes2smali(dexDir, targetdir, dextool="baksmali.jar"):
"""
Transfer all dex in dexDir to smali
"""
if not os.path.exists(dexDir):
log_utils.error("the dexDir is not exists:" + dexDir)
return 1
files = file_utils.list_files(dexDir, [], [])
for f in files:
if not f.endswith(".dex"):
continue
ret = dex2smali(f, targetdir, dextool)
if not ret:
return False
return True
def ground_truth(args):
# initiate pretrained network
net = CRAFT() # initialize
print('Loading weights from checkpoint (' + args.trained_model + ')')
if args.cuda:
net.load_state_dict(test.copyStateDict(torch.load(args.trained_model)))
else:
net.load_state_dict(test.copyStateDict(torch.load(args.trained_model, map_location='cpu')))
if args.cuda:
net = net.cuda()
net = torch.nn.DataParallel(net)
cudnn.benchmark = False
net.eval()
filelist, _, _ = file_utils.list_files('/home/ubuntu/Kyumin/Autotation/data/IC13/images')
for img_name in filelist:
# get datapath
if 'train' in img_name:
label_name = img_name.replace('images/train/', 'labels/train/gt_').replace('jpg', 'txt')
else:
label_name = img_name.replace('images/test/', 'labels/test/gt_').replace('jpg', 'txt')
label_dir = img_name.replace('Autotation', 'craft').replace('images', 'labels').replace('.jpg', '/')
os.makedirs(label_dir, exist_ok=True)
image = imgproc.loadImage(img_name)
gt_boxes = []
gt_words = []
with open(label_name, 'r', encoding='utf-8-sig') as f:
lines = f.readlines()
for line in lines:
if 'IC13' in img_name: # IC13
gt_box, gt_word, _ = line.split('"')
if 'train' in img_name:
x1, y1, x2, y2 = [int(a) for a in gt_box.strip().split(' ')]
else:
x1, y1, x2, y2 = [int(a.strip()) for a in gt_box.split(',') if a.strip().isdigit()]
gt_boxes.append(np.array([[x1, y1], [x2, y1], [x2, y2], [x1, y2]]))
gt_words.append(gt_word)
elif 'IC15' in img_name:
gt_data = line.strip().split(',')
gt_box = gt_data[:8]
if len(gt_data) > 9:
gt_word = ','.join(gt_data[8:])
else:
gt_word = gt_data[-1]
gt_box = [int(a) for a in gt_box]
gt_box = np.reshape(np.array(gt_box), (4, 2))
gt_boxes.append(gt_box)
gt_words.append(gt_word)
score_region, score_link, conf_map = generate_gt(net, image, gt_boxes, gt_words, args)
torch.save(score_region, label_dir + 'region.pt')
torch.save(score_link, label_dir + 'link.pt')
torch.save(conf_map, label_dir + 'conf.pt')
parser.add_argument('--batch-size',help='prediction batch size',type=int,default=8)
parser.add_argument('--nfiles',help='processes only the x first files',type=int,default=-1)
parser.add_argument('--nmatches',help='max final number of matches per reference image',type=int,default=20)
parser.add_argument('--sort-best',help='final ordering is best top match first',action='store_true')
parser.add_argument('--website',help='generates splash.json in addition to a version of matches without candidates',action='store_true')
parser.add_argument('--medium',help='filters out all photographs and undefined medium in art from Tate',action='store_true')
parser.add_argument('--no-tga',help='filter out images from TGA archive',action='store_true')
parser.add_argument('--last-hour',help='select images from last hour',default=-1,type=int)
parser.add_argument('--freq-filter',help='artwork daily frequency filtering database, if any',default='',type=str)
parser.add_argument('--event-filter',help='whether to filter events, one match per event',action='store_true')
parser.add_argument('--concat',help='concat results with those of existing file',action='store_true')
parser.add_argument('--gallery-output',help='JSON gallery output',default='gallery.json',type=str)
parser.add_argument('--splash-output',help='JSON splash output',default='splash.json',type=str)
args = parser.parse_args()
image_files = list_files(args.input_imgs,ext='.JPG',nfiles=args.nfiles,pattern='*_2_*',last_hour=args.last_hour)
json_files = list_files(args.input_imgs,ext='.json',nfiles=args.nfiles)
xml_files = list_files(args.input_imgs,ext='.XML',nfiles=-1,pattern='*')
json_mapi_files = json_mapi_emo_files = []
if 'mapi' in args.generators or 'all' in args.generators:
try:
json_mapi_files = list_files(args.input_imgs + '/mapi/',ext='.json')
json_mapi_emo_files = list_files(args.input_imgs + '/mapi_emo/',ext='.json')
print 'json_mapi_files=',len(json_mapi_files)
print 'json_mapi_emo_files=',len(json_mapi_emo_files)
except:
pass
if len(image_files) == 0:
print 'no new images'
sys.exit(0)
def delete_aibei_res(decompileDir):
"""
remove aibei resources in base apk
"""
filelist = [
"colors", "dimens", "ids", "public", "strings", "styles", "values"
]
for f in filelist:
fpath = decompileDir + "/res/values/" + f + ".xml"
if os.path.exists(fpath):
tree = ET.parse(fpath)
root = tree.getroot()
for node in list(root):
item = {}
attribName = node.attrib.get('name')
if attribName is None:
continue
if attribName.lower().startswith("ipay_"):
root.remove(node)
log_utils.debug("remove debug res index name:" +
attribName + " from" + fpath)
elif attribName.lower() == 'custom_dialog':
root.remove(node)
log_utils.debug("remove debug res index name:" +
attribName + " from" + fpath)
tree.write(fpath, "UTF-8")
filelist = ["ab_login_values.xml", "ab_pay_values.xml"]
for f in filelist:
path = os.path.join(decompileDir, 'res/values', f)
file_utils.del_file_folder(path)
log_utils.debug("remove debug res file:" + path)
resPath = decompileDir + "/res"
filelist = file_utils.list_files(resPath, [], [])
for f in filelist:
if os.path.basename(f).lower().startswith("ipay_"):
file_utils.del_file_folder(f)
log_utils.debug("remove debug res file:" + f)
devConfig = decompileDir + "/assets/u8_developer_config.properties"
pluginConfig = decompileDir + "/assets/u8_plugin_config.xml"
permissionConfig = decompileDir + "/assets/u8_permissions.xml"
file_utils.del_file_folder(devConfig)
file_utils.del_file_folder(pluginConfig)
file_utils.del_file_folder(permissionConfig)
smaliPath = decompileDir + "smali"
smaliIndex = 1
otherSmaliFolder = "smali"
while os.path.exists(os.path.join(decompileDir, otherSmaliFolder)):
#check class
dels = [
"com/u8/sdk/ABSDK", "com/u8/sdk/ABUser", "com/u8/sdk/ABPay",
"com/iapppay/", "com/alipay/", "com/ta/utdid2/", "com/ut/device/",
"org/json/alipay/", "org/apache/commons/", "org/apache/http/"
]
for toDel in dels:
file_utils.del_file_folder(
os.path.join(decompileDir, otherSmaliFolder, toDel))
log_utils.debug("remove debug res :" + toDel)
smaliIndex = smaliIndex + 1
otherSmaliFolder = "smali_classes" + str(smaliIndex)
manifestFile = decompileDir + "/AndroidManifest.xml"
ET.register_namespace('android', androidNS)
tree = ET.parse(manifestFile)
root = tree.getroot()
permissionDels = [
"android.permission.ACCESS_NETWORK_STATE",
"android.permission.CHANGE_NETWORK_STATE",
"android.permission.ACCESS_WIFI_STATE",
"android.permission.ACCESS_FINE_LOCATION",
"android.permission.ACCESS_COARSE_LOCATION",
"android.permission.READ_SMS", "android.permission.WRITE_SMS",
"android.permission.RECEIVE_SMS", "android.permission.SEND_SMS",
"android.permission.MOUNT_UNMOUNT_FILESYSTEMS",
"android.permission.SYSTEM_ALERT_WINDOW"
]
componentDes = [
"com.iapppay.sdk.main.WebActivity",
"com.iapppay.ui.activity.PersonCenterActivity",
"com.iapppay.ui.activity.PayHubActivity",
"com.iapppay.ui.activity.SelectAmountActivity",
"com.iapppay.ui.activity.ChargeActivity",
"com.iapppay.ui.card.GamepayActivity",
"com.alipay.sdk.app.H5PayActivity", "com.alipay.sdk.auth.AuthActivity",
"com.iapppay.openid.channel.ui.LoginActivity",
"com.iapppay.openid.channel.ui.RegisterActivity",
"com.iapppay.openid.channel.ui.OneRegisterActivity",
"com.iapppay.openid.channel.ui.OpenIdBaseActivity",
"com.iapppay.openid.channel.ui.BindPhoneActivity",
"com.iapppay.openid.channel.ui.NameAuthActivity",
"com.iapppay.openid.channel.ui.FindPasswordActivity",
"com.iapppay.openid.channel.ui.ModifyUserNameActivity",
"com.iapppay.openid.channel.ui.ModifyPasswordActivity",
"com.iapppay.openid.channel.ui.SettingCenterActivity",
"com.iapppay.openid.channel.ui.UnbindPhoneActivity",
"com.iapppay.openid.channel.ui.CommonWebActivity"
]
key = '{' + androidNS + '}name'
permissionLst = root.findall('uses-permission')
if permissionLst != None and len(permissionLst) > 0:
for aNode in permissionLst:
permissionName = aNode.attrib[key]
if permissionName in permissionDels:
root.remove(aNode)
log_utils.debug("remove debug permission:" + permissionName)
permission23 = root.find('uses-permission-sdk-23')
if permission23 != None:
root.remove(permission23)
log_utils.debug("remove debug permission:" + permission23.attrib[key])
appNode = root.find('application')
if appNode != None:
activityLst = appNode.findall('activity')
if activityLst != None and len(activityLst) > 0:
for aNode in activityLst:
activityName = aNode.attrib[key]
if activityName in componentDes:
appNode.remove(aNode)
log_utils.debug("remove debug activity:" + activityName)
tree.write(manifestFile, "UTF-8")