def main(opts):
# parser = build_parser()
# opts = parser.parse_args()
# check_opts(opts)
# Execute for single file
opts.in_path = opts.resized_path # Replace original input path with resized path
if not os.path.isdir(opts.in_path):
if os.path.exists(opts.out_path) and os.path.isdir(opts.out_path):
out_path = \
os.path.join(opts.out_path,os.path.basename(opts.in_path))
else:
out_path = opts.out_path
ffwd_to_img(opts.in_path,
out_path,
opts.checkpoint_dir,
device=opts.device)
else: # Execute for all files in directory
files = list_files(opts.in_path)
full_in = [os.path.join(opts.in_path, x) for x in files]
full_out = [os.path.join(opts.out_path, x) for x in files]
if opts.allow_different_dimensions:
ffwd_different_dimensions(full_in,
full_out,
opts.checkpoint_dir,
device_t=opts.device,
batch_size=opts.batch_size)
else:
ffwd(full_in,
full_out,
opts.checkpoint_dir,
device_t=opts.device,
batch_size=opts.batch_size)
def process_video(video_in, checkpoint, video_out):
in_dir = os.path.join(TMP_DIR, 'in')
out_dir = os.path.join(TMP_DIR, 'out')
if not os.path.exists(in_dir):
os.makedirs(in_dir)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
in_args = [
'ffmpeg',
'-i', video_in,
'%s/frame_%%d.png' % in_dir
]
subprocess.call(" ".join(in_args), shell=True)
base_names = list_files(in_dir)
in_files = [os.path.join(in_dir, x) for x in base_names]
out_files = [os.path.join(out_dir, x) for x in base_names]
evaluate.ffwd(in_files, out_files, checkpoint, device_t=DEVICE,
batch_size=BATCH_SIZE)
fr = 30 # wtf
out_args = [
'ffmpeg',
'-i', '%s/frame_%%d.png' % out_dir,
'-f', 'mp4',
'-q:v', '0',
'-vcodec', 'mpeg4',
'-r', str(fr),
video_out
]
subprocess.call(" ".join(out_args), shell=True)
print('Video at: %s' % video_out)
shutil.rmtree(TMP_DIR)
def main():
parser = build_parser()
opts = parser.parse_args()
check_opts(opts)
if not os.path.isdir(opts.in_path):
if os.path.exists(opts.out_path) and os.path.isdir(opts.out_path):
out_path = os.path.join(opts.out_path,
os.path.basename(opts.in_path))
else:
out_path = opts.out_path
ffwd_to_img(opts.in_path,
out_path,
opts.checkpoint_dir,
device=opts.device)
else:
files = list_files(opts.in_path)
full_in = map(lambda x: os.path.join(opts.in_path, x), files)
full_out = map(lambda x: os.path.join(opts.out_path, x), files)
ffwd(full_in,
full_out,
opts.checkpoint_dir,
device_t=opts.device,
batch_size=opts.batch_size)
def main():
parser = build_parser()
opts = parser.parse_args()
check_opts(opts)
if not os.path.isdir(opts.in_path): #单个图片
if os.path.exists(opts.out_path) and os.path.isdir(
opts.out_path): #out_path是文件夹
out_path = \
os.path.join(opts.out_path, os.path.basename(opts.in_path))#xx/xx.jpg
else:
out_path = opts.out_path #可以自定义name:xx/xx.jpg
print("!!!!!!" + opts.in_path, out_path, opts.checkpoint_dir,
opts.device)
ffwd_to_img(opts.in_path,
out_path,
opts.checkpoint_dir,
device=opts.device)
else: #in_path是文件夹，一群图片
files = list_files(opts.in_path)
full_in = [os.path.join(opts.in_path, x) for x in files]
full_out = [os.path.join(opts.out_path, x) for x in files]
if opts.allow_different_dimensions: #允许
ffwd_different_dimensions(full_in,
full_out,
opts.checkpoint_dir,
device_t=opts.device,
batch_size=opts.batch_size)
else:
ffwd(full_in,
full_out,
opts.checkpoint_dir,
device_t=opts.device,
batch_size=opts.batch_size)
def main():
parser = build_parser()
options = parser.parse_args()
check_opts(options) #check参数有没有问题
style_target = get_img(options.style)
content_targets = list_files(options.train_path)
#指定参数
kwargs = {
"epochs": options.epochs,
"print_iterations": options.checkpoint_iterations,
"batch_size": options.batch_size,
"checkpoint_dir": os.path.join(options.checkpoint_dir, 'fns.ckpt'),
"learning_rate": options.learning_rate
}
args = [
content_targets, style_target, options.content_weight,
options.style_weight, options.tv_weight, options.vgg_path
]
#注意for循环, i = iterations
for preds, losses, i, epoch in optimize(*args, **kwargs):
#提出来单个
#need more than 3 values to unpack
style_loss, content_loss, tv_loss, loss = losses
print('Epoch %d, Iteration: %d, Loss: %s' % (epoch, i, loss))
to_print = (style_loss, content_loss, tv_loss)
print('style: %s, content: %s, tv: %s' % to_print)
def main():
parser = build_parser()
opts = parser.parse_args()
check_opts(opts)
if not os.path.isdir(opts.in_path):
if os.path.exists(opts.out_path) and os.path.isdir(opts.out_path):
out_path = \
os.path.join(opts.out_path,os.path.basename(opts.in_path))
else:
out_path = opts.out_path
ffwd_to_img(opts.in_path,
out_path,
opts.checkpoint_dir,
device=opts.device)
else:
files = list_files(opts.in_path)
full_in = [os.path.join(opts.in_path, x) for x in files]
full_out = [os.path.join(opts.out_path, x) for x in files]
if opts.allow_different_dimensions:
ffwd_different_dimensions(full_in,
full_out,
opts.checkpoint_dir,
device_t=opts.device,
batch_size=opts.batch_size)
else:
ffwd(full_in,
full_out,
opts.checkpoint_dir,
device_t=opts.device,
batch_size=opts.batch_size)
def train_test(trainFP, testFP, malwareFP, popularFP, randomFP):
"""
gets train and test arrays
"""
malware_apks = utils.list_files(malwareFP)
popular_apks = utils.list_files(popularFP)
random_apks = utils.list_files(randomFP)
apks = malware_apks + popular_apks + random_apks
if (not os.path.exists(trainFP)) & (not os.path.exists(testFP)):
apks = [utils.dir_and_app(app)[1] for app in apks]
np.random.shuffle(apks)
split = round(len(apks) * 0.8)
train = apks[:split]
test = apks[split:]
np.savetxt(trainFP, train, fmt="%s")
np.savetxt(testFP, test, fmt="%s")
else:
train = np.loadtxt(trainFP, dtype=object)
test = np.loadtxt(testFP, dtype=object)
return [train, test]
def run(opt):
path_pairs = []
src_path = os.path.abspath(opt.src)
dst_path = os.path.abspath(opt.dst)
utils.make_dir(dst_path)
if os.path.isfile(src_path):
dir_path, file_name = os.path.split(src_path)
src_files = [(os.path.join(dir_path, file_name), file_name)]
else:
src_files = utils.list_files(src_path)
for src_file_path, src_file_name in src_files:
dst_file_name = os.path.splitext(src_file_name)[0] + '.gif'
dst_file_path = os.path.join(dst_path, dst_file_name)
path_pairs.append((src_file_path, dst_file_path))
for fpath_src, fpath_dst in path_pairs:
cmd = 'ffmpeg -y -i {} {}'.format(fpath_src, fpath_dst)
print('Exec:', cmd)
subprocess.run(cmd)
def main():
parser = build_parser()
opts = parser.parse_args()
if opts.no_disk:
evaluate.from_pipe(opts)
else:
in_dir = os.path.join(opts.tmp_dir, 'in')
out_dir = os.path.join(opts.tmp_dir, 'out')
if not os.path.exists(in_dir):
os.makedirs(in_dir)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
in_args = [
'ffmpeg',
'-i', opts.in_path,
'%s/frame_%%d.png' % in_dir
]
subprocess.call(" ".join(in_args), shell=True)
base_names = list_files(in_dir)
in_files = [os.path.join(in_dir, x) for x in base_names]
out_files = [os.path.join(out_dir, x) for x in base_names]
evaluate.ffwd(in_files, out_files, opts.checkpoint, device_t=opts.device,
batch_size=opts.batch_size)
fr = 30 # wtf
out_args = [
'ffmpeg',
'-i', '%s/frame_%%d.png' % out_dir,
'-f', 'mp4',
'-q:v', '0',
'-vcodec', 'mpeg4',
'-r', str(fr),
opts.out
]
subprocess.call(" ".join(out_args), shell=True)
print('Video at: %s' % opts.out)
shutil.rmtree(opts.tmp_dir)
def extract_portfolio_to_csv(self, data_dir):
portfolio_data_dir = self._get_data_dir(data_dir, 'portfolio')
data_header = "table[class='table portfolio'] thead tr td"
data_row = "table[class='table portfolio'] tbody tr"
row_data = []
for f in list_files(portfolio_data_dir, '*.html'):
with open(f, 'r') as fp:
html = fp.read()
soup = BeautifulSoup(html, "html.parser")
if soup:
headers = []
for item in soup.select(data_header):
headers.append(item.text.strip())
rows = soup.select(data_row)
for row in rows:
data = dict()
for index, cell in enumerate(row.select('td')):
data[headers[index]] = cell.text.strip()
row_data.append(data)
print('Num rows:', len(row_data))
# Normalize and export
records = []
for row in row_data:
row['ID'] = int(row['ID'])
row['Prestado'] = self._norm_money(row['Prestado'])
row['Pagado'] = self._norm_money(row['Pagado'])
row['Te debe'] = self._norm_money(row['Te debe'])
balance = self._norm_money(row.get('balance', '0'))
record = tuple(row[header] for header in headers)
records.append(record)
headers = tuple(map(str.upper, headers))
records.sort(key=lambda r: r[0])
write_to_csv(records, './data/portfolio.csv', headers=headers)
#-*- coding: utf-8 -*-
from keras.applications.resnet50 import preprocess_input
import numpy as np
import cv2
from src.model_builder import CamModelBuilder
from src.utils import plot_img, list_files
if __name__ == "__main__":
detector = CamModelBuilder().get_cam_model()
detector.load_weights("weights.h5", by_name=True)
detector.summary()
imgs = list_files("dataset//train//text")
for i, img_path in enumerate(imgs):
original_img = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
img = cv2.resize(original_img, (224, 224))
img = np.expand_dims(img, 0).astype(np.float64)
cam_map = detector.predict(preprocess_input(img))
cam_map = cam_map[0, :, :, 1]
cam_map = cv2.resize(cam_map, (original_img.shape[1], original_img.shape[0]))
plot_img(original_img, cam_map, show=False, save_filename="{}.png".format(i+1))
def _get_files(img_dir):
files = utils.list_files(img_dir)
return files
def _get_files(img_dir):
"""List all files in directory"""
files = list_files(img_dir)
return [os.path.join(img_dir, x) for x in files]
def _get_files(img_dir):
files = list_files(img_dir)
return [os.path.join(img_dir, x) for x in files]
results = basePath + "//" + expNo + "//results//"
testData = basePath + "//" + expNo + "//test//"
# results=basePath+"//"+expNo+"//results1//"
# testData=basePath+"//"+expNo+"//delMe//nonFocus//"
detector = CamModelBuilder().get_cam_model(indx)
#t="/home/kapitsa/PycharmProjects/MyOCRService/objectDetection/Weakly-Supervised-Text-Detection/backUP/paperModel"
#detector.load_weights(".//backUP//weights.19-0.01.h5", by_name=True)
detector.load_weights(modelPath, by_name=True)
#hardPath="/home/kapitsa/PycharmProjects/MyOCRService/objectDetection/Weakly-Supervised-Text-Detection//"#backUP/paperModel//"
#detector.load_weights(hardPath+"//weights.19-0.01.h5", by_name=True)
detector.summary()
imgs = list_files(testData)
for i, img_path in enumerate(imgs):
original_img = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
img = cv2.resize(255 - original_img, (224, 224))
#img = cv2.resize(original_img, (224, 224))
img = np.expand_dims(img, 0).astype(np.float64)
cam_map = detector.predict(preprocess_input(img))
cam_map = cam_map[0, :, :, 1]
cam_map1 = cv2.cvtColor(cam_map, cv2.COLOR_GRAY2BGR)
cam_map1 = cv2.resize(cam_map1,
(original_img.shape[1], original_img.shape[0]))
def extract_transactions_to_csv(self, data_dir):
transactions_data_dir = self._get_data_dir(data_dir, 'transactions')
data_row = "tr[class^='account_statements']"
fields = {
'record_id': ('aut', 'span'),
'date': ('date', ''),
'time': ('date', 'span'),
'reference': ('reference', 'span'),
'type': ('type', ''),
'amount': ('amount', ''),
'balance': ('balance', ''),
}
records = []
for f in list_files(transactions_data_dir, '*.html'):
with open(f, 'r') as fp:
html = fp.read()
soup = BeautifulSoup(html, "html.parser")
if soup:
row_data = list()
for item in soup.select(data_row):
data = dict()
for data_key, selector_data in fields.items():
css_class, sub_elem = selector_data
selector = f"td[class='{css_class}']"
if sub_elem:
selector += f' {sub_elem}'
elem = item.select(selector)
if isinstance(elem, list):
elem = elem[0]
if elem and elem.text:
text = ' '.join(elem.text.strip().split())
else:
text = ''
data[data_key] = text
row_data.append(data)
for row in row_data:
record_id = row.get('record_id', '')
assert record_id
date = row.get('date', '').split()[0]
date = datetime.datetime.strptime(
date, "%d/%m/%Y").strftime('%Y-%m-%d')
time = row.get('time', '').lower()
twenty_four_hour_time = time[:4] # remove the am/pm
hour = int(twenty_four_hour_time.split(':')[0])
minute = int(twenty_four_hour_time.split(':')[1])
is_pm = time[4].lower() == 'p'
if is_pm:
hour = (hour + 12) % 24
time = f'{str(hour).zfill(2)}:{str(minute).zfill(2)}'
reference = row.get('reference', '')
type_ = row.get('type', '').lower()
amount = self._norm_money(row.get('amount', '0'))
balance = self._norm_money(row.get('balance', '0'))
record = (
record_id,
date,
time,
reference,
type_,
amount,
balance,
)
records.append(record)
headers = ('ID', 'DATE', 'TIME', 'REFERENCE', 'TYPE', 'AMOUNT',
'BALANCE')
records.sort(key=lambda r: (r[1], r[2]))
write_to_csv(records, './data/transactions.csv', headers=headers)
def main(targets):
"""
runs the targets
targets --> a list of targets
"""
#["data", "feature", "model", "train", "test"]
if "data" in targets:
data_config = load_params(DATA_PARAMS)
data_androguard.train_txt(data_config["malware_dir"],
data_config["popular_dir"],
data_config["random_dir"],
data_config["app_id_out"])
elif "feature" in targets:
feature_config = load_params(FEATURE_PARAMS)
train, test = (feature_config["trainFP"], feature_config["testFP"],
feature_config["malware_dir"],
feature_config["popular_dir"],
feature_config["random_dir"])
apps = train + test
for app in apps:
data_androguard.wrapper(app, feature_config["commonFP"],
feature_config["metapathsFP"],
feature_config["walksFP"])
elif 'test' in targets:
with open(TEST_PARAMS) as fh:
test_cfg = json.load(fh)
benign_dir = test_cfg['test_benign']
malicious_dir = test_cfg['test_malicious']
apks = utils.list_files(benign_dir) + utils.list_files(malicious_dir)
np.random.shuffle(apks)
split = round(len(apks) * 0.8)
train = apks[:split]
test = apks[split:]
target = test_cfg["test_outputs"]
# get txt with labels:
txt = test_cfg["app_label"]
if os.path.exists(txt):
os.remove(txt)
data_androguard.train_txt(malicious_dir, benign_dir, benign_dir,
target)
else:
data_androguard.train_txt(malicious_dir, benign_dir, benign_dir,
target)
print(train)
# create features using wrapper
apks = train + test
for apk in apks:
data_androguard.wrapper(apk, target, target, target)
# build a common graph, run metapath2vec
data = pd.read_csv(txt)
X, y = coco.metapath2vec(target,
train,
target,
txt,
target,
test_cfg["modelFP"],
testing=True)
md.actuals(X, y, X, y)
elif "mamadroid" in targets:
with open(MAMA_PARAMS) as fh:
mama_cfg = json.load(fh)
malicious_apks = os.listdir(mama_cfg["malware_dir"])
popular_apks = os.listdir(mama_cfg["popular_dir"])
random_apks = os.listdir(mama_cfg["random_dir"])
target = mama_cfg["out_dir"]
# # currently use vectorize, have to test with concurrency
# mama.create_markovs(mama_cfg["malware_dir"], target, "PACKAGE")
# mama.create_markovs(mama_cfg["malware_dir"], target, "FAMILY")
# mama.create_markovs(mama_cfg["popular_dir"], target, "PACKAGE")
# mama.create_markovs(mama_cfg["popular_dir"], target, "FAMILY")
# mama.create_markovs(mama_cfg["random_dir"], target, "PACKAGE")
# mama.create_markovs(mama_cfg["random_dir"], target, "FAMILY")
# directories = [target]
# mama.create_X(directories)
# perform PCA and classify
X_family = load_npz(mama_cfg["family_npz"])
X_package = load_npz(mama_cfg["package_npz"])
y_family = np.loadtxt(mama_cfg["family_label"])
y_package = np.loadtxt(mama_cfg["package_label"])
family_mode = model.baseline(X_family, X_package, y_family, y_package,
[10], mama_cfg["mamadroid_results"])
elif "cocodroid" in targets:
with open(DATA_PARAMS) as fh:
coco_cfg = json.load(fh)
train, test = train_test(coco_cfg["trainFP"], coco_cfg["testFP"],
coco_cfg["malware_dir"],
coco_cfg["popular_dir"],
coco_cfg["random_dir"])
# get X and y
X_train, y_train = coco.metapath2vec(coco_cfg["commonFP"],
train,
coco_cfg["metapathsFP"],
coco_cfg["app_ids_txt"],
coco_cfg["walksFP"],
coco_cfg["cocodroid_model_path"],
reduced=True,
subset=True)
X_test, y_test = coco.metapath2vec_testing(
coco_cfg["commonFP"], coco_cfg["cocodroid_model_path"],
coco_cfg["walksFP"], coco_cfg["metapathsFP"], test,
coco_cfg["app_ids_txt"])
# classification process using models in baseline
md.actuals(X_train, y_train, X_test, y_test, coco_cfg["model_results"],
"cocodroid")
elif "doc2vec" in targets:
with open(DATA_PARAMS) as fh:
d2v_cfg = json.load(fh)
train, test = train_test(d2v_cfg["trainFP"], d2v_cfg["testFP"],
d2v_cfg["malware_dir"],
d2v_cfg["popular_dir"], d2v_cfg["random_dir"])
wrapper_vec = np.vectorize(data_androguard.wrapper)
wrapper_vec(np.array(train), d2v_cfg["commonFP"],
d2v_cfg["metapathsFP"], d2v_cfg["walksFP"])
# if model exists, no need to do another:
if (not os.path.exists(d2v_cfg["doc2vec_model_path"])) & (
not os.path.exists(d2v_cfg["doc2vec_labels"])):
model, labels = data_androguard.doc2vec_train(
train, d2v_cfg["walksFP"], d2v_cfg["app_ids_txt"])
model.save(d2v_cfg["doc2vec_model_path"])
np.savetxt(d2v_cfg["doc2vec_labels"], labels)
else:
print("Model exists, will load the model instead.........")
model = Doc2Vec.load(d2v_cfg["doc2vec_model_path"])
labels = np.loadtxt(d2v_cfg["doc2vec_labels"])
X_train = [model.docvecs[i] for i in range(len(model.docvecs))]
y_train = labels
X_test, y_test = data_androguard.doc2vec_test(
test, d2v_cfg["walksFP"], d2v_cfg["app_ids_txt"], model,
d2v_cfg["commonFP"], d2v_cfg["metapathsFP"], d2v_cfg["walksFP"])
print(len(X_test))
print(len(y_test))
md.actuals(X_train, y_train, X_test, y_test, d2v_cfg["model_results"],
"doc2vec")
return None
from os.path import dirname
from src.utils import list_files
__all__ = list_files(dirname(__path__[0]),
excludes=["__init__.py"],
ftype=".py")
#-*- coding: utf-8 -*-
from keras.applications.resnet50 import preprocess_input
import numpy as np
import cv2
from src.model_builder import CamModelBuilder
from src.utils import plot_img, list_files
if __name__ == "__main__":
detector = CamModelBuilder().get_cam_model()
detector.load_weights("weights.h5", by_name=True)
detector.summary()
imgs = list_files("dataset//train//text")
for i, img_path in enumerate(imgs):
original_img = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
img = cv2.resize(original_img, (224, 224))
img = np.expand_dims(img, 0).astype(np.float64)
cam_map = detector.predict(preprocess_input(img))
cam_map = cam_map[0, :, :, 1]
cam_map = cv2.resize(cam_map,
(original_img.shape[1], original_img.shape[0]))
plot_img(original_img,
cam_map,
show=False,
save_filename="{}.png".format(i + 1))
