def main(opts): # parser = build_parser() # opts = parser.parse_args() # check_opts(opts) # Execute for single file opts.in_path = opts.resized_path # Replace original input path with resized path if not os.path.isdir(opts.in_path): if os.path.exists(opts.out_path) and os.path.isdir(opts.out_path): out_path = \ os.path.join(opts.out_path,os.path.basename(opts.in_path)) else: out_path = opts.out_path ffwd_to_img(opts.in_path, out_path, opts.checkpoint_dir, device=opts.device) else: # Execute for all files in directory files = list_files(opts.in_path) full_in = [os.path.join(opts.in_path, x) for x in files] full_out = [os.path.join(opts.out_path, x) for x in files] if opts.allow_different_dimensions: ffwd_different_dimensions(full_in, full_out, opts.checkpoint_dir, device_t=opts.device, batch_size=opts.batch_size) else: ffwd(full_in, full_out, opts.checkpoint_dir, device_t=opts.device, batch_size=opts.batch_size)
def process_video(video_in, checkpoint, video_out): in_dir = os.path.join(TMP_DIR, 'in') out_dir = os.path.join(TMP_DIR, 'out') if not os.path.exists(in_dir): os.makedirs(in_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) in_args = [ 'ffmpeg', '-i', video_in, '%s/frame_%%d.png' % in_dir ] subprocess.call(" ".join(in_args), shell=True) base_names = list_files(in_dir) in_files = [os.path.join(in_dir, x) for x in base_names] out_files = [os.path.join(out_dir, x) for x in base_names] evaluate.ffwd(in_files, out_files, checkpoint, device_t=DEVICE, batch_size=BATCH_SIZE) fr = 30 # wtf out_args = [ 'ffmpeg', '-i', '%s/frame_%%d.png' % out_dir, '-f', 'mp4', '-q:v', '0', '-vcodec', 'mpeg4', '-r', str(fr), video_out ] subprocess.call(" ".join(out_args), shell=True) print('Video at: %s' % video_out) shutil.rmtree(TMP_DIR)
def main(): parser = build_parser() opts = parser.parse_args() check_opts(opts) if not os.path.isdir(opts.in_path): if os.path.exists(opts.out_path) and os.path.isdir(opts.out_path): out_path = os.path.join(opts.out_path, os.path.basename(opts.in_path)) else: out_path = opts.out_path ffwd_to_img(opts.in_path, out_path, opts.checkpoint_dir, device=opts.device) else: files = list_files(opts.in_path) full_in = map(lambda x: os.path.join(opts.in_path, x), files) full_out = map(lambda x: os.path.join(opts.out_path, x), files) ffwd(full_in, full_out, opts.checkpoint_dir, device_t=opts.device, batch_size=opts.batch_size)
def main(): parser = build_parser() opts = parser.parse_args() check_opts(opts) if not os.path.isdir(opts.in_path): #单个图片 if os.path.exists(opts.out_path) and os.path.isdir( opts.out_path): #out_path是文件夹 out_path = \ os.path.join(opts.out_path, os.path.basename(opts.in_path))#xx/xx.jpg else: out_path = opts.out_path #可以自定义name:xx/xx.jpg print("!!!!!!" + opts.in_path, out_path, opts.checkpoint_dir, opts.device) ffwd_to_img(opts.in_path, out_path, opts.checkpoint_dir, device=opts.device) else: #in_path是文件夹,一群图片 files = list_files(opts.in_path) full_in = [os.path.join(opts.in_path, x) for x in files] full_out = [os.path.join(opts.out_path, x) for x in files] if opts.allow_different_dimensions: #允许 ffwd_different_dimensions(full_in, full_out, opts.checkpoint_dir, device_t=opts.device, batch_size=opts.batch_size) else: ffwd(full_in, full_out, opts.checkpoint_dir, device_t=opts.device, batch_size=opts.batch_size)
def main(): parser = build_parser() options = parser.parse_args() check_opts(options) #check参数有没有问题 style_target = get_img(options.style) content_targets = list_files(options.train_path) #指定参数 kwargs = { "epochs": options.epochs, "print_iterations": options.checkpoint_iterations, "batch_size": options.batch_size, "checkpoint_dir": os.path.join(options.checkpoint_dir, 'fns.ckpt'), "learning_rate": options.learning_rate } args = [ content_targets, style_target, options.content_weight, options.style_weight, options.tv_weight, options.vgg_path ] #注意for循环, i = iterations for preds, losses, i, epoch in optimize(*args, **kwargs): #提出来单个 #need more than 3 values to unpack style_loss, content_loss, tv_loss, loss = losses print('Epoch %d, Iteration: %d, Loss: %s' % (epoch, i, loss)) to_print = (style_loss, content_loss, tv_loss) print('style: %s, content: %s, tv: %s' % to_print)
def main(): parser = build_parser() opts = parser.parse_args() check_opts(opts) if not os.path.isdir(opts.in_path): if os.path.exists(opts.out_path) and os.path.isdir(opts.out_path): out_path = \ os.path.join(opts.out_path,os.path.basename(opts.in_path)) else: out_path = opts.out_path ffwd_to_img(opts.in_path, out_path, opts.checkpoint_dir, device=opts.device) else: files = list_files(opts.in_path) full_in = [os.path.join(opts.in_path, x) for x in files] full_out = [os.path.join(opts.out_path, x) for x in files] if opts.allow_different_dimensions: ffwd_different_dimensions(full_in, full_out, opts.checkpoint_dir, device_t=opts.device, batch_size=opts.batch_size) else: ffwd(full_in, full_out, opts.checkpoint_dir, device_t=opts.device, batch_size=opts.batch_size)
def train_test(trainFP, testFP, malwareFP, popularFP, randomFP): """ gets train and test arrays """ malware_apks = utils.list_files(malwareFP) popular_apks = utils.list_files(popularFP) random_apks = utils.list_files(randomFP) apks = malware_apks + popular_apks + random_apks if (not os.path.exists(trainFP)) & (not os.path.exists(testFP)): apks = [utils.dir_and_app(app)[1] for app in apks] np.random.shuffle(apks) split = round(len(apks) * 0.8) train = apks[:split] test = apks[split:] np.savetxt(trainFP, train, fmt="%s") np.savetxt(testFP, test, fmt="%s") else: train = np.loadtxt(trainFP, dtype=object) test = np.loadtxt(testFP, dtype=object) return [train, test]
def run(opt): path_pairs = [] src_path = os.path.abspath(opt.src) dst_path = os.path.abspath(opt.dst) utils.make_dir(dst_path) if os.path.isfile(src_path): dir_path, file_name = os.path.split(src_path) src_files = [(os.path.join(dir_path, file_name), file_name)] else: src_files = utils.list_files(src_path) for src_file_path, src_file_name in src_files: dst_file_name = os.path.splitext(src_file_name)[0] + '.gif' dst_file_path = os.path.join(dst_path, dst_file_name) path_pairs.append((src_file_path, dst_file_path)) for fpath_src, fpath_dst in path_pairs: cmd = 'ffmpeg -y -i {} {}'.format(fpath_src, fpath_dst) print('Exec:', cmd) subprocess.run(cmd)
def main(): parser = build_parser() opts = parser.parse_args() if opts.no_disk: evaluate.from_pipe(opts) else: in_dir = os.path.join(opts.tmp_dir, 'in') out_dir = os.path.join(opts.tmp_dir, 'out') if not os.path.exists(in_dir): os.makedirs(in_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) in_args = [ 'ffmpeg', '-i', opts.in_path, '%s/frame_%%d.png' % in_dir ] subprocess.call(" ".join(in_args), shell=True) base_names = list_files(in_dir) in_files = [os.path.join(in_dir, x) for x in base_names] out_files = [os.path.join(out_dir, x) for x in base_names] evaluate.ffwd(in_files, out_files, opts.checkpoint, device_t=opts.device, batch_size=opts.batch_size) fr = 30 # wtf out_args = [ 'ffmpeg', '-i', '%s/frame_%%d.png' % out_dir, '-f', 'mp4', '-q:v', '0', '-vcodec', 'mpeg4', '-r', str(fr), opts.out ] subprocess.call(" ".join(out_args), shell=True) print('Video at: %s' % opts.out) shutil.rmtree(opts.tmp_dir)
def extract_portfolio_to_csv(self, data_dir): portfolio_data_dir = self._get_data_dir(data_dir, 'portfolio') data_header = "table[class='table portfolio'] thead tr td" data_row = "table[class='table portfolio'] tbody tr" row_data = [] for f in list_files(portfolio_data_dir, '*.html'): with open(f, 'r') as fp: html = fp.read() soup = BeautifulSoup(html, "html.parser") if soup: headers = [] for item in soup.select(data_header): headers.append(item.text.strip()) rows = soup.select(data_row) for row in rows: data = dict() for index, cell in enumerate(row.select('td')): data[headers[index]] = cell.text.strip() row_data.append(data) print('Num rows:', len(row_data)) # Normalize and export records = [] for row in row_data: row['ID'] = int(row['ID']) row['Prestado'] = self._norm_money(row['Prestado']) row['Pagado'] = self._norm_money(row['Pagado']) row['Te debe'] = self._norm_money(row['Te debe']) balance = self._norm_money(row.get('balance', '0')) record = tuple(row[header] for header in headers) records.append(record) headers = tuple(map(str.upper, headers)) records.sort(key=lambda r: r[0]) write_to_csv(records, './data/portfolio.csv', headers=headers)
#-*- coding: utf-8 -*- from keras.applications.resnet50 import preprocess_input import numpy as np import cv2 from src.model_builder import CamModelBuilder from src.utils import plot_img, list_files if __name__ == "__main__": detector = CamModelBuilder().get_cam_model() detector.load_weights("weights.h5", by_name=True) detector.summary() imgs = list_files("dataset//train//text") for i, img_path in enumerate(imgs): original_img = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB) img = cv2.resize(original_img, (224, 224)) img = np.expand_dims(img, 0).astype(np.float64) cam_map = detector.predict(preprocess_input(img)) cam_map = cam_map[0, :, :, 1] cam_map = cv2.resize(cam_map, (original_img.shape[1], original_img.shape[0])) plot_img(original_img, cam_map, show=False, save_filename="{}.png".format(i+1))
def _get_files(img_dir): files = utils.list_files(img_dir) return files
def _get_files(img_dir): """List all files in directory""" files = list_files(img_dir) return [os.path.join(img_dir, x) for x in files]
def _get_files(img_dir): files = list_files(img_dir) return [os.path.join(img_dir, x) for x in files]
results = basePath + "//" + expNo + "//results//" testData = basePath + "//" + expNo + "//test//" # results=basePath+"//"+expNo+"//results1//" # testData=basePath+"//"+expNo+"//delMe//nonFocus//" detector = CamModelBuilder().get_cam_model(indx) #t="/home/kapitsa/PycharmProjects/MyOCRService/objectDetection/Weakly-Supervised-Text-Detection/backUP/paperModel" #detector.load_weights(".//backUP//weights.19-0.01.h5", by_name=True) detector.load_weights(modelPath, by_name=True) #hardPath="/home/kapitsa/PycharmProjects/MyOCRService/objectDetection/Weakly-Supervised-Text-Detection//"#backUP/paperModel//" #detector.load_weights(hardPath+"//weights.19-0.01.h5", by_name=True) detector.summary() imgs = list_files(testData) for i, img_path in enumerate(imgs): original_img = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB) img = cv2.resize(255 - original_img, (224, 224)) #img = cv2.resize(original_img, (224, 224)) img = np.expand_dims(img, 0).astype(np.float64) cam_map = detector.predict(preprocess_input(img)) cam_map = cam_map[0, :, :, 1] cam_map1 = cv2.cvtColor(cam_map, cv2.COLOR_GRAY2BGR) cam_map1 = cv2.resize(cam_map1, (original_img.shape[1], original_img.shape[0]))
def extract_transactions_to_csv(self, data_dir): transactions_data_dir = self._get_data_dir(data_dir, 'transactions') data_row = "tr[class^='account_statements']" fields = { 'record_id': ('aut', 'span'), 'date': ('date', ''), 'time': ('date', 'span'), 'reference': ('reference', 'span'), 'type': ('type', ''), 'amount': ('amount', ''), 'balance': ('balance', ''), } records = [] for f in list_files(transactions_data_dir, '*.html'): with open(f, 'r') as fp: html = fp.read() soup = BeautifulSoup(html, "html.parser") if soup: row_data = list() for item in soup.select(data_row): data = dict() for data_key, selector_data in fields.items(): css_class, sub_elem = selector_data selector = f"td[class='{css_class}']" if sub_elem: selector += f' {sub_elem}' elem = item.select(selector) if isinstance(elem, list): elem = elem[0] if elem and elem.text: text = ' '.join(elem.text.strip().split()) else: text = '' data[data_key] = text row_data.append(data) for row in row_data: record_id = row.get('record_id', '') assert record_id date = row.get('date', '').split()[0] date = datetime.datetime.strptime( date, "%d/%m/%Y").strftime('%Y-%m-%d') time = row.get('time', '').lower() twenty_four_hour_time = time[:4] # remove the am/pm hour = int(twenty_four_hour_time.split(':')[0]) minute = int(twenty_four_hour_time.split(':')[1]) is_pm = time[4].lower() == 'p' if is_pm: hour = (hour + 12) % 24 time = f'{str(hour).zfill(2)}:{str(minute).zfill(2)}' reference = row.get('reference', '') type_ = row.get('type', '').lower() amount = self._norm_money(row.get('amount', '0')) balance = self._norm_money(row.get('balance', '0')) record = ( record_id, date, time, reference, type_, amount, balance, ) records.append(record) headers = ('ID', 'DATE', 'TIME', 'REFERENCE', 'TYPE', 'AMOUNT', 'BALANCE') records.sort(key=lambda r: (r[1], r[2])) write_to_csv(records, './data/transactions.csv', headers=headers)
def main(targets): """ runs the targets targets --> a list of targets """ #["data", "feature", "model", "train", "test"] if "data" in targets: data_config = load_params(DATA_PARAMS) data_androguard.train_txt(data_config["malware_dir"], data_config["popular_dir"], data_config["random_dir"], data_config["app_id_out"]) elif "feature" in targets: feature_config = load_params(FEATURE_PARAMS) train, test = (feature_config["trainFP"], feature_config["testFP"], feature_config["malware_dir"], feature_config["popular_dir"], feature_config["random_dir"]) apps = train + test for app in apps: data_androguard.wrapper(app, feature_config["commonFP"], feature_config["metapathsFP"], feature_config["walksFP"]) elif 'test' in targets: with open(TEST_PARAMS) as fh: test_cfg = json.load(fh) benign_dir = test_cfg['test_benign'] malicious_dir = test_cfg['test_malicious'] apks = utils.list_files(benign_dir) + utils.list_files(malicious_dir) np.random.shuffle(apks) split = round(len(apks) * 0.8) train = apks[:split] test = apks[split:] target = test_cfg["test_outputs"] # get txt with labels: txt = test_cfg["app_label"] if os.path.exists(txt): os.remove(txt) data_androguard.train_txt(malicious_dir, benign_dir, benign_dir, target) else: data_androguard.train_txt(malicious_dir, benign_dir, benign_dir, target) print(train) # create features using wrapper apks = train + test for apk in apks: data_androguard.wrapper(apk, target, target, target) # build a common graph, run metapath2vec data = pd.read_csv(txt) X, y = coco.metapath2vec(target, train, target, txt, target, test_cfg["modelFP"], testing=True) md.actuals(X, y, X, y) elif "mamadroid" in targets: with open(MAMA_PARAMS) as fh: mama_cfg = json.load(fh) malicious_apks = os.listdir(mama_cfg["malware_dir"]) popular_apks = os.listdir(mama_cfg["popular_dir"]) random_apks = os.listdir(mama_cfg["random_dir"]) target = mama_cfg["out_dir"] # # currently use vectorize, have to test with concurrency # mama.create_markovs(mama_cfg["malware_dir"], target, "PACKAGE") # mama.create_markovs(mama_cfg["malware_dir"], target, "FAMILY") # mama.create_markovs(mama_cfg["popular_dir"], target, "PACKAGE") # mama.create_markovs(mama_cfg["popular_dir"], target, "FAMILY") # mama.create_markovs(mama_cfg["random_dir"], target, "PACKAGE") # mama.create_markovs(mama_cfg["random_dir"], target, "FAMILY") # directories = [target] # mama.create_X(directories) # perform PCA and classify X_family = load_npz(mama_cfg["family_npz"]) X_package = load_npz(mama_cfg["package_npz"]) y_family = np.loadtxt(mama_cfg["family_label"]) y_package = np.loadtxt(mama_cfg["package_label"]) family_mode = model.baseline(X_family, X_package, y_family, y_package, [10], mama_cfg["mamadroid_results"]) elif "cocodroid" in targets: with open(DATA_PARAMS) as fh: coco_cfg = json.load(fh) train, test = train_test(coco_cfg["trainFP"], coco_cfg["testFP"], coco_cfg["malware_dir"], coco_cfg["popular_dir"], coco_cfg["random_dir"]) # get X and y X_train, y_train = coco.metapath2vec(coco_cfg["commonFP"], train, coco_cfg["metapathsFP"], coco_cfg["app_ids_txt"], coco_cfg["walksFP"], coco_cfg["cocodroid_model_path"], reduced=True, subset=True) X_test, y_test = coco.metapath2vec_testing( coco_cfg["commonFP"], coco_cfg["cocodroid_model_path"], coco_cfg["walksFP"], coco_cfg["metapathsFP"], test, coco_cfg["app_ids_txt"]) # classification process using models in baseline md.actuals(X_train, y_train, X_test, y_test, coco_cfg["model_results"], "cocodroid") elif "doc2vec" in targets: with open(DATA_PARAMS) as fh: d2v_cfg = json.load(fh) train, test = train_test(d2v_cfg["trainFP"], d2v_cfg["testFP"], d2v_cfg["malware_dir"], d2v_cfg["popular_dir"], d2v_cfg["random_dir"]) wrapper_vec = np.vectorize(data_androguard.wrapper) wrapper_vec(np.array(train), d2v_cfg["commonFP"], d2v_cfg["metapathsFP"], d2v_cfg["walksFP"]) # if model exists, no need to do another: if (not os.path.exists(d2v_cfg["doc2vec_model_path"])) & ( not os.path.exists(d2v_cfg["doc2vec_labels"])): model, labels = data_androguard.doc2vec_train( train, d2v_cfg["walksFP"], d2v_cfg["app_ids_txt"]) model.save(d2v_cfg["doc2vec_model_path"]) np.savetxt(d2v_cfg["doc2vec_labels"], labels) else: print("Model exists, will load the model instead.........") model = Doc2Vec.load(d2v_cfg["doc2vec_model_path"]) labels = np.loadtxt(d2v_cfg["doc2vec_labels"]) X_train = [model.docvecs[i] for i in range(len(model.docvecs))] y_train = labels X_test, y_test = data_androguard.doc2vec_test( test, d2v_cfg["walksFP"], d2v_cfg["app_ids_txt"], model, d2v_cfg["commonFP"], d2v_cfg["metapathsFP"], d2v_cfg["walksFP"]) print(len(X_test)) print(len(y_test)) md.actuals(X_train, y_train, X_test, y_test, d2v_cfg["model_results"], "doc2vec") return None
from os.path import dirname from src.utils import list_files __all__ = list_files(dirname(__path__[0]), excludes=["__init__.py"], ftype=".py")
#-*- coding: utf-8 -*- from keras.applications.resnet50 import preprocess_input import numpy as np import cv2 from src.model_builder import CamModelBuilder from src.utils import plot_img, list_files if __name__ == "__main__": detector = CamModelBuilder().get_cam_model() detector.load_weights("weights.h5", by_name=True) detector.summary() imgs = list_files("dataset//train//text") for i, img_path in enumerate(imgs): original_img = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB) img = cv2.resize(original_img, (224, 224)) img = np.expand_dims(img, 0).astype(np.float64) cam_map = detector.predict(preprocess_input(img)) cam_map = cam_map[0, :, :, 1] cam_map = cv2.resize(cam_map, (original_img.shape[1], original_img.shape[0])) plot_img(original_img, cam_map, show=False, save_filename="{}.png".format(i + 1))