def save_to_db(self): ImageModel.create(author=self.author, url=self.url, number=self.number, upvote=self.upvote, downvote=self.downvote, crawl_time=self.crawl_time)
def save_to_db(self): ImageModel.create(author=self.author, url=self.url, number=self.number, upvote=self.upvote, downvote=self.downvote, crawl_time=self.crawl_time )
def upload_pic(request): if request.method == 'POST': form = ImageUploadForm(request.POST, request.FILES) if form.is_valid(): image = ImageModel(image=request.FILES['image']) image.save() return render(request, 'success.html') else: form = ImageUploadForm() return render(request, 'upload.html', {'form': form})
def __init__(self, model, dataset_name): dataset_path = f"../data/dataset/{dataset_name}" items_data, items_idx = read_item_data(dataset_path) user_items, _ = read_user_item_data(dataset_path) self.model = model self.image_model = ImageModel('resnet50') self.user_items = user_items self.items_data = items_data self.items_idx = items_idx self.n_users = model.n_users self.n_items = model.n_items self.image_folder = dataset_path + "/images"
def save_image(): if request.method == 'POST': image_b64 = request.values['imageBase64'] right_number = request.values['right'] image_encoded = image_b64.split(',')[1] image_bin = base64.decodebytes(image_encoded.encode('utf-8')) image_model = ImageModel() image_model.save(right_number, image_bin) return jsonify({'success': 1})
def map_to_image_list(data_list): result = [] for data in data_list: if len(data) != 3: raise InternalServerErrorException image_id = data[0] width = data[1] height = data[2] result.append(ImageModel(image_id, width, height)) return result
def check_image(): if request.method == 'POST': if model == None: return jsonify({'success': 0}) image_b64 = request.values['imageBase64'] image_encoded = image_b64.split(',')[1] image_bin = base64.decodebytes(image_encoded.encode('utf-8')) image_model = ImageModel() temp_path = image_model.save_temp(image_bin) parser = ImageParser() result = parser.parse(temp_path) return jsonify(result)
def main(): device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') print(f"Training on {device}") cnn_model = models.resnet18(pretrained = True) tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased") transformer_model = AutoModelForSequenceClassification.from_pretrained( "distilbert-base-uncased" ) if FREEZE: freeze_model(cnn_model) freeze_model(transformer_model) image_model = ImageModel(cnn_model, num_out=H) text_model = TextModel(transformer_model, num_out=H) if PRETRAINED_MODEL: model = torch.load(PRETRAINED_MODEL) else: model = MegaModel(image_model, text_model, num_hidden=H) # model = MegaModelAggregator(image_model, text_model, num_hidden=2*H) model.to(device) train_dataloader, test_dataloader = load_dataloaders( path=PATH, image_folder=IMAGE_FOLDER, descriptor=DESCRIPTOR, batch_size=BATCH_SIZE ) loss_fn = torch.nn.CrossEntropyLoss(reduction='mean') train_optim( model=model, tokenizer=tokenizer, train_dataloader=train_dataloader, test_dataloader=train_dataloader, loss_fn=loss_fn, epochs=EPOCHS, log_frequency=LOG_FREQUENCY, device=device, save_file=SAVE_INFO, save_dir=SAVE_MODEL, learning_rate=LEARNING_RATE )
def create(cls, source, origin_url=None, image_data=None, save_in_origin=False): _store_hash = None if origin_url != None: _image_id = Image.get_image_id_by_origin_url(origin_url) if _image_id != None: return cls(_image_id) elif image_data != None: _store_hash = md5(image_data).hexdigest() if save_in_origin: _store_hash = md5(_store_hash + 'GUOKUIMAGESAVEINORIGINSIZE').hexdigest() _image_id = Image.get_image_id_by_store_hash(_store_hash) if _image_id != None: return cls(_image_id) cls.save_origin_image_data(_store_hash, image_data) else: _store_hash = md5( _store_hash + 'GUOKUIMAGECUTINSQUREANDSAVEINSEVERALSIZES').hexdigest() _image_id = Image.get_image_id_by_store_hash(_store_hash) if _image_id != None: return cls(_image_id) cls.save_square_image_data_fixed(_store_hash, image_data) _image_obj = ImageModel(source=source, created_time=datetime.datetime.now(), updated_time=datetime.datetime.now()) if origin_url != None: _image_obj.origin_url = origin_url if _store_hash != None: _image_obj.store_hash = _store_hash _image_obj.save() _inst = cls(str(_image_obj.id)) _inst.image_obj = _image_obj return _inst
def parse_images(self): url = '%s/pic/' % self.base_url content = self.request_content(url) # print content if content: pattern = re.compile( '<img.*?"user-img lazy-load left".*?data-src="(.*?)".*?onerror.*?>.*?' + '<div.*?class="name-time-wrapper left">.*?<span.*?class="name">(.*?)</span>.*?' + '<span.*?class="time timeago".*?>(.*?)</span>.*?' + '<div.*?class="upload-txt.*?<p>(.*?)</p>.*?' + '<img.*?class="upload-img lazy".*?data-src="(.*?)".*?>', re.S) # items = re.findall(pattern, content) models = [] for item in items: user = User(item[0], item[1]) # (self, user, time, image_url, text, content_from): model = ImageModel(user, item[2], item[3], item[4], self.name) models.append(model) return models
def download(dir="images/"): for item in ImageModel.select(): if not item.url: continue filename = dir + str(item.number) + os.path.splitext(item.url)[1] if os.path.exists(filename): print("skipping {}".format(item.number)) continue print("donwloading {}".format(item.number)) try: r = requests.get(item.url, stream=True, headers=HEADERS, timeout=10.0) with open(filename, 'wb') as f: r.raw.decode_content = True # force decode from gzip for chunk in r.iter_content(chunk_size=100*1024): if chunk: f.write(chunk) f.flush() except requests.ConnectionError as e: print("Network Error") except requests.exceptions.RequestException as e: print("Error")
def parse_images(self): url = '%s/pic/' % self.base_url content = self.request_content(url) # print content if content: pattern = re.compile( '<img.*?u-logo lazy".*?data-original="(.*?)".*?></a>.*?' + # user icon url '<a.*?u-user-name.*?target="_blank">(.*?)</a>.*?' + # user name '<span.*?f-ib f-fr">(.*?)</span>.*?' + # release time '<div.*?j-r-list-c-desc">(.*?)</div>.*?' + # text '<img.*?class="lazy".*?data-original="(.*?)".*?>', re.S) #image url items = re.findall(pattern, content) models = [] for item in items: user = User(item[0], item[1]) # (self, user, time, image_url, text, content_from): model = ImageModel(user, item[2], item[3], item[4], self.name) models.append(model) return models
def new_image(): if 'file' not in request.files: flash('No file part') return redirect(url_for('image_gallery')) dicom_file = request.files['file'] if dicom_file.filename == '': flash('No selected file') elif not allowed_file(dicom_file.filename): flash('File type not allowed') else: filename = secure_filename(dicom_file.filename) dataset = read_dicom_file(dicom_file) image_filename = write_image_and_thumbnail(dataset, filename) new_image = ImageModel(filename=image_filename, dicom_data=str(dataset)) session.add(new_image) session.commit() flash('Image saved ({})'.format(image_filename)) return redirect(url_for('image_gallery'))
def download(dir="images/"): for item in ImageModel.select(): if not item.url: continue filename = dir + str(item.number) + os.path.splitext(item.url)[1] if os.path.exists(filename): print("skipping {}".format(item.number)) continue print("donwloading {}".format(item.number)) try: r = requests.get(item.url, stream=True, headers=HEADERS, timeout=10.0) with open(filename, 'wb') as f: r.raw.decode_content = True # force decode from gzip for chunk in r.iter_content(chunk_size=100 * 1024): if chunk: f.write(chunk) f.flush() except requests.ConnectionError as e: print("Network Error") except requests.exceptions.RequestException as e: print("Error")
import utils from models import ImageModel image_folder = "../data/Clothing_Shoes_and_Jewelry" class InvalidImage(Exception): pass items_data, items_idx = utils.read_item_data() images = [(a[0], a[2]) for a in items_data] image_model = ImageModel("vgg16") def worker(msg, q): try: item, name = msg filename = os.path.join(image_folder, name) img = Image.open(filename) try: img_t = image_model.transform(img) except: raise InvalidImage() if img_t.shape != (3, 224, 224): raise InvalidImage() batch_t = torch.unsqueeze(img_t, 0) features = image_model.get_features(batch_t)
import utils from models import ImageModel path = "../" image_folder = path + "data/Clothing_Shoes_and_Jewelry" input_size = (3, 244, 244) use_gpu = torch.cuda.is_available() items_data, items_idx = utils.read_item_data() user_items = utils.read_user_item_data() im = ImageModel('resnet50') image_feat_size = im.image_feat_size class InvalidImage(Exception): pass def work(msg): item, _, name = msg try: filename = os.path.join(image_folder, name) img = Image.open(filename) img_t = im.transform(img) return item, img_t
def create_models(image_features_len, embedding_weights, embeddings_dim, similarity): image_model = ImageModel(image_features_len, embeddings_dim) caption_model = CaptionModel(embedding_weights, embeddings_dim) main_model = MainModel(image_model, caption_model, similarity) return main_model, image_model, caption_model
class Experimentation: def __init__(self, model, dataset_name): dataset_path = f"../data/dataset/{dataset_name}" items_data, items_idx = read_item_data(dataset_path) user_items, _ = read_user_item_data(dataset_path) self.model = model self.image_model = ImageModel('resnet50') self.user_items = user_items self.items_data = items_data self.items_idx = items_idx self.n_users = model.n_users self.n_items = model.n_items self.image_folder = dataset_path + "/images" def search(self, u, n=10): res = self.model.score(u) res = res.argsort(dim=0, descending=True).squeeze().numpy() return res[:n] def load_image(self, i): name = self.items_data[i][2] filename = os.path.join(self.image_folder, name) img = Image.open(filename) return pad_resize(img) def get_image_features(self, img): return self.image_model.get_features(img) / 60 def get_rank(self, u_scores, score): return int(np.searchsorted(-u_scores, -score)) + 1 def run_wb_single_user(self, args, logger): u = args.user steps = args.steps epsilon = int(args.epsilon * 255) if hasattr(args, 'item') and args.item is not None: t = args.item result = self.search(u, -1) from_rank = np.where(result == t)[0][0].item() + 1 else: from_rank = args.from_rank result = self.search(u, from_rank) t = result[from_rank - 1].item() print( f"Run single user, white-box, for user {u} from rank {from_rank}") logger.log({ 'user': u, 'item': t, 'rank': from_rank, }) u_scores = self.model.score(u).detach().numpy().squeeze() u_scores.sort() u_scores = np.flip(u_scores) img = self.load_image(t) img_M = np.array(img) for step in range(steps): t_img_tran = self.image_model.transform(img) t_img_tran = torch.unsqueeze(t_img_tran, 0) t_img_var = nn.Parameter(t_img_tran) t_feat = self.get_image_features(t_img_var) zero_gradients(t_img_var) loss = self.model.pointwise_forward([u], [t], t_feat) loss.backward(retain_graph=True) grad = t_img_var.grad.data grad = grad.sign().squeeze(0).numpy().transpose((1, 2, 0)) img_M = np.clip(img_M.astype(np.int) - epsilon * grad, 0, 255).astype(np.uint8) img = Image.fromarray(img_M) img_t = self.image_model.transform(img) feat_t = self.get_image_features(img_t.unsqueeze(0)) score = self.model.score_user_item(u, t, feat_t.detach()).item() rank = self.get_rank(u_scores, score) print(f"Step: {step+1}, rank={rank}") logger.log({ 'step': step, 'rank': rank, }) if rank == 1: break return img def run_wb_single_user_o(self, args, logger): u = args.user steps = args.steps epsilon = args.epsilon if hasattr(args, 'item') and args.item is not None: t = args.item result = self.search(u, -1) from_rank = np.where(result == t)[0][0].item() + 1 else: from_rank = args.from_rank result = self.search(u, from_rank) t = result[from_rank - 1] print( f"Run single user, white-box, for user {u} from rank {from_rank}") t_img = self.load_image(t) t_img_tran = self.image_model.transform(t_img) t_img_tran = torch.unsqueeze(t_img_tran, 0) t_img_var = nn.Parameter(t_img_tran) t_feat = self.get_image_features(t_img_var) u_scores = self.model.score(u).detach().numpy().squeeze() u_scores.sort() u_scores = np.flip(u_scores) for step in range(steps): zero_gradients(t_img_var) loss = self.model.pointwise_forward([u], [t], t_feat) score = -loss.item() rank = self.get_rank(u_scores, score) print(f"Step: {step+1}, Rank: {rank}") logger.log({ 'step': step + 1, 'rank': rank, }) if rank == 1: break loss.backward(retain_graph=True) grad = torch.sign(t_img_var.grad.data) adversarial = t_img_var.data - epsilon * grad t_img_var.data = adversarial t_feat = self.get_image_features(t_img_var) return Image.fromarray(to_img(adversarial)) def bb_attack(self, u, t, img, img_M, ori_img_M, n_examples, n_features, u_scores, do_pca, pca_fc, by_rank, rank_distribution, epsilon, eps, attack="ifgsm"): img_t = self.image_model.transform(img) img_t = img_t.unsqueeze(0) var_t = nn.Parameter(img_t) feat_t = self.get_image_features(var_t) y_t = self.model.score_user_item(u, t, feat_t.detach()).item() rank = self.get_rank(u_scores, y_t) if do_pca: pca_t = pca_fc.transform(feat_t) def score_rank(rank): quantile = 1 - rank / (self.n_items + 1) if rank_distribution == "normal": return scipy.stats.norm.ppf(quantile) return quantile if by_rank: s_t = score_rank(rank) W = np.zeros((n_examples, n_features), dtype=np.float32) y = np.zeros(n_examples, dtype=np.float32) noise = 7 for i in tqdm(range(n_examples)): d = np.random.choice(range(-noise, noise + 1), size=img_M.shape) img_i = np.clip(img_M.astype(np.int) + d, 0, 255).astype(np.uint8) img_i = Image.fromarray(img_i) img_i = self.image_model.transform(img_i) feat_i = self.get_image_features(img_i.unsqueeze(0)) if do_pca: pca_i = pca_fc.transform(feat_i) y_i = self.model.score_user_item(u, t, feat_i).item() if by_rank: s_i = score_rank(self.get_rank(u_scores, y_i)) diff = pca_i - pca_t if do_pca else feat_i - feat_t W[i] = diff.detach().numpy() y[i] = s_t - s_i if by_rank else y_t - y_i x = solve(W, y) if do_pca: pca_t.backward(torch.tensor(x).view(1, n_features), retain_graph=True) else: feat_t.backward(torch.tensor(x).view(1, n_features), retain_graph=True) grad = var_t.grad # fgsm, ifgsm, pgd, first if attack == "first": grad = grad.squeeze(0).numpy().transpose((1, 2, 0)) grad /= norm(grad) grad = np.clip(grad * 255, -epsilon, epsilon) img_M = np.clip(img_M.astype(np.int) - grad, 0, 255).astype(np.uint8) elif attack == "ifgsm": grad = grad.sign().squeeze(0).numpy().transpose((1, 2, 0)) img_M = np.clip(img_M.astype(np.int) - epsilon * grad, 0, 255).astype(np.uint8) elif attack == "pgd": grad = grad.sign().squeeze(0).numpy().transpose((1, 2, 0)) img_M = np.clip(img_M.astype(np.int) - epsilon * grad, 0, 255) eta = np.clip(img_M - ori_img_M, -eps, eps) img_M = np.clip(ori_img_M + eta, 0, 255).astype(np.uint8) return img_M def run_bb_single_user(self, args, logger): u = args.user do_pca = args.do_pca by_rank = args.by_rank rank_distribution = "uniform" if by_rank: rank_distribution = args.rank_distribution steps = args.steps epsilon = int(args.epsilon * 255) n_examples = args.examples u_scores = self.model.score(u).detach().numpy().squeeze() u_scores.sort() u_scores = np.flip(u_scores) if hasattr(args, 'item') and args.item is not None: t = args.item result = self.search(u, -1) from_rank = np.where(result == t)[0][0].item() + 1 else: from_rank = args.from_rank result = self.search(u, from_rank) t = result[from_rank - 1].item() print( f"Run single user, black-box, for user {u} from rank {from_rank}") logger.log({ 'user': u, 'item': t, 'rank': from_rank, }) backup = self.model.F[t].unsqueeze(0).clone().detach() n_features = backup.shape[1] pca_fc = None if do_pca: n_components = args.n_components pca_fc = TorchPCA.get_pca(self.model.F.numpy(), n_components) n_features = n_components img = self.load_image(t) img_M = np.array(img) if not os.path.exists(f"images/{t}"): os.makedirs(f"images/{t}") img.save(f"images/{t}/original.jpeg", "JPEG") ori_img_M = img_M eps = 10 for step in range(steps): img_M = self.bb_attack(u, t, img, img_M, ori_img_M, n_examples, n_features, u_scores, do_pca, pca_fc, by_rank, rank_distribution, epsilon, eps) img = Image.fromarray(img_M) img_t = self.image_model.transform(img) feat_t = self.get_image_features(img_t.unsqueeze(0)) score = self.model.score_user_item(u, t, feat_t.detach()).item() rank = self.get_rank(u_scores, score) print(f"Step: {step+1}, rank={rank}") logger.log({ 'step': step, 'rank': rank, }) img.save(f"images/{t}/step_{step}.jpeg", "JPEG") if rank == 1: break return img def run_bb_segment_attack(self, args, logger): do_pca = args.do_pca by_rank = args.by_rank if by_rank: rank_distribution = args.rank_distribution steps = args.steps epsilon = int(args.epsilon * 255) n_examples = args.examples i = args.item while True: users = [ u for u, items in enumerate(self.user_items) if i in items ] if len(users) >= 20: break i += 1 random_users = random.sample(range(self.model.n_users), k=100) args.user = user = self.model.add_fake_user_by_item(i) user_scores = self.model.score(user).detach().numpy().squeeze() user_scores.sort() user_scores = np.flip(user_scores) t = random.choice(range(self.model.n_items)) print(f"Segment bb experiment for user={user}, item={t}") logger.log({ 'user': user, 'target_item': t, 'seed_item': i, }) users_scores = {} for us in [users, random_users]: ranks = [] for u in us: u_scores = self.model.score(u).detach().numpy().squeeze() u_scores.sort() u_scores = np.flip(u_scores) users_scores[u] = u_scores score = self.model.score_user_item(u, t).item() rank = self.get_rank(u_scores, score) ranks.append(rank) logger.log({ 'step': 0, 'rank': ranks, }) ranks = np.array(ranks) backup = self.model.F[t].unsqueeze(0).clone().detach() n_features = backup.shape[1] pca_fc = None if do_pca: n_components = args.n_components pca_fc = TorchPCA.get_pca(self.model.F.numpy(), n_components) n_features = n_components img = self.load_image(t) img_M = np.array(img) ori_img_M = img_M eps = 10 for step in range(steps): print(f"Step: {step}") img_M = self.bb_attack(user, t, img, img_M, ori_img_M, n_examples, n_features, user_scores, do_pca, pca_fc, by_rank, rank_distribution, epsilon, eps) img = Image.fromarray(img_M) img_t = self.image_model.transform(img) feat_t = self.get_image_features(img_t.unsqueeze(0)).detach() for us in [users, random_users]: ranks = [] for u in us: u_scores = users_scores[u] score = self.model.score_user_item(u, t, feat_t).item() rank = self.get_rank(u_scores, score) ranks.append(rank) logger.log({ 'step': step + 1, 'rank': ranks, }) ranks = np.array(ranks) print((ranks <= 20).mean()) return img def run_wb_segment_attack(self, args, logger): steps = args.steps epsilon = args.epsilon i = args.item while True: users = [ u for u, items in enumerate(self.user_items) if i in items ] if len(users) >= 20: break i += 1 random_users = random.sample(range(self.model.n_users), k=100) args.user = user = self.model.add_fake_user_by_item(i) t = random.choice(range(self.model.n_items)) print(f"Segment wb experiment for user={user}, item={t}") logger.log({ 'user': user, 'target_item': t, 'seed_item': i, }) users_scores = {} for us in [users, random_users]: ranks = [] for u in us: u_scores = self.model.score(u).detach().numpy().squeeze() u_scores.sort() u_scores = np.flip(u_scores) users_scores[u] = u_scores score = self.model.score_user_item(u, t).item() rank = self.get_rank(u_scores, score) ranks.append(rank) logger.log({ 'step': 0, 'rank': ranks, }) ranks = np.array(ranks) img = self.load_image(t) img_M = np.array(img) for step in range(steps): print(f"Step: {step}") t_img_tran = self.image_model.transform(img) t_img_tran = torch.unsqueeze(t_img_tran, 0) t_img_var = nn.Parameter(t_img_tran) t_feat = self.get_image_features(t_img_var) zero_gradients(t_img_var) loss = self.model.pointwise_forward([user], [t], t_feat) loss.backward(retain_graph=True) grad = t_img_var.grad.data grad = grad.sign().squeeze(0).numpy().transpose((1, 2, 0)) img_M = np.clip(img_M.astype(np.int) - epsilon * grad, 0, 255).astype(np.uint8) img = Image.fromarray(img_M) img_t = self.image_model.transform(img) feat_t = self.get_image_features(img_t.unsqueeze(0)) for us in [users, random_users]: ranks = [] for u in us: u_scores = users_scores[u] score = -self.model.pointwise_forward([u], [t], feat_t).item() rank = self.get_rank(u_scores, score) ranks.append(rank) logger.log({ 'step': step + 1, 'rank': ranks, }) ranks = np.array(ranks) # print(ranks) # print(ranks.mean()) print((ranks <= 20).mean()) return img def run_baseline(self, args, logger): u = args.user u_scores = self.model.score(u).detach().numpy().squeeze() u_scores.sort() u_scores = np.flip(u_scores) if hasattr(args, 'item') and args.item is not None: t = args.item result = self.search(u, -1) from_rank = np.where(result == t)[0][0].item() + 1 else: from_rank = args.from_rank result = self.search(u, from_rank) t = result[from_rank - 1].item() top = self.search(u, 1)[0].item() t_img = self.load_image(t) top_img = self.load_image(top) print(f"Run single user baseline, for user {u} from rank {from_rank}") logger.log({ 'user': u, 'item': t, 'rank': from_rank, }) img_M = np.array(t_img) top_M = np.array(top_img) img_M = np.clip( img_M.astype(np.float) - 0.07 * top_M.astype(np.float), 0, 255).astype(np.uint8) img = Image.fromarray(img_M) img_t = self.image_model.transform(img) feat_t = self.get_image_features(img_t.unsqueeze(0)) score = self.model.score_user_item(u, t, feat_t.detach()).item() rank = self.get_rank(u_scores, score) print(f"Rank: {rank}") logger.log({ 'step': 0, 'rank': rank, }) return img def run(self, name, args): random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) # ts = time.strftime("%y%m%d-%H%M%S") logfile = f"{name}.log" with JsonLooger(logfile) as logger: logger.log({k: v for k, v in vars(args).items() if "__" not in k}) if args.experiment == "single_user": if args.blackbox > 0: return self.run_bb_single_user(args, logger) else: return self.run_wb_single_user(args, logger) elif args.experiment == "segment": if args.blackbox > 0: return self.run_bb_segment_attack(args, logger) else: return self.run_wb_segment_attack(args, logger) elif args.experiment == "baseline": return self.run_baseline(args, logger)
def HookFS(fs): ImageModel.SetFS(fs)