def __init__(self, mode='train'):
self.mode = mode
self.augmentation = None
# Download dataset
if not os.path.isdir(os.path.join(DATASET_DIR, mode)):
print('Downloading CIFAR10 dataset..')
os.makedirs(DATASET_DIR)
zip_filename = os.path.join(DATASET_DIR, 'tmp.zip')
download_file_from_google_drive(GDRIVE_HASH, zip_filename)
# Unzip train and val files
with zipfile.ZipFile(zip_filename, 'r') as zip_file:
zip_file.extractall(DATASET_DIR)
print(f'CIFAR10 dataset downloaded to {DATASET_DIR}.\n')
# Delete zip file
os.remove(zip_filename)
self.filenames = dict()
self.filenames['image'] = sorted(
glob(os.path.join(DATASET_DIR, mode, 'image_*.png')))
self.filenames['label'] = sorted(
glob(os.path.join(DATASET_DIR, mode, 'label_*.txt')))
assert len(self.filenames['image']) == len(self.filenames['label']), \
'Mismatch in the size of input images and labels.'
def callback(ch, method, properties, body):
resultado = False
print(" [x] Received %r" % body)
task = json.loads(body)
file_id = task['file_id']
now = datetime.now()
## Veficar si la task es de concepts o vocabulary
if (task['document_type'] == 'concepts'):
path_file = "./received_files/concepts/concepts_{0}_{1}.tsv".format(
file_id, now.strftime("%Y%m%d%H%M%S"))
utils.download_file_from_google_drive(file_id, path_file)
print("executing concepts etl with file: {}".format(path_file))
resultado = concepts_etl_final.execute(path_file)
elif (task['document_type'] == 'vocabulary'):
path_file = "./received_files/vocabulary/vocabulary_{0}_{1}.csv".format(
file_id, now.strftime("%Y%m%d%H%M%S"))
utils.download_file_from_google_drive(file_id, path_file)
print("executing vocabulary etl with file: {}".format(path_file))
resultado = vocabulary_etl_final.execute(path_file)
ch.basic_ack(delivery_tag=method.delivery_tag)
if resultado:
# update task in success
update_status_task(task['uuid'], True)
else:
# update task in error
update_status_task(str(task['uuid']), False)
def main():
data_path = os.path.join(_CURRENT_DIR, "../data")
file_id = "16IQjiGu-jl2oTqr5wsp9MmJxtQiuyIWq"
destination = os.path.join(data_path, "bird_dataset.zip")
if not os.path.isfile(destination) and not os.path.isdir(os.path.join(data_path, "bird_dataset")):
download_file_from_google_drive(file_id, destination)
os.system("cd {} && unzip bird_dataset.zip".format(data_path))
def main():
data_path = os.path.join(_CURRENT_DIR, "../saved_models")
os.system("mkdir -p {}".format(data_path))
file_id = "1_-FQFU1i79WySBehqdUXAdbI_-RSvFqb"
destination = os.path.join(data_path, "darknet53.conv.74")
if not os.path.isfile(destination):
download_file_from_google_drive(file_id, destination)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--image', help="Path to input image")
parser.add_argument('--style', help="Which style to apply", choices=list(STYLE_IDS.keys()))
parser.add_argument('--outdir', help="Path to output directory")
parser.add_argument('--min-image-dim', help="Minimum image dimension", default=1000, type=int)
parser.add_argument('--num-images', help="Number of images", default=50, type=int)
parser.add_argument('--shimmer', help="Amount of movement", default=10, type=int)
args = parser.parse_args()
print("Loading image")
img = load_image(args.image, args.min_image_dim)
print("Downloading model")
checkpoint_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '%s.ckpt' % args.style)
download_file_from_google_drive(STYLE_IDS[args.style], checkpoint_path)
noise1 = args.shimmer * np.random.uniform(size=img.shape) + 177
noise2 = args.shimmer * np.random.uniform(size=img.shape) + 177
with imageio.get_writer(os.path.join(args.outdir, 'output.gif'), mode='I') as writer:
for i in range(args.num_images):
mult = np.sin(i * 2 * np.pi / args.num_images) / 2 + 0.5
noise = mult * noise1 + (1 - mult) * noise2
input_img = img + noise
print("Transferring style (%d/%d)" % (i+1, args.num_images))
out = style_transfer(input_img, checkpoint_path)
writer.append_data(out)
print("Image saved")
def __init__(self,
n_cluster=3,
alpha=1,
device='cpu',
lam=0.1,
pre_train=False,
max_cycles=None):
super().__init__()
self.n_cluster = n_cluster
self.alpha = alpha
self.device = device
self.lam = lam
self.max_cycles = max_cycles
if pre_train:
if not os.path.exists('vgg_normalised_conv5_1.pth'):
download_file_from_google_drive(
'1IAOFF5rDkVei035228Qp35hcTnliyMol',
'vgg_normalised_conv5_1.pth')
if not os.path.exists('decoder_relu4_1.pth'):
download_file_from_google_drive(
'1kkoyNwRup9y5GT1mPbsZ_7WPQO9qB7ZZ', 'decoder_relu4_1.pth')
self.vgg_encoder = VGGEncoder('vgg_normalised_conv5_1.pth')
self.decoder = Decoder(4, 'decoder_relu4_1.pth')
else:
self.vgg_encoder = VGGEncoder()
self.decoder = Decoder(4)
self.multimodal_style_feature_transfer = MultimodalStyleTransfer(
n_cluster, alpha, device, lam, max_cycles)
def main():
data_path = os.path.join(_CURRENT_DIR, "../data")
file_id = "1EWEhmvDaYYm0SsydUEGWUDrnBzkLEQc_"
destination = os.path.join(data_path, "switch_detection.zip")
if not os.path.isfile(destination) and not os.path.isdir(
os.path.join(data_path, "switch_detection")):
download_file_from_google_drive(file_id, destination)
os.system("cd {} && unzip switch_detection.zip".format(data_path))
def main():
data_path = os.path.join(_CURRENT_DIR, "../data")
file_id = "1-VMsdeKxOYATf4xC1qrPr_MtkD3SeVkN"
destination = os.path.join(data_path, "tu_simple.zip")
if not os.path.isfile(destination) and not os.path.isdir(
os.path.join(data_path, "tu_simple")):
download_file_from_google_drive(file_id, destination)
os.system("cd {} && unzip tu_simple.zip -d {}".format(
data_path, data_path))
def download_word2vec():
# from https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit
download_file = 'GoogleNews-vectors-negative300.bin.gz'
destination = join(WORDVEC_DIR, download_file)
print("Downloading...")
utils.download_file_from_google_drive('0B7XkCwpI5KDYNlNUTTlSS21pQmM',
destination)
unzip_file = 'GoogleNews-vectors-negative300.bin'
unzip_destination = join(WORDVEC_DIR, unzip_file)
print("Unzipping...")
with gzip.open(destination, 'rb') as f_in, open(unzip_destination,
'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
def callback(ch, method, properties, body):
print(" [x] Received %r" % body)
task = json.loads(body)
file_id = task['file_id']
now = datetime.now()
path_file = "./received_files/vocabulary/vocabulary_{0}_{1}.csv".format(
file_id, now.strftime("%Y%m%d%H%M%S"))
utils.download_file_from_google_drive(file_id, path_file)
print("executing vocabulary etl with file: {}".format(path_file))
resultado = vocabulary_etl_final.execute(path_file)
if resultado:
# update task in success
update_status_task(task['uuid'], True)
else:
# update task in error
update_status_task(str(task['uuid']), False)
def download(self, root, remove_zip=True):
filename = cfg['dataset_dir'] + '.zip'
if os.path.exists(root):
return
file_id = cfg['gdrive_file_id']
download_file_from_google_drive(file_id, filename)
with zipfile.ZipFile(filename, 'r') as f:
f.extractall()
if remove_zip:
os.remove(filename)
shutil.move(cfg['dataset_dir'], root)
def get_celeba_tfrec(size):
data_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'data/celeba/')
zip_data_path = os.path.join(data_path, 'img_align_celeba.zip')
raw_data_path = os.path.join(data_path, 'img_align_celeba/')
training_data_path = os.path.join(data_path, 'train_64x64.tfrec')
test_data_path = os.path.join(data_path, 'test_64x64.tfrec')
if not os.path.exists(data_path):
print('data folder doesn\'t exist, create data folder')
Path(data_path).mkdir(parents=True, exist_ok=True)
if not glob(zip_data_path):
print('Downloading CelebA dataset')
download_file_from_google_drive('0B7EVK8r0v71pZjFTYXZWM3FlRnM',
zip_data_path)
if not glob(raw_data_path):
print('Extracting CelebA dataset')
with zipfile.ZipFile(zip_data_path, 'r') as zip_ref:
zip_ref.extractall('data/celeba/')
if not glob(training_data_path) or not glob(test_data_path):
print('Creating CelebA TFrecord')
create_celeba_tfrec()
def parse(x):
result = tf.io.parse_tensor(x, out_type=tf.float32)
result = tf.reshape(result, [size, size, 3])
return result
if size == 128:
train_dataset = tf.data.TFRecordDataset(
'data/celeba/train_128x128.tfrec').map(
parse, num_parallel_calls=tf.data.experimental.AUTOTUNE)
test_dataset = tf.data.TFRecordDataset(
'data/celeba/test_128x128.tfrec').map(
parse, num_parallel_calls=tf.data.experimental.AUTOTUNE)
elif size == 64:
train_dataset = tf.data.TFRecordDataset(
'data/celeba/train_64x64.tfrec').map(
parse, num_parallel_calls=tf.data.experimental.AUTOTUNE)
test_dataset = tf.data.TFRecordDataset(
'data/celeba/test_64x64.tfrec').map(
parse, num_parallel_calls=tf.data.experimental.AUTOTUNE)
return train_dataset, test_dataset, [-1, size, size, 3]
def __init__(self,
alpha=1,
device='cpu',
use_kmeans_gpu=True,
pre_train=False):
super().__init__()
self.alpha = alpha
self.device = device
self.kmeans_device = device if use_kmeans_gpu else torch.device('cpu')
if pre_train:
if not os.path.exists('vgg_normalised_conv5_1.pth'):
download_file_from_google_drive('1IAOFF5rDkVei035228Qp35hcTnliyMol',
'vgg_normalised_conv5_1.pth')
if not os.path.exists('decoder_relu4_1.pth'):
download_file_from_google_drive('1kkoyNwRup9y5GT1mPbsZ_7WPQO9qB7ZZ',
'decoder_relu4_1.pth')
self.vgg_encoder = VGGEncoder('vgg_normalised_conv5_1.pth').to(device)
self.decoder = Decoder(4, 'decoder_relu4_1.pth').to(device)
else:
self.vgg_encoder = VGGEncoder().to(device)
self.decoder = Decoder(4).to(device)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--video', help="Path to input video")
parser.add_argument('--style',
help="Which style to apply",
choices=list(STYLE_IDS.keys()))
parser.add_argument('--outdir', help="Path to output directory")
parser.add_argument('--min-image-dim',
help="Minimum image dimension",
default=1000,
type=int)
args = parser.parse_args()
print("Downloading model")
checkpoint_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'%s.ckpt' % args.style)
download_file_from_google_drive(STYLE_IDS[args.style], checkpoint_path)
print("Transferring style")
out_path = os.path.join(args.outdir, 'output.avi')
style_transfer_video(args.video, checkpoint_path, out_path)
print("Video saved to %s" % out_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--image', help="Path to input image")
parser.add_argument('--style', help="Which style to apply", choices=list(STYLE_IDS.keys()))
parser.add_argument('--outdir', help="Path to output directory")
parser.add_argument('--min-image-dim', help="Minimum image dimension", default=1000, type=int)
args = parser.parse_args()
print("Loading image")
img = load_image(args.image, args.min_image_dim)
print("Downloading model")
checkpoint_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '%s.ckpt' % args.style)
download_file_from_google_drive(STYLE_IDS[args.style], checkpoint_path)
print("Transferring style")
out = style_transfer(img, checkpoint_path)
if not os.path.exists(args.outdir):
os.makedirs(args.outdir)
output_path = os.path.join(args.outdir, 'output.jpg')
save_image(out, output_path)
print("Image saved to %s" % output_path)
def _download(self,
file_name,
url=None,
dataset_id=None,
file_path=None,
use_tqdm=True):
r""" download file from google drive.
Args:
dataset_id (str): id of file on google drive. guide to get it (https://www.wonderplugin.com/wordpress-tutorials/how-to-apply-for-a-google-drive-api-key/)
use_tqdm (boolean): use tqdm process bar when downloading
"""
os.makedirs(os.path.join(self.root_dir, self.dataset_dir, 'raw'),
exist_ok=True)
if dataset_id != None:
print("Downloading...")
try:
try:
download_file_from_google_drive(
dataset_id,
os.path.join(self.root_dir, self.dataset_dir, 'raw'),
use_tqdm)
except:
url = "https://www.googleapis.com/drive/v3/files/" + dataset_id + "?alt=media&key=AIzaSyBEp1hj-WxRxAezSd5sGfPmWnLbuxuxSvI"
download_with_url(
url,
os.path.join(self.root_dir, self.dataset_dir, 'raw'),
file_name, use_tqdm)
except:
try:
if os.path.exists(
os.path.join(self.root_dir, self.dataset_dir,
'raw', file_name)):
os.remove(
os.path.join(self.root_dir, self.dataset_dir,
'raw', file_name))
download_file_from_google_drive(
dataset_id,
os.path.join(self.root_dir, self.dataset_dir, 'raw'),
use_tqdm)
except:
if os.path.exists(
os.path.join(self.root_dir, self.dataset_dir,
'raw', file_name)):
os.remove(
os.path.join(self.root_dir, self.dataset_dir,
'raw', file_name))
url = "https://www.googleapis.com/drive/v3/files/" + dataset_id + "?alt=media&key=AIzaSyBEp1hj-WxRxAezSd5sGfPmWnLbuxuxSvI"
download_with_url(
url,
os.path.join(self.root_dir, self.dataset_dir, 'raw'),
file_name, use_tqdm)
print("Downloaded!")
elif url != None:
download_with_url(
url, os.path.join(self.root_dir, self.dataset_dir, 'raw'),
file_name, use_tqdm)
elif file_path != None:
print('Copying data...')
copy2(
file_path,
os.path.join(self.root_dir, self.dataset_dir, 'raw',
file_name))
print("Copied!")
else:
if not os.path.exists(
os.path.join(self.root_dir, self.dataset_dir, 'raw',
file_name)):
raise FileExistsError(
'please download file %s into %s' %
(file_name,
os.path.join(self.root_dir, self.dataset_dir, 'raw')))
import utils
import config
# https://drive.google.com/file/d/1-Fnm3tRx6zedcc-syHfms-yqZypByiIl/view?usp=sharing
if __name__ == "__main__":
# TAKE ID FROM SHAREABLE LINK
gdrive_file_id = '1-CdkbAmK_tPVANKSEaE_uMktcjiV44BR'
# DESTINATION FILE ON YOUR DISK
destination = config.model_path
utils.download_file_from_google_drive(gdrive_file_id, destination)
if not os.path.exists(Img_img_align_celeba_png):
os.makedirs(Img_img_align_celeba_png)
if not os.path.exists(Anno):
os.makedirs(Anno)
if not os.path.exists(Eval):
os.makedirs(Eval)
# download
for i, (fileid, path) in enumerate(zip(ids, paths)):
print('{}/{} downloading {}'.format(i + 1, len(ids), path))
path = os.path.join(root, path)
if not os.path.exists(path):
download_file_from_google_drive(fileid, path)
# unzip
try:
subprocess.call([
'7z', 'x', '-o' + os.path.relpath(os.path.join(root, 'Img')),
os.path.join(Img_img_celeba, 'img_celeba.7z.*')
])
except:
print('can\'t unzip img_celeba')
try:
subprocess.call([
'7z', 'x', '-o' + os.path.relpath(os.path.join(root, 'Img')),
os.path.join(Img_img_align_celeba_png, 'img_align_celeba_png.7z.*')
])
import os
import sys
import subprocess
from utils import download_file_from_google_drive
# url and path
google_drive_id = "19oAw8wWn3Y7z6CKChRdAyGOB9yupL_Xt"
# directory
try:
root = os.path.join(sys.argv[1], 'jsv_ver1/')
except:
root = './vsj_ver1/'
if not os.path.exists(root):
os.makedirs(root)
# download
download_file_from_google_drive(google_drive_id,
os.path.join(root, "jvs_ver1.zip"))
# unzip
try:
subprocess.call(['unzip', os.path.join(root, 'jvs_ver1.zip'), '-d', root])
except:
print('can\'t unzip jvs_ver1.zip')
else:
queries = pd.read_csv(os.path.join('data', 'queries_train.tsv'),
sep='\t')
logging.info("Successfully loaded queries data.")
import configuration
config = configuration.ConfigClass()
# do we need to download a pretrained model?
model_url = config.get_model_url()
if model_url is not None and config.get_download_model():
import utils
dest_path = 'model.zip'
utils.download_file_from_google_drive(model_url, dest_path)
if not os.path.exists(model_dir):
os.mkdir(model_dir)
if os.path.exists(dest_path):
utils.unzip_file(dest_path, model_dir)
logging.info(
f'Successfully downloaded and extracted pretrained model into {model_dir}.'
)
else:
logging.error('model.zip file does not exists.')
# test for each search engine module
engine_modules = [
'search_engine_' + name for name in ['1', '2', '3', 'best']
]
for engine_module in engine_modules:
# Initialize model
config = FashionConfig()
config.display()
MODEL_DIR = os.path.join(CUR_DIR, "logs")
model = modellib.MaskRCNN(model_dir=MODEL_DIR, config=config)
# if torch.cuda.is_available():
# model = model.cuda()
# Download pretrained weights
coco_pretrained = os.path.join(CUR_DIR, 'mask_rcnn_coco.pth')
if not os.path.isfile(coco_pretrained):
print("Downloading Pretrained Model...")
share_id = '1RhdD8PkR_AQ1-uP3JS-nbcXaTOILhXua'
utils.download_file_from_google_drive(share_id, coco_pretrained)
model.load_state_dict(torch.load(coco_pretrained))
data_path = os.path.join(CUR_DIR, 'images')
results_path = os.path.join(CUR_DIR, 'results')
img_list = [f for f in os.listdir(data_path) if '.jpg' in f]
for img_name in img_list:
save_name = os.path.join(results_path, img_name.split('.')[0] + '.pth')
if os.path.isfile(save_name):
continue
img_path = os.path.join(data_path, img_name)
