import librosa
import argparse
import numpy as np
import moviepy.editor as mpy
import random
import torch
import runway
import numpy as np
from scipy.misc import toimage
from tqdm import tqdm
from pytorch_pretrained_biggan import (BigGAN, one_hot_from_names, truncated_noise_sample,
save_as_images, display_in_terminal)
@runway.setup(options={"checkpoint": runway.category(description="Pretrained checkpoints to use.",
choices=['celebAHQ-512', 'celebAHQ-256', 'celeba'],
default='celebAHQ-512')})
def setup(opts):
checkpoint = opts['checkpoint']
'PGAN', model_name=checkpoint,
return model
@runway.command('generate',
inputs={ 'z': runway.vector(length=512, sampling_std=0.5)},
outputs={ 'image': runway.image })
def generate(model, inputs):
# Generate ♾ infinite ♾ images
z = inputs['z']
latents = z.reshape((1, 559))
latents = torch.from_numpy(latents)
- noise, _ = model.buildNoiseData(1)
with torch.no_grad():
# load face detection and warping
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')
warper = FaceWarper(predictor,
desiredFaceWidth=256,
desiredLeftEye=(0.371, 0.480))
# tags that can be modified
tags = "5_o_Clock_Shadow Arched_Eyebrows Attractive Bags_Under_Eyes Bald Bangs Big_Lips Big_Nose Black_Hair Blond_Hair Blurry Brown_Hair Bushy_Eyebrows Chubby Double_Chin Eyeglasses Goatee Gray_Hair Heavy_Makeup High_Cheekbones Male Mouth_Slightly_Open Mustache Narrow_Eyes No_Beard Oval_Face Pale_Skin Pointy_Nose Receding_Hairline Rosy_Cheeks Sideburns Smiling Straight_Hair Wavy_Hair Wearing_Earrings Wearing_Hat Wearing_Lipstick Wearing_Necklace Wearing_Necktie Young"
tags = tags.split()
command_inputs = {
'image': runway.image,
'feature': runway.category(choices=tags, default=tags[2]),
'amount': runway.number(default=0.5, min=0, max=1, step=0.1)
}
@runway.command('manipulate',
inputs=command_inputs,
outputs={'output': runway.image})
def detect(sess, inp):
img = np.array(inp['image'])
amount = inp['amount']
feature = inp['feature']
z_addition = amount * z_manipulate[tags.index(feature)]
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
rects = detector(gray, 2)
if len(rects) == 0:
import runway
import numpy
from cnocr import CnOcr
lineCategory = runway.category(
description="Number of lines of characters. Improves accurary.",
choices=['single', 'multiple'],
default='multiple')
@runway.setup(options={'lines': lineCategory})
def setup(opts):
model = CnOcr()
return model.ocr if opts[
'lines'] == 'multiple' else model.ocr_for_single_line
classifyInput = {
'image': runway.image(description="Image with Chinese characters.")
}
classifyOutput = {
'characters':
runway.array(item_type=runway.array(item_type=runway.text),
description="2D character array.")
}
@runway.command('classify', inputs=classifyInput, outputs=classifyOutput)
def classify(model, inputs):
image = numpy.array(inputs['image'])
res = model(image)
})
@runway.command(
'mask_all',
inputs={'image': runway.image},
outputs={'image': runway.segmentation(label_to_id=label_to_id)})
def mask_all(model, inputs):
labelmap = run_model(model, inputs).astype(np.uint8)
return {'image': labelmap}
@runway.command('mask_one',
inputs={
'image': runway.image,
'class': runway.category(choices=classes_list)
},
outputs={'image': runway.image})
def mask_one(model, inputs):
labelmap = run_model(model, inputs)
labelmap = 255.0 * np.array(
labelmap == classes_list.index(inputs['class']))
image_out = np.dstack([labelmap] * 3).astype(np.uint8)
return {'image': image_out}
@runway.command('detect',
inputs={'image': runway.image},
outputs={'classes': runway.array(runway.text)})
def detect(model, inputs):
labelmap = run_model(model, inputs)
import lucid.optvis.render as render
import lucid.optvis.transform as transform
import time
import runway
import random
@runway.setup(options={})
def setup(opts):
model = models.InceptionV1()
model.load_graphdef()
return model
generate_inputs = {
'z': runway.vector(length=1, sampling_std=0.5),
'layer': runway.category(choices=["Mixed3a", "Mixed4a", "Mixed5a"]
, default="Mixed4a"),
'steps': runway.number(min=64, max=512, step=10, default=128)
}
@runway.command('generate', inputs=generate_inputs, outputs={'image': runway.image})
def convert(model, inputs):
print(inputs['z'])
print(inputs['layer'])
num_neurons = {'mixed3a':255, 'mixed4a':507, 'mixed5a':831}
start = time.time()
#set up parameters
layer = inputs['layer'].lower()
neuron = int(np.clip(np.abs(np.float(inputs['z'])) * 1000, 0, num_neurons[layer]))
steps = int(inputs['steps'])
import runway
from runway.data_types import number, text, image
from PIL import Image
import matcap_model
import torch
import numpy
@runway.setup(
options={
"checkpoint":
runway.category(description="Pretrained checkpoints to use.",
choices=['skip'],
default='skip')
})
def setup(opts):
msg = '[SETUP] Running Model'
print(msg)
model = matcap_model.get_generator()
return model
# Every model needs to have at least one command. Every command allows to send
# inputs and process outputs. To see a complete list of supported inputs and
# outputs data types: https://sdk.runwayml.com/en/latest/data_types.html
@runway.command(name='generate',
inputs={'patch': image(width=16, height=16)},
outputs={'image': image(width=16, height=16)})
def generate(model, args):
# Generate a PIL or Numpy image based on the input caption, and return it
img = args['patch']
data, sr = soundfile.read(buffer)
return data, sr
def serialize(self, value):
data, sr = value
buffer = IO()
soundfile.write(buffer, data, sr, format='WAV', subtype='PCM_16')
body = base64.b64encode(buffer.getvalue()).decode('utf8')
return 'data:audio/wav;base64,{body}'.format(body=body)
def to_dict(self):
ret = super(audio, self).to_dict()
ret['channels'] = self.channels
return ret
@runway.setup(options={"stems": runway.category(description="Number of stems to extract",
choices=['2', '4', '5'],
default='2')})
def setup(opts):
return Separator('spleeter:' + opts['stems'] + 'stems')
from spleeter.utils.audio.adapter import get_default_audio_adapter
@runway.command('split', inputs={'audio': audio}, outputs={'vocals': audio})
def synth(separator, inputs):
soundfile.write('temp.wav', inputs['audio'][0], inputs['audio'][1], format='WAV', subtype='PCM_32')
audio_loader = get_default_audio_adapter()
sample_rate = 44100
waveform, _ = audio_loader.load('temp.wav', sample_rate=sample_rate)
prediction = separator.separate(waveform)
return {
# tags that can be modified
tags = "5_o_Clock_Shadow Arched_Eyebrows Attractive Bags_Under_Eyes Bald Bangs Big_Lips Big_Nose Black_Hair Blond_Hair Blurry Brown_Hair Bushy_Eyebrows Chubby Double_Chin Eyeglasses Goatee Gray_Hair Heavy_Makeup High_Cheekbones Male Mouth_Slightly_Open Mustache Narrow_Eyes No_Beard Oval_Face Pale_Skin Pointy_Nose Receding_Hairline Rosy_Cheeks Sideburns Smiling Straight_Hair Wavy_Hair Wearing_Earrings Wearing_Hat Wearing_Lipstick Wearing_Necklace Wearing_Necktie Young"
tags = tags.split()
# launch Runway
model_ready = False
@runway.setup(options={"model_name": runway.text })
def setup():
global model_ready
print('setup model')
model_ready = True
return None
@runway.command('convert', inputs={'image': runway.image, 'feature': runway.category(choices=tags, default=tags[2]), 'amount': runway.number}, outputs={'output': runway.image})
def detect(sess, inp):
img = np.array(inp['image'])
amount = inp['amount']
feature = inp['feature']
z_addition = amount * z_manipulate[tags.index(feature)]
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
rects = detector(gray, 2)
if len(rects) == 0 or not model_ready:
print('nothing found')
return dict(output=img)
img = warper.align(img[:, :, ::-1], gray, rects[0], z_addition)[:, :, ::-1]
img = np.array(Image.fromarray(img).convert('RGB'))
output = np.clip(img, 0, 255).astype(np.uint8)
return dict(output=output)
ctx.step = step
for k, v in metrics.items():
ctx.metrics[k] = v
for k, v in samples.items():
ctx.samples.add(k, v)
for k, v in checkpoints.items():
ctx.checkpoints.add(k, v)
if __name__ == '__main__':
training_sdk.run(train, {
'datasets': {
'images': runway.file(is_directory=True)
},
'options': {
'checkpoint': runway.file(extension='.pkl', description='Model checkpoint to start training from'),
'max_steps': runway.number(default=3000, min=1, max=25000, description='Total number of training iterations'),
'generator_learning_rate': runway.number(default=0.002, min=0.001, max=0.05, description='Generator learning rate'),
'discriminator_learning_rate': runway.number(default=0.002, min=0.001, max=0.05, description='Discriminator learning rate'),
'weight_averaging_half_life': runway.number(default=3, min=0.1, max=10, step=0.1, description='Weight averaging half life'),
'crop_method': runway.category(choices=['Center Crop', 'Random Crop', 'No Crop'], description='Crop method to use during preprocessing'),
},
'samples': {
'generated_images_grid': runway.image,
'interpolation_video': runway.file(extension='.mp4'),
'interpolation_gif': runway.file(extension='.gif')
},
'metrics': {
'FID': runway.number
}
})
#------------------------------------------------------------------------------
# Libraries
#------------------------------------------------------------------------------
import torch, argparse
from models import UNet
import runway
import cv2, torch
import numpy as np
from time import time
from torch.nn import functional as F
from PIL import Image
from dataloaders import transforms
from utils import utils
cat = runway.category(choices=["mobilenetv2", "resnet18"], default="mobilenetv2")
@runway.setup(options={"backbone" : cat, "checkpoint" : runway.file(extension=".pth")})
def setup(opts):
model = UNet(backbone=opts["backbone"], num_classes=2)
if torch.cuda.is_available():
print("Using CUDA")
trained_dict = torch.load(opts["checkpoint"])['state_dict']
model.load_state_dict(trained_dict, strict=False)
model.cuda()
else:
print("Using CPU")
trained_dict = torch.load(opts["checkpoint"], map_location="cpu")['state_dict']
model.load_state_dict(trained_dict, strict=False)
return model
encode_images.main()
@runway.setup(options={'checkpoint': runway.file(extension='.pkl')})
def setup(opts):
global Gs
tflib.init_tf()
with open(opts['checkpoint'], 'rb') as file:
G, D, Gs = pickle.load(file)
return Gs
generate_inputs = {
'image': runway.image,
'coeff': runway.number(min=-20, max=20, default=0.0, step=0.01)
}
@runway.command('convert', inputs=generate_inputs, 'feature':runway.category(choices=tags,default=tags[1]), outputs={'image': runway.image})
def convert(model, inputs):
img = np.array(inputs['image'])
coeff = inputs['coeff']
z_addition = coeff * z_manipulate[tags.index(feature)]
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
if len(rects) == 0 or not model_ready:
print('nothing found')
return dict(output=img)
img = align.main()
encoded = encoder.main()
img = np.array(Image.fromarray(img).convert('RGB'))
output = np.clip(img, 0, 255).astype(np.uint8)
return {'image': output}
return new_latent_vector
@runway.setup(options={'checkpoint': runway.file(extension='.pth')})
def setup(opts):
global Gs
if opts['checkpoint'] is None or opts['checkpoint'] == '':
opts['checkpoint'] = 'checkpoint/Gs.pth'
state = torch.load(opts['checkpoint'], map_location=device)
Gs = models.load(state, device)
Gs.to(device)
return Gs
generate_inputs = {
'z': runway.vector(512, sampling_std=0.5),
'truncation': runway.number(min=0, max=1, default=0.8, step=0.01),
'hair_color': runway.category(choices=hairs),
'hair_coeff': runway.number(min=-5, max=5, default=0, step=0.01),
'eyes_color': runway.category(choices=eyes),
'eyes_coeff': runway.number(min=-5, max=5, default=0, step=0.01),
}
@runway.command('generate', inputs=generate_inputs, outputs={'image': runway.image})
def convert(model, inputs):
truncation = inputs['truncation']
Gs.set_truncation(truncation_psi=truncation)
qlatents = torch.Tensor(inputs['z']).reshape(1, 512).to(device=device, dtype=torch.float32)
dlatents = Gs.G_mapping(qlatents)
swifted_dlatents = shift_latents(dlatents, inputs)
generated = Gs(dlatents=swifted_dlatents)
images = utils.tensor_to_PIL(generated)
return {'image': images[0]}
result = result - vec
if plus_vecs:
for i, vec in enumerate(plus_vecs):
result = result + vec
excluded_words = [start_word] + minus_words + plus_words
neighbors = find_nearest(excluded_words, result, id_to_word, faiss_index,
num_results)
return ', '.join(neighbors)
setup_options = {
'word_vector_dimensions': category(
choices=['50', '100', '200', '300'],
default='100',
description='The number of dimensions used to represent each word in the latent '\
'space. Higher dimensions increase accuracy but take longer to run and use '\
'more memory.'
),
'number_of_words': number(
min=100,
max=400000,
default=400000,
description='The number of words in the corpus. More words create more variety '\
' but take longer to run.'
)
}
@runway.setup(options=setup_options)
def setup(opts):
# for img_array, dlatent, img_name in zip(generated_images, generated_dlatents, names):
# img = PIL.Image.fromarray(img_array, 'RGB')
# img.resize((512, 512))
# print(latent_vectors)
inputs['encode'] = False
encodeCount += 1
else:
print("Did not encode.")
return {"image": image}
# GENERATION
cat_1 = runway.category(choices=["1", "2", "3", "4"], default="1")
cat_2 = runway.category(choices=["1", "2", "3", "4"], default="1")
generate_inputs_2 = {
'person_1': cat_1,
'person_2': cat_2,
'mix': runway.number(min=0.0, max=100.0, default=50.0, step=1.0),
'age': runway.number(min=-10.0, max=10.0, default=5.0, step=0.1),
'fine_age': runway.number(min=-1.0, max=1.0, default=0.0, step=0.01),
'smile': runway.number(min=-5.0, max=5.0, default=0, step=0.1),
'gender': runway.number(min=-5.0, max=5.0, default=0, step=0.1),
'glitch': runway.number(min=-1.0, max=5.0, default=1.0, step=0.1)
}
generate_outputs_2 = {'image': runway.image(width=512, height=512)}
import runway
import numpy as np
from cnocr import CnOcr
@runway.setup(options={
"network_size":
runway.category(choices=['64', '128', '256', '512'])
})
def setup(opts):
model = CnOcr()
return model
@runway.command('classify',
inputs={'image': runway.image},
outputs={'text': runway.text})
def classify(model, inputs):
image = np.array(inputs['image'])
res = model.ocr(image)
return {'text': res}
if __name__ == '__main__':
runway.run(host='0.0.0.0', port=9000)
model_dict = model.state_dict()
# because the model is trained by multiple gpus, prefix module should be removed
for k in checkpoint.keys():
model_dict[k.replace('module.', '')] = checkpoint[k]
model.load_state_dict(model_dict)
if mode == 'gpu':
cudnn.benchmark = True
model = model.cuda()
return model.eval()
inputs = {
'content_image':
runway.image,
'output_type':
runway.category(choices=['pose', 'depth', 'points', 'skin'],
default='pose')
}
@runway.command('classify', inputs=inputs, outputs={'image': image})
def classify(model, inputs):
in_img = inputs['content_image']
img_ori = np.array(in_img)
img_fp = 'samples/test1.jpg'
face_detector = dlib.get_frontal_face_detector()
# 3. forward
tri = sio.loadmat('visualize/tri.mat')['tri']
transform = transforms.Compose(
[ToTensorGjz(), NormalizeGjz(mean=127.5, std=128)])
