import librosa import argparse import numpy as np import moviepy.editor as mpy import random import torch import runway import numpy as np from scipy.misc import toimage from tqdm import tqdm from pytorch_pretrained_biggan import (BigGAN, one_hot_from_names, truncated_noise_sample, save_as_images, display_in_terminal) @runway.setup(options={"checkpoint": runway.category(description="Pretrained checkpoints to use.", choices=['celebAHQ-512', 'celebAHQ-256', 'celeba'], default='celebAHQ-512')}) def setup(opts): checkpoint = opts['checkpoint'] 'PGAN', model_name=checkpoint, return model @runway.command('generate', inputs={ 'z': runway.vector(length=512, sampling_std=0.5)}, outputs={ 'image': runway.image }) def generate(model, inputs): # Generate ♾ infinite ♾ images z = inputs['z'] latents = z.reshape((1, 559)) latents = torch.from_numpy(latents) - noise, _ = model.buildNoiseData(1) with torch.no_grad():
# load face detection and warping detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat') warper = FaceWarper(predictor, desiredFaceWidth=256, desiredLeftEye=(0.371, 0.480)) # tags that can be modified tags = "5_o_Clock_Shadow Arched_Eyebrows Attractive Bags_Under_Eyes Bald Bangs Big_Lips Big_Nose Black_Hair Blond_Hair Blurry Brown_Hair Bushy_Eyebrows Chubby Double_Chin Eyeglasses Goatee Gray_Hair Heavy_Makeup High_Cheekbones Male Mouth_Slightly_Open Mustache Narrow_Eyes No_Beard Oval_Face Pale_Skin Pointy_Nose Receding_Hairline Rosy_Cheeks Sideburns Smiling Straight_Hair Wavy_Hair Wearing_Earrings Wearing_Hat Wearing_Lipstick Wearing_Necklace Wearing_Necktie Young" tags = tags.split() command_inputs = { 'image': runway.image, 'feature': runway.category(choices=tags, default=tags[2]), 'amount': runway.number(default=0.5, min=0, max=1, step=0.1) } @runway.command('manipulate', inputs=command_inputs, outputs={'output': runway.image}) def detect(sess, inp): img = np.array(inp['image']) amount = inp['amount'] feature = inp['feature'] z_addition = amount * z_manipulate[tags.index(feature)] gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) rects = detector(gray, 2) if len(rects) == 0:
import runway import numpy from cnocr import CnOcr lineCategory = runway.category( description="Number of lines of characters. Improves accurary.", choices=['single', 'multiple'], default='multiple') @runway.setup(options={'lines': lineCategory}) def setup(opts): model = CnOcr() return model.ocr if opts[ 'lines'] == 'multiple' else model.ocr_for_single_line classifyInput = { 'image': runway.image(description="Image with Chinese characters.") } classifyOutput = { 'characters': runway.array(item_type=runway.array(item_type=runway.text), description="2D character array.") } @runway.command('classify', inputs=classifyInput, outputs=classifyOutput) def classify(model, inputs): image = numpy.array(inputs['image']) res = model(image)
}) @runway.command( 'mask_all', inputs={'image': runway.image}, outputs={'image': runway.segmentation(label_to_id=label_to_id)}) def mask_all(model, inputs): labelmap = run_model(model, inputs).astype(np.uint8) return {'image': labelmap} @runway.command('mask_one', inputs={ 'image': runway.image, 'class': runway.category(choices=classes_list) }, outputs={'image': runway.image}) def mask_one(model, inputs): labelmap = run_model(model, inputs) labelmap = 255.0 * np.array( labelmap == classes_list.index(inputs['class'])) image_out = np.dstack([labelmap] * 3).astype(np.uint8) return {'image': image_out} @runway.command('detect', inputs={'image': runway.image}, outputs={'classes': runway.array(runway.text)}) def detect(model, inputs): labelmap = run_model(model, inputs)
import lucid.optvis.render as render import lucid.optvis.transform as transform import time import runway import random @runway.setup(options={}) def setup(opts): model = models.InceptionV1() model.load_graphdef() return model generate_inputs = { 'z': runway.vector(length=1, sampling_std=0.5), 'layer': runway.category(choices=["Mixed3a", "Mixed4a", "Mixed5a"] , default="Mixed4a"), 'steps': runway.number(min=64, max=512, step=10, default=128) } @runway.command('generate', inputs=generate_inputs, outputs={'image': runway.image}) def convert(model, inputs): print(inputs['z']) print(inputs['layer']) num_neurons = {'mixed3a':255, 'mixed4a':507, 'mixed5a':831} start = time.time() #set up parameters layer = inputs['layer'].lower() neuron = int(np.clip(np.abs(np.float(inputs['z'])) * 1000, 0, num_neurons[layer])) steps = int(inputs['steps'])
import runway from runway.data_types import number, text, image from PIL import Image import matcap_model import torch import numpy @runway.setup( options={ "checkpoint": runway.category(description="Pretrained checkpoints to use.", choices=['skip'], default='skip') }) def setup(opts): msg = '[SETUP] Running Model' print(msg) model = matcap_model.get_generator() return model # Every model needs to have at least one command. Every command allows to send # inputs and process outputs. To see a complete list of supported inputs and # outputs data types: https://sdk.runwayml.com/en/latest/data_types.html @runway.command(name='generate', inputs={'patch': image(width=16, height=16)}, outputs={'image': image(width=16, height=16)}) def generate(model, args): # Generate a PIL or Numpy image based on the input caption, and return it img = args['patch']
data, sr = soundfile.read(buffer) return data, sr def serialize(self, value): data, sr = value buffer = IO() soundfile.write(buffer, data, sr, format='WAV', subtype='PCM_16') body = base64.b64encode(buffer.getvalue()).decode('utf8') return 'data:audio/wav;base64,{body}'.format(body=body) def to_dict(self): ret = super(audio, self).to_dict() ret['channels'] = self.channels return ret @runway.setup(options={"stems": runway.category(description="Number of stems to extract", choices=['2', '4', '5'], default='2')}) def setup(opts): return Separator('spleeter:' + opts['stems'] + 'stems') from spleeter.utils.audio.adapter import get_default_audio_adapter @runway.command('split', inputs={'audio': audio}, outputs={'vocals': audio}) def synth(separator, inputs): soundfile.write('temp.wav', inputs['audio'][0], inputs['audio'][1], format='WAV', subtype='PCM_32') audio_loader = get_default_audio_adapter() sample_rate = 44100 waveform, _ = audio_loader.load('temp.wav', sample_rate=sample_rate) prediction = separator.separate(waveform) return {
# tags that can be modified tags = "5_o_Clock_Shadow Arched_Eyebrows Attractive Bags_Under_Eyes Bald Bangs Big_Lips Big_Nose Black_Hair Blond_Hair Blurry Brown_Hair Bushy_Eyebrows Chubby Double_Chin Eyeglasses Goatee Gray_Hair Heavy_Makeup High_Cheekbones Male Mouth_Slightly_Open Mustache Narrow_Eyes No_Beard Oval_Face Pale_Skin Pointy_Nose Receding_Hairline Rosy_Cheeks Sideburns Smiling Straight_Hair Wavy_Hair Wearing_Earrings Wearing_Hat Wearing_Lipstick Wearing_Necklace Wearing_Necktie Young" tags = tags.split() # launch Runway model_ready = False @runway.setup(options={"model_name": runway.text }) def setup(): global model_ready print('setup model') model_ready = True return None @runway.command('convert', inputs={'image': runway.image, 'feature': runway.category(choices=tags, default=tags[2]), 'amount': runway.number}, outputs={'output': runway.image}) def detect(sess, inp): img = np.array(inp['image']) amount = inp['amount'] feature = inp['feature'] z_addition = amount * z_manipulate[tags.index(feature)] gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) rects = detector(gray, 2) if len(rects) == 0 or not model_ready: print('nothing found') return dict(output=img) img = warper.align(img[:, :, ::-1], gray, rects[0], z_addition)[:, :, ::-1] img = np.array(Image.fromarray(img).convert('RGB')) output = np.clip(img, 0, 255).astype(np.uint8) return dict(output=output)
ctx.step = step for k, v in metrics.items(): ctx.metrics[k] = v for k, v in samples.items(): ctx.samples.add(k, v) for k, v in checkpoints.items(): ctx.checkpoints.add(k, v) if __name__ == '__main__': training_sdk.run(train, { 'datasets': { 'images': runway.file(is_directory=True) }, 'options': { 'checkpoint': runway.file(extension='.pkl', description='Model checkpoint to start training from'), 'max_steps': runway.number(default=3000, min=1, max=25000, description='Total number of training iterations'), 'generator_learning_rate': runway.number(default=0.002, min=0.001, max=0.05, description='Generator learning rate'), 'discriminator_learning_rate': runway.number(default=0.002, min=0.001, max=0.05, description='Discriminator learning rate'), 'weight_averaging_half_life': runway.number(default=3, min=0.1, max=10, step=0.1, description='Weight averaging half life'), 'crop_method': runway.category(choices=['Center Crop', 'Random Crop', 'No Crop'], description='Crop method to use during preprocessing'), }, 'samples': { 'generated_images_grid': runway.image, 'interpolation_video': runway.file(extension='.mp4'), 'interpolation_gif': runway.file(extension='.gif') }, 'metrics': { 'FID': runway.number } })
#------------------------------------------------------------------------------ # Libraries #------------------------------------------------------------------------------ import torch, argparse from models import UNet import runway import cv2, torch import numpy as np from time import time from torch.nn import functional as F from PIL import Image from dataloaders import transforms from utils import utils cat = runway.category(choices=["mobilenetv2", "resnet18"], default="mobilenetv2") @runway.setup(options={"backbone" : cat, "checkpoint" : runway.file(extension=".pth")}) def setup(opts): model = UNet(backbone=opts["backbone"], num_classes=2) if torch.cuda.is_available(): print("Using CUDA") trained_dict = torch.load(opts["checkpoint"])['state_dict'] model.load_state_dict(trained_dict, strict=False) model.cuda() else: print("Using CPU") trained_dict = torch.load(opts["checkpoint"], map_location="cpu")['state_dict'] model.load_state_dict(trained_dict, strict=False) return model
encode_images.main() @runway.setup(options={'checkpoint': runway.file(extension='.pkl')}) def setup(opts): global Gs tflib.init_tf() with open(opts['checkpoint'], 'rb') as file: G, D, Gs = pickle.load(file) return Gs generate_inputs = { 'image': runway.image, 'coeff': runway.number(min=-20, max=20, default=0.0, step=0.01) } @runway.command('convert', inputs=generate_inputs, 'feature':runway.category(choices=tags,default=tags[1]), outputs={'image': runway.image}) def convert(model, inputs): img = np.array(inputs['image']) coeff = inputs['coeff'] z_addition = coeff * z_manipulate[tags.index(feature)] gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) if len(rects) == 0 or not model_ready: print('nothing found') return dict(output=img) img = align.main() encoded = encoder.main() img = np.array(Image.fromarray(img).convert('RGB')) output = np.clip(img, 0, 255).astype(np.uint8) return {'image': output}
return new_latent_vector @runway.setup(options={'checkpoint': runway.file(extension='.pth')}) def setup(opts): global Gs if opts['checkpoint'] is None or opts['checkpoint'] == '': opts['checkpoint'] = 'checkpoint/Gs.pth' state = torch.load(opts['checkpoint'], map_location=device) Gs = models.load(state, device) Gs.to(device) return Gs generate_inputs = { 'z': runway.vector(512, sampling_std=0.5), 'truncation': runway.number(min=0, max=1, default=0.8, step=0.01), 'hair_color': runway.category(choices=hairs), 'hair_coeff': runway.number(min=-5, max=5, default=0, step=0.01), 'eyes_color': runway.category(choices=eyes), 'eyes_coeff': runway.number(min=-5, max=5, default=0, step=0.01), } @runway.command('generate', inputs=generate_inputs, outputs={'image': runway.image}) def convert(model, inputs): truncation = inputs['truncation'] Gs.set_truncation(truncation_psi=truncation) qlatents = torch.Tensor(inputs['z']).reshape(1, 512).to(device=device, dtype=torch.float32) dlatents = Gs.G_mapping(qlatents) swifted_dlatents = shift_latents(dlatents, inputs) generated = Gs(dlatents=swifted_dlatents) images = utils.tensor_to_PIL(generated) return {'image': images[0]}
result = result - vec if plus_vecs: for i, vec in enumerate(plus_vecs): result = result + vec excluded_words = [start_word] + minus_words + plus_words neighbors = find_nearest(excluded_words, result, id_to_word, faiss_index, num_results) return ', '.join(neighbors) setup_options = { 'word_vector_dimensions': category( choices=['50', '100', '200', '300'], default='100', description='The number of dimensions used to represent each word in the latent '\ 'space. Higher dimensions increase accuracy but take longer to run and use '\ 'more memory.' ), 'number_of_words': number( min=100, max=400000, default=400000, description='The number of words in the corpus. More words create more variety '\ ' but take longer to run.' ) } @runway.setup(options=setup_options) def setup(opts):
# for img_array, dlatent, img_name in zip(generated_images, generated_dlatents, names): # img = PIL.Image.fromarray(img_array, 'RGB') # img.resize((512, 512)) # print(latent_vectors) inputs['encode'] = False encodeCount += 1 else: print("Did not encode.") return {"image": image} # GENERATION cat_1 = runway.category(choices=["1", "2", "3", "4"], default="1") cat_2 = runway.category(choices=["1", "2", "3", "4"], default="1") generate_inputs_2 = { 'person_1': cat_1, 'person_2': cat_2, 'mix': runway.number(min=0.0, max=100.0, default=50.0, step=1.0), 'age': runway.number(min=-10.0, max=10.0, default=5.0, step=0.1), 'fine_age': runway.number(min=-1.0, max=1.0, default=0.0, step=0.01), 'smile': runway.number(min=-5.0, max=5.0, default=0, step=0.1), 'gender': runway.number(min=-5.0, max=5.0, default=0, step=0.1), 'glitch': runway.number(min=-1.0, max=5.0, default=1.0, step=0.1) } generate_outputs_2 = {'image': runway.image(width=512, height=512)}
import runway import numpy as np from cnocr import CnOcr @runway.setup(options={ "network_size": runway.category(choices=['64', '128', '256', '512']) }) def setup(opts): model = CnOcr() return model @runway.command('classify', inputs={'image': runway.image}, outputs={'text': runway.text}) def classify(model, inputs): image = np.array(inputs['image']) res = model.ocr(image) return {'text': res} if __name__ == '__main__': runway.run(host='0.0.0.0', port=9000)
model_dict = model.state_dict() # because the model is trained by multiple gpus, prefix module should be removed for k in checkpoint.keys(): model_dict[k.replace('module.', '')] = checkpoint[k] model.load_state_dict(model_dict) if mode == 'gpu': cudnn.benchmark = True model = model.cuda() return model.eval() inputs = { 'content_image': runway.image, 'output_type': runway.category(choices=['pose', 'depth', 'points', 'skin'], default='pose') } @runway.command('classify', inputs=inputs, outputs={'image': image}) def classify(model, inputs): in_img = inputs['content_image'] img_ori = np.array(in_img) img_fp = 'samples/test1.jpg' face_detector = dlib.get_frontal_face_detector() # 3. forward tri = sio.loadmat('visualize/tri.mat')['tri'] transform = transforms.Compose( [ToTensorGjz(), NormalizeGjz(mean=127.5, std=128)])