print('Restored from {}'.format(ckpt.model_checkpoint_path)) return sess def convert_rect(rect, width, height): x = rect[0] / width y = rect[1] / height w = rect[2] / width h = rect[3] / height return x, y, w, h caption_inputs = { 'image': runway.image, 'max_detections': runway.number(default=10, min=1, max=50, step=1) } caption_outputs = { 'bboxes': runway.array(runway.image_bounding_box), 'classes': runway.array(runway.text), 'scores': runway.array(runway.number) } @runway.command('caption', inputs=caption_inputs, outputs=caption_outputs) def caption(sess, inp): img = np.array(inp['image']) width = img.shape[1] height = img.shape[0] scores, boxes, captions = im_detect(sess, net, img, None, use_box_at=-1)
import dnnlib.tflib as tflib import config import matplotlib.pyplot as plt from encoder.generator_model import Generator from encoder.perceptual_model import PerceptualModel import runway prevIterations = -1 generated_dlatents = 0 @runway.setup( options={ 'checkpoint': runway.file(extension='.pkl'), 'image dimensions': runway.number(min=128, max=1024, default=512, step=128) }) def setup(opts): # Initialize generator and perceptual model global perceptual_model global generator tflib.init_tf() model = opts['checkpoint'] print("open model %s" % model) with open(model, 'rb') as file: G, D, Gs = pickle.load(file) Gs.print_layers() generator = Generator(Gs, batch_size=1, randomize_noise=False) perceptual_model = PerceptualModel(opts['image dimensions'], layer=9,
new_latent_vector[:8] = (latent_vector + coeff*direction)[:8] return new_latent_vector @runway.setup(options={'checkpoint': runway.file(extension='.pth')}) def setup(opts): global Gs if opts['checkpoint'] is None or opts['checkpoint'] == '': opts['checkpoint'] = 'checkpoint/Gs.pth' state = torch.load(opts['checkpoint'], map_location=device) Gs = models.load(state, device) Gs.to(device) return Gs generate_inputs = { 'z': runway.vector(512, sampling_std=0.5), 'truncation': runway.number(min=0, max=1, default=0.8, step=0.01), 'hair_color': runway.category(choices=hairs), 'hair_coeff': runway.number(min=-5, max=5, default=0, step=0.01), 'eyes_color': runway.category(choices=eyes), 'eyes_coeff': runway.number(min=-5, max=5, default=0, step=0.01), } @runway.command('generate', inputs=generate_inputs, outputs={'image': runway.image}) def convert(model, inputs): truncation = inputs['truncation'] Gs.set_truncation(truncation_psi=truncation) qlatents = torch.Tensor(inputs['z']).reshape(1, 512).to(device=device, dtype=torch.float32) dlatents = Gs.G_mapping(qlatents) swifted_dlatents = shift_latents(dlatents, inputs) generated = Gs(dlatents=swifted_dlatents) images = utils.tensor_to_PIL(generated)
tags = "Neutral Smiling" tags = tags.split() align_images.main() encode_images.main() @runway.setup(options={'checkpoint': runway.file(extension='.pkl')}) def setup(opts): global Gs tflib.init_tf() with open(opts['checkpoint'], 'rb') as file: G, D, Gs = pickle.load(file) return Gs generate_inputs = { 'image': runway.image, 'coeff': runway.number(min=-20, max=20, default=0.0, step=0.01) } @runway.command('convert', inputs=generate_inputs, 'feature':runway.category(choices=tags,default=tags[1]), outputs={'image': runway.image}) def convert(model, inputs): img = np.array(inputs['image']) coeff = inputs['coeff'] z_addition = coeff * z_manipulate[tags.index(feature)] gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) if len(rects) == 0 or not model_ready: print('nothing found') return dict(output=img) img = align.main() encoded = encoder.main() img = np.array(Image.fromarray(img).convert('RGB')) output = np.clip(img, 0, 255).astype(np.uint8)
def setup(opts): global Gs tflib.init_tf() if opts['checkpoint'] is None: opts['checkpoint'] = 'checkpoints\\network-snapshot-000600.pkl' with open(opts['checkpoint'], 'rb') as file: _G, _D, Gs = pickle.load(file, encoding='latin1') noise_vars = [var for name, var in Gs.components.synthesis.vars.items() if name.startswith('noise')] rnd = np.random.RandomState() tflib.set_vars({var: rnd.randn(*var.shape.as_list()) for var in noise_vars}) return Gs generate_inputs = { 'z': runway.vector(256, sampling_std=0.5), 'truncation': runway.number(min=0, max=1, default=0.8, step=0.01) } for i in range(4): for j in range(4): generate_inputs.update({'style_%i/%i' %(i, j): runway.number(min=0, max=1, default=0.5, step=0.01)}) @runway.command('generate', inputs=generate_inputs, outputs={'image': runway.image}) def convert(model, inputs): z = inputs['z'] truncation = inputs['truncation'] latent = z.reshape((1, 256)) style_mix = np.zeros([4, 4]) for i in range(4): for j in range(4): style_mix[i, j] = inputs['style_%i/%i' %(i, j)]
with open(opts['checkpoint'], 'rb') as file: _G, _D, Gs = pickle.load(file, encoding='latin1') noise_vars = [ var for name, var in Gs.components.synthesis.vars.items() if name.startswith('noise') ] rnd = np.random.RandomState() tflib.set_vars( {var: rnd.randn(*var.shape.as_list()) for var in noise_vars}) return Gs generate_inputs = { 'z': runway.vector(512, sampling_std=0.5), 'truncation': runway.number(min=0, max=1, default=0.8, step=0.01) } @runway.command('generate', inputs=generate_inputs, outputs={'image': runway.image}) def convert(model, inputs): z = inputs['z'] truncation = inputs['truncation'] latents = z.reshape((1, 512)) images = model.run(latents, None, truncation_psi=truncation, randomize_noise=False, output_transform=fmt)
# return Gs def generate_image(generator, latent_vector): latent_vector = latent_vector.reshape((1, 18, 512)) generator.set_dlatents(latent_vector) img_array = generator.generate_images()[0] img = PIL.Image.fromarray(img_array, 'RGB') return img.resize((512, 512)) # ENCODING generate_inputs_1 = { 'portrait': runway.image(), 'iterations': runway.number(min=1, max=5000, default=10, step=1.0), 'encode': runway.boolean(default=False) } generate_outputs_1 = {'image': runway.image(width=512, height=512)} encodeCount = 0 latent_vectors = [] # latent_vectors.append(preLoad1) # latent_vectors.append(preLoad2) # latent_vectors.append(preLoad1) # latent_vectors.append(preLoad2) @runway.command('encode', inputs=generate_inputs_1, outputs=generate_outputs_1) def find_in_space(model, inputs): global generated_dlatents
# # Initialize generator and perceptual model # tflib.init_tf() # model = opts['checkpoint'] # print("open model %s" % model) # with open(model, 'rb') as file: # G, D, Gs = pickle.load(file) # Gs.print_layers() # global generator # generator = Generator(Gs, batch_size=1, randomize_noise=False) # perceptual_model = PerceptualModel(256, layer=9, batch_size=1) # perceptual_model.build_perceptual_model(generator.generated_image) # return Gs generate_inputs = { 'portrait': runway.image(), 'iterations': runway.number(min=1, max=5000, default=10, step=1.0), 'age': runway.number(min=-30, max=20, default=4, step=0.2) } # generate_outputs = { # 'latent_vector': runway.file() # } generate_outputs = { # 'generated': runway.image(width=512, height=512) # 'vector': runway.vector(length=512) "hextext": runway.text } @runway.command('encode', inputs=generate_inputs, outputs=generate_outputs) def find_in_space(model, inputs):
inputs = { 'composite_image': runway.image, 'foreground_mask': runway.image, 'transfer_color': runway.boolean( default=True, description= "Transfer colors back to source image for high-resolution output"), 'transfer_resolution': runway.number(default=512, min=256, max=1024, step=128, description="Which resolution to transfer colors from") } outputs = {'harmonized_image': runway.image} @runway.command('harmonize', inputs=inputs, outputs=outputs) def harmonize(model, inputs): og_image = np.array(inputs["composite_image"]) og_mask = np.array(inputs["foreground_mask"]) # Re-shape inputs to transfer resolution image_size = og_image.shape[:2]
class BaseConfig(Config): NAME = "baseConfig" GPU_COUNT = 1 IMAGES_PER_GPU = 1 NUM_CLASSES = 1 + 1 # Background + pasta STEPS_PER_EPOCH = 100 DETECTION_MIN_CONFIDENCE = 0.7 config = BaseConfig() setup_options = { 'checkpoint': runway.file(extension='.h5'), 'min_confidence': runway.number(min=0, max=1, step=.1, default=.7), } @runway.setup(options=setup_options) def setup(opts): config.DETECTION_MIN_CONFIDENCE = opts['min_confidence'] model = modellib.MaskRCNN(mode="inference", config=config, model_dir='logs') model.load_weights(opts['checkpoint'], by_name=True) return model @runway.command('detect', inputs={'image': runway.image},
context=context, batch_size=1, temperature=temperature, top_k=top_k) saver = tf.train.Saver() ckpt = tf.train.latest_checkpoint(opts['checkpoint_dir']) saver.restore(sess, ckpt) g = tf.get_default_graph() g.finalize() return sess, enc, context, length_ph command_inputs = { 'prompt': runway.text, 'seed': runway.number(default=0, min=0, max=999, step=1), 'sequence_length': runway.number(default=128, min=1, max=256, step=1) } @runway.command('generate', inputs=command_inputs, outputs={'text': runway.text}) def generate(model, inputs): with g.as_default(): sess, enc, context, length_ph = model seed = inputs['seed'] np.random.seed(seed) tf.set_random_seed(seed) context_tokens = enc.encode(inputs['prompt']) out = sess.run(output,
import pickle from tqdm import tqdm import PIL.Image import numpy as np import dnnlib import dnnlib.tflib as tflib import config import matplotlib.pyplot as plt from encoder.generator_model import Generator from encoder.perceptual_model import PerceptualModel import runway prevIterations = -1 generated_dlatents = 0 @runway.setup(options={'checkpoint': runway.file(extension='.pkl'), 'image dimensions': runway.number(min=128, max=1024, default=512, step=128)}) def setup(opts): # Initialize generator and perceptual model global perceptual_model global generator tflib.init_tf() model = opts['checkpoint'] print("open model %s" % model) with open(model, 'rb') as file: G, D, Gs = pickle.load(file) Gs.print_layers() generator = Generator(Gs, batch_size=1, randomize_noise=False) perceptual_model = PerceptualModel(opts['image dimensions'], layer=9, batch_size=1) perceptual_model.build_perceptual_model(generator.generated_image) return Gs
# generator = Generator(Gs, batch_size=1, randomize_noise=False) # return Gs def generate_image(generator, latent_vector): latent_vector = latent_vector.reshape((1, 18, 512)) generator.set_dlatents(latent_vector) img_array = generator.generate_images()[0] img = PIL.Image.fromarray(img_array, 'RGB') return img.resize((512, 512)) # ENCODING generate_inputs_1 = { 'portrait': runway.image(), 'iterations': runway.number(min=1, max=5000, default=10, step=1.0) } generate_outputs_1 = { "text": runway.text } encodeCount = 0 latent_vectors = [] @runway.command('encode', inputs=generate_inputs_1, outputs=generate_outputs_1) def find_in_space(model, inputs): global generated_dlatents global prevIterations s2 = "Did not encode." if (inputs['iterations'] != prevIterations): generator.reset_dlatents()
pt.net_G.out_size).to(device) pt._save_stroke_params(PARAMS) final_rendered_image = pt._render(PARAMS, save_jpgs=False, save_video=True) return final_rendered_image @runway.command( "translate", inputs={ "source_imgs": runway.image(description="input image to be translated"), "Strokes": runway.number(min=100, max=700, default=100, description="number of strokes"), }, outputs={ "image": runway.image( description="output image containing the translated result") }, ) def translate(learn, inputs): os.makedirs("images", exist_ok=True) inputs["source_imgs"].save("images/temp.jpg") paths = os.path.join("images", "temp.jpg") args.img_path = paths args.max_m_strokes = inputs["Strokes"] pt = ProgressivePainter(args=args)
@runway.setup def setup(): init_fn = slim.assign_from_checkpoint_fn( './arbitrary_style_transfer/model.ckpt', slim.get_variables_to_restore() ) sess.run([tf.local_variables_initializer()]) init_fn(sess) return sess stylize_inputs = { 'content_image': runway.image, 'style_image': runway.image, 'interpolation_weight': runway.number(default=0.5, min=0, max=1, step=0.1) } @runway.command('stylize', inputs=stylize_inputs, outputs={'image': runway.image}) def stylize(sess, inputs): content_img_np = np.array(inputs['content_image']) style_image_np = np.array(inputs['style_image']) interpolation_weight = inputs['interpolation_weight'] identity_params = sess.run(bottleneck_feat, feed_dict={style_img_ph: content_img_np}) style_params = sess.run(bottleneck_feat, feed_dict={style_img_ph: style_image_np}) stylized_image_res = sess.run( stylized_images, feed_dict={ bottleneck_feat: identity_params * (1 - interpolation_weight) + style_params * interpolation_weight, content_img_ph: content_img_np })
params = argparse.Namespace(**importlib.import_module(config_file).params) model = BPEmbVaeSampler(lang=params.bpemb['lang'], vs=params.bpemb['vs'], dim=params.bpemb['dim'], decode_from=model_path, params=params, cuda=use_gpu) return model @runway.command('generate', inputs={ 'z': runway.vector(length=64), 'temperature': runway.number(default=0.5, min=0.05, max=2.0, step=0.05) }, outputs={'out': runway.text}) def generate(model, inputs): z = torch.from_numpy(inputs['z']).float().unsqueeze(0).to(model.device) temperature = inputs['temperature'] with torch.no_grad(): return model.sample(z, temperature)[0] @runway.command('reconstruct', inputs={ 'in': runway.text, 'temperature': runway.number(default=0.5, min=0.05, max=2.0, step=0.05)
neighbors = find_nearest(excluded_words, result, id_to_word, faiss_index, num_results) return ', '.join(neighbors) setup_options = { 'word_vector_dimensions': category( choices=['50', '100', '200', '300'], default='100', description='The number of dimensions used to represent each word in the latent '\ 'space. Higher dimensions increase accuracy but take longer to run and use '\ 'more memory.' ), 'number_of_words': number( min=100, max=400000, default=400000, description='The number of words in the corpus. More words create more variety '\ ' but take longer to run.' ) } @runway.setup(options=setup_options) def setup(opts): dimensions = int(opts['word_vector_dimensions']) vector_file = 'data/glove/glove.6B.{}d.txt'.format(dimensions) df, labels_array = build_word_vector_matrix(vector_file, opts['number_of_words']) word_to_id, id_to_word = get_label_dictionaries(labels_array) faiss_index = faiss.IndexFlatL2(dimensions) faiss_index.add(df)
global Gs print('init') tflib.init_tf() print('reading pickle file') with open(opts['checkpoint'], 'rb') as file: _G, _D, Gs = pickle.load(file, encoding='latin1') print('Create noise') noise_vars = [var for name, var in Gs.components.synthesis.vars.items() if name.startswith('noise')] rnd = np.random.RandomState() tflib.set_vars({var: rnd.randn(*var.shape.as_list()) for var in noise_vars}) return Gs generate_inputs = { 'z': runway.vector(512, sampling_std=0.5), 'label': runway.number(min=0, max=100000, default=0, step=1), # generate random labels 'scale': runway.number(min=-2, max=2, default=0, step=0.05), # magnitude of labels - 0 = no labels 'truncation': runway.number(min=0, max=1, default=1, step=0.1) } @runway.command('generate', inputs=generate_inputs, outputs={'image': runway.image}) def convert(model, inputs): z = inputs['z'] print(z) label = int(inputs['label']) scale = inputs['scale'] truncation = inputs['truncation'] latents = z.reshape((1, 512)) labels = scale * np.random.RandomState(label).randn(167) labels = labels.reshape((1,167)).astype(np.float32) images = model.run(latents, labels, truncation_psi=truncation, randomize_noise=False, output_transform=fmt)
tflib.init_tf() with open(opts['checkpoint'], 'rb') as file: _, _, Gs = pickle.load(file, encoding='latin1') return Gs #@runway.command('project', inputs=project_inputs, outputs={'images': runway.array(item_type=runway.image, max_length=10)}) @runway.command('project', inputs={ 'projectionImage': runway.image(min_width=1024, min_height=1024, max_width=1024, max_height=1024), 'steps': runway.number(min=10, max=1000, default=200) }, outputs={'image': runway.image}) def project(model, inputs): im = inputs['projectionImage'] if not os.path.exists('./projection'): os.mkdir('./projection') if not os.path.exists('./projection/imgs'): os.mkdir('./projection/imgs') if not os.path.exists('./projection/records'): os.mkdir('./projection/records') if not os.path.exists('./projection/out'): os.mkdir('./projection/out') if os.path.isfile('./projection/imgs/project.png'): os.remove('./projection/imgs/project.png')