def main(): logger.debug('copying and importing client module') script_path = fs_tracker.get_artifact('clientscript') script_name = os.path.basename(script_path) new_script_path = os.path.join(os.getcwd(), '_clientscript.py') shutil.copy(script_path, new_script_path) script_path = new_script_path logger.debug("script path: " + script_path) mypath = os.path.dirname(script_path) sys.path.append(mypath) # os.path.splitext(os.path.basename(script_path))[0] module_name = '_clientscript' client_module = importlib.import_module(module_name) logger.debug('loading args') with open(fs_tracker.get_artifact('args')) as f: args = pickle.loads(f.read()) logger.debug('getting file mappings') artifacts = fs_tracker.get_artifacts() logger.debug('calling client funciton') retval = client_module.clientFunction(args, artifacts) logger.debug('saving the return value') with open(fs_tracker.get_artifact('retval'), 'w') as f: f.write(pickle.dumps(retval))
def pack_weights(self, persist_weights, model, metrics, verbose): """ Save weights if persist_weights flag is True """ if model is None: # This happens if a ResourceExhaustedError is caught metrics['weights_l2norm'] = None return if persist_weights: metrics['weights_l2norm'] = self.l2norm(model.get_weights()) weights_file = os.path.join(get_artifact('modeldir'), self.get_model_name()) if verbose: print("Saving weights file to {}".format(weights_file)) model.save(weights_file) if verbose: print("Saving complete")
def unpack_weights(self, verbose): """ Load weights if present """ try: weights_file = os.path.join(get_artifact('modeldir'), self.get_model_name()) if verbose: print("Loading weights from {}".format(weights_file)) weights = self.load_model_weights(weights_file) if verbose: print("Loaded successfully, L2 norm of weights = {}".format( self.l2norm(weights))) return weights except BaseException as exception: if verbose: print("Weight loading failed due to {}".format(exception)) print("unpack_weights returns None") return None
def clientFunction(args, files): print('client function call with args ' + str(args) + ' and files ' + str(files)) modelfile = 'model.dat' filename = files.get('model') or \ os.path.join(fs_tracker.get_artifact('modeldir'), modelfile) print("Trying to load file {}".format(filename)) if os.path.exists(filename): with open(filename, 'rb') as f: args = pickle.loads(f.read()) + 1 else: print("Trying to write file {}".format(filename)) with open(filename, 'wb') as f: f.write(pickle.dumps(args, protocol=2)) return args
def main(): argparser = argparse.ArgumentParser(description='Serve studio model') argparser.add_argument('--wrapper', '-w', help='python script with function create_model ' + 'that takes modeldir ' '(that is, directory where experiment saves ' + 'the checkpoints etc)' + 'and returns dict -> dict function (model).' + 'By default, studio-serve will try to determine ' + 'this function automatically.', default=None) argparser.add_argument('--port', help='port to run Flask server on', type=int, default=5000) argparser.add_argument('--host', help='host name.', default='0.0.0.0') argparser.add_argument( '--killafter', help='Shut down after this many seconds of inactivity', default=3600) options = argparser.parse_args(sys.argv[1:]) global model modeldir = fs_tracker.get_artifact('modeldata') if options.wrapper: module_name = re.sub('.py\Z', '', options.wrapper) wrapper_module = importlib.import_module(module_name) model = wrapper_module.create_model(modeldir) else: model = auto_generate_model(modeldir) restart_killtimer(int(options.killafter)) app.run(host=options.host, port=options.port)
def main(): logger.setLevel(logs.DEBUG) logger.debug('copying and importing client module') logger.debug('getting file mappings') artifacts = fs_tracker.get_artifacts() files = {} logger.debug("Artifacts = {}".format(artifacts)) for tag, path in six.iteritems(artifacts): if tag not in {'workspace', 'modeldir', 'tb', '_runner'}: if os.path.isfile(path): files[tag] = path elif os.path.isdir(path): dirlist = os.listdir(path) if any(dirlist): files[tag] = os.path.join(path, dirlist[0]) logger.debug("Files = {}".format(files)) script_path = files['clientscript'] # script_name = os.path.basename(script_path) new_script_path = os.path.join(os.getcwd(), '_clientscript.py') shutil.copy(script_path, new_script_path) script_path = new_script_path logger.debug("script path: " + script_path) mypath = os.path.dirname(script_path) sys.path.append(mypath) # os.path.splitext(os.path.basename(script_path))[0] module_name = '_clientscript' client_module = importlib.import_module(module_name) logger.debug('loading args') args_path = files['args'] with open(args_path, 'rb') as f: args = pickle.loads(f.read()) logger.debug('calling client funciton') retval = client_module.clientFunction(args, files) logger.debug('saving the return value') retval_path = fs_tracker.get_artifact('retval') if os.path.isdir(fs_tracker.get_artifact('clientscript')): # on go runner: logger.debug("Running in a go runner, creating {} for retval".format( retval_path)) try: os.mkdir(retval_path) except OSError: logger.debug('retval dir present') retval_path = os.path.join(retval_path, 'retval') logger.debug("New retval_path is {}".format(retval_path)) logger.debug('Saving retval') with open(retval_path, 'wb') as f: f.write(pickle.dumps(retval, protocol=2)) logger.debug('Done')
from studio import fs_tracker import numpy as np if fs_tracker.get_artifact('lr') is not None: lr = np.load(fs_tracker.get_artifact('lr')) else: lr = np.random.random(10) print "fitness: %s" % np.abs(np.sum(lr))
no_samples = 100 dim_samples = 5 learning_rate = 0.01 no_steps = 10 X = np.random.random((no_samples, dim_samples)) y = np.random.random((no_samples, )) w = np.random.random((dim_samples, )) for step in range(no_steps): yhat = X.dot(w) err = (yhat - y) dw = err.dot(X) w -= learning_rate * dw loss = 0.5 * err.dot(err) print("step = {}, loss = {}, L2 norm = {}".format(step, loss, w.dot(w))) # with open(os.path.expanduser('~/weights/lr_w_{}_{}.pck' # .format(step, loss)), 'w') as f: # f.write(pickle.dumps(w)) from studio import fs_tracker with open( os.path.join(fs_tracker.get_artifact('weights'), 'lr_w_{}_{}.pck'.format(step, loss)), 'w') as f: f.write(pickle.dumps(w))
from studio import fs_tracker import numpy as np try: lr = np.load(fs_tracker.get_artifact('lr')) except BaseException: lr = np.random.random(10) print("fitness: %s" % np.abs(np.sum(lr)))
#!/usr/bin/env python import os import matplotlib.image as mpimg import cv2 import numpy as np import torch import subprocess import tempfile try: from studio import fs_tracker DEFAULT_ZIP_PATH = fs_tracker.get_artifact('data') # IMG_DIR = fs_tracker.get_artifact('data') except ImportError: fs_tracker = None DEFAULT_ZIP_PATH = 'data/img_align_celeba_attr.zip' IMG_DIR = os.path.join(tempfile.gettempdir(), 'data') DEFAULT_ATTR_PATH = os.path.join(IMG_DIR, 'attributes.txt') IMG_ZIP_PATH = os.environ.get('IMG_ZIP_PATH', DEFAULT_ZIP_PATH) IMG_ATTR_PATH = os.environ.get('IMG_ATTR_PATH', DEFAULT_ATTR_PATH) IMG_SIZE = 128 IMG_PATH = os.path.join(IMG_DIR, 'images_%i_%i.pth' % (IMG_SIZE, IMG_SIZE)) IMG20K_PATH = os.path.join(IMG_DIR, 'images_%i_%i_20000.pth' % (IMG_SIZE, IMG_SIZE)) ATTR_PATH = os.path.join(IMG_DIR, 'attributes.pth')
import sys from studio import fs_tracker print(fs_tracker.get_artifact('f')) with open(fs_tracker.get_artifact('f'), 'r') as f: print(f.read()) if len(sys.argv) > 1: with open(fs_tracker.get_artifact('f'), 'w') as f: f.write(sys.argv[1]) sys.stdout.flush()
params = parser.parse_args() params.img_sz = preprocess.IMG_SIZE # check parameters check_attr(params) assert len(params.name.strip()) > 0 assert params.n_skip <= params.n_layers - 1 assert params.deconv_method in ['convtranspose', 'upsampling', 'pixelshuffle'] assert 0 <= params.smooth_label < 0.5 assert not params.ae_reload or os.path.isfile(params.ae_reload) assert not params.lat_dis_reload or os.path.isfile(params.lat_dis_reload) assert not params.ptc_dis_reload or os.path.isfile(params.ptc_dis_reload) assert not params.clf_dis_reload or os.path.isfile(params.clf_dis_reload) eval_clf_artifact = fs_tracker.get_artifact('eval_clf') if fs_tracker else '.' if params.eval_clf != '': eval_clf_path = os.path.join(eval_clf_artifact, params.eval_clf) else: eval_clf_path = eval_clf_artifact assert os.path.isfile(eval_clf_path) assert params.lambda_lat_dis == 0 or params.n_lat_dis > 0 assert params.lambda_ptc_dis == 0 or params.n_ptc_dis > 0 assert params.lambda_clf_dis == 0 or params.n_clf_dis > 0 # initialize experiment / load dataset logger = initialize_exp(params) data, attributes = load_images(params)
import glob import os from studio import fs_tracker import pickle weights_list = sorted( glob.glob(os.path.join(fs_tracker.get_artifact('w'), '*.pck'))) print('*****') print(weights_list[-1]) with open(weights_list[-1], 'r') as f: w = pickle.load(f) print w.dot(w) print('*****')
parser.add_argument( "--debug", type=bool_flag, default=True, help="Debug mode (only load a subset of the whole dataset)") params = parser.parse_args() # check parameters assert params.n_images >= 1 and params.n_interpolations >= 2 # create logger / load trained model logger = create_logger(None) if fs_tracker: model_path = os.path.join(fs_tracker.get_artifact('model'), params.model_path) else: model_path = params.model_path assert os.path.isfile(model_path), "model_path {} is not a file".format( model_path) ae = torch.load(model_path, map_location=lambda storage, loc: storage).eval() # restore main parameters params.debug = False params.batch_size = 32 params.v_flip = False params.h_flip = False params.img_sz = ae.img_sz params.attr = ae.attr