def build(mode=None, optimize=None): """ Build SORD code. """ cf = util.namespace(configure.configure()) if not optimize: optimize = cf.optimize if not mode: mode = cf.mode if not mode: mode = 'sm' base = ( 'globals.f90', 'diffcn.f90', 'diffnc.f90', 'hourglass.f90', 'bc.f90', 'surfnormals.f90', 'util.f90', 'frio.f90', ) common = ( 'arrays.f90', 'fieldio.f90', 'stats.f90', 'parameters.f90', 'setup.f90', 'gridgen.f90', 'attenuation.f90', 'material.f90', 'source.f90', 'inivolstress.f90', 'rupture.f90', 'resample.f90', 'checkpoint.f90', 'timestep.f90', 'stress.f90', 'acceleration.f90', 'sord.f90', ) cwd = os.getcwd() path = os.path.realpath(os.path.dirname(__file__)) f = os.path.join(path, 'bin') if not os.path.isdir(f): os.mkdir(f) new = False os.chdir(os.path.join(path, 'src')) if 's' in mode: source = base + ('serial.f90', ) + common for opt in optimize: object_ = os.path.join('..', 'bin', 'sord-s' + opt) compiler = cf.fortran_serial + cf.fortran_flags[opt] new |= util.make(compiler, object_, source) if 'm' in mode and cf.fortran_mpi: source = base + ('mpi.f90', ) + common for opt in optimize: object_ = os.path.join('..', 'bin', 'sord-m' + opt) compiler = cf.fortran_mpi + cf.fortran_flags[opt] new |= util.make(compiler, object_, source) os.chdir(path) if new: try: import bzrlib except ImportError: print( 'Warning: bzr not installed. Install bzr if you want to save a\ copy of the source code for posterity with each run.') else: os.system('bzr export sord.tgz') os.chdir(cwd) return
def generate_task(infile, experiment, run_test_set, save_weights, weightfile, tmp_dir): """ Model definition: tower network on basis letters to get autoencoding, feed through shared dense layers to retrieve all 26 letters (caps) """ # Input X = keras.layers.Input(shape=(4, 64, 64), name='input') # Lambda layers pull out the 4 basis characters x1 = keras.layers.Lambda(lambda x: x[:, 0, :, :], output_shape=(64, 64), name='x1')(X) x2 = keras.layers.Lambda(lambda x: x[:, 1, :, :], output_shape=(64, 64), name='x2')(X) x3 = keras.layers.Lambda(lambda x: x[:, 2, :, :], output_shape=(64, 64), name='x3')(X) x4 = keras.layers.Lambda(lambda x: x[:, 3, :, :], output_shape=(64, 64), name='x4')(X) # Flatten the images into 64 * 64 dimensional vectors x1 = keras.layers.Flatten()(x1) x2 = keras.layers.Flatten()(x2) x3 = keras.layers.Flatten()(x3) x4 = keras.layers.Flatten()(x4) # The towers consist of a fully connected layer neurons = 32 if experiment == 2 or experiment == 4 or experiment == 5 else 16 x1 = keras.layers.Dense(neurons, activation='relu')(x1) x2 = keras.layers.Dense(neurons, activation='relu')(x2) x3 = keras.layers.Dense(neurons, activation='relu')(x3) x4 = keras.layers.Dense(neurons, activation='relu')(x4) if experiment == 3 or experiment == 4 or experiment == 5: x1 = keras.layers.Dense(neurons, activation='relu')(x1) x2 = keras.layers.Dense(neurons, activation='relu')(x2) x3 = keras.layers.Dense(neurons, activation='relu')(x3) x4 = keras.layers.Dense(neurons, activation='relu')(x4) # Concatenates the towers together and feed through fully connected layers added = keras.layers.Concatenate()([x1, x2, x3, x4]) neurons = 400 if experiment == 2 or experiment == 3 or experiment == 5 else 200 fc = keras.layers.Dense(neurons, activation='relu')(added) fc = keras.layers.Dense(neurons, activation='relu')(fc) if experiment == 3 or experiment == 4 or experiment == 5: fc = keras.layers.Dense(neurons, activation='relu')(fc) fc = keras.layers.Dense(26 * 64 * 64, activation='relu')(fc) # Reshape for 2D convolution and upsample fc = keras.layers.Reshape((26, 64, 64))(fc) if experiment == 0: fc = keras.layers.Conv2DTranspose(26, data_format='channels_first', kernel_size=(4, 4), padding='same', activation='relu')(fc) fc = keras.layers.Conv2DTranspose(26, data_format='channels_first', kernel_size=(4, 4), padding='same', activation='relu')(fc) elif experiment == 2 or experiment == 3 or experiment == 4 or experiment == 5: fc = keras.layers.Conv2DTranspose(26, data_format='channels_first', kernel_size=(4, 4), padding='same', activation='relu')(fc) else: pass # No convolutional layers out = fc model = keras.models.Model(inputs=X, outputs=out) model.compile(optimizer='adam', loss='mean_squared_error', metrics=[]) model.summary() # Plotting # keras.utils.plot_model(model, to_file='model.png') # exit() if weightfile is not None: model.load_weights('./gen-task/{}.hdf5'.format(weightfile)) namespace = util.namespace(infile, experiment) def dump_history(history): with open('{}/{}history.pickle'.format(WORK_PATH, namespace), 'wb') as f: pickle.dump(history, f) print('Dumped history.') # Open and prepare training set train = h5py.File('./gen-task-dsets/gen-task-{}-train.hdf5'.format(infile), 'r') outputs = train['outputs'][:] basis = train['basis'][:] print('Training model on {} fonts...'.format(train['basis'].shape[0])) def display_picture(arr, name): img = PIL.Image.fromarray(np.hstack([arr[idx] for idx in range(26)])) if img.mode != 'L': img = img.convert('L') # img.show() # Debug (disable when running) img.save('./{}/{}{}.png'.format(tmp_dir, namespace, name)) class ImageHistory(keras.callbacks.Callback): """ Runs predict on the model and test set to visualize how the NN is learning. In a Keras callback, we have access to model and params as class properties. """ def __init__(self): super() # Parent class constructor self.image_idx = 0 def on_train_begin(self, logs={}): predictions = self.model.predict(basis[:1]) display_picture(predictions[0], 'train-viz-{}'.format(self.image_idx)) self.image_idx += 1 def on_batch_end(self, batch, logs={}): predictions = self.model.predict(basis[:1]) display_picture(predictions[0], 'train-viz-{}'.format(self.image_idx)) self.image_idx += 1 history = model.fit(x=basis, y=outputs, epochs=EPOCHS, batch_size=512, callbacks=[ImageHistory()]) # See Keras docs for the history object dump_history(history.history) if save_weights: model.save_weights('./gen-task/{}weights.hdf5'.format(namespace)) # Open and prepare val set test = h5py.File('./gen-task-dsets/gen-task-{}-val.hdf5'.format(infile), 'r') outputs = test['outputs'][:] basis = test['basis'][:] print('Validating model on {} fonts...'.format(test['basis'].shape[0])) loss = model.evaluate(x=basis, y=outputs) # View classified examples from the validation set predictions = model.predict(basis) display_picture(predictions[0], 'val') if run_test_set: # Open and prepare test set test = h5py.File('./gen-task-dsets/gen-task-{}-test.hdf5'.format(infile), 'r') outputs = test['outputs'][:] basis = test['basis'][:] print('Testing model on {} fonts...'.format(test['basis'].shape[0])) loss = model.evaluate(x=basis, y=outputs) # View classified examples from the validation set predictions = model.predict(basis) display_picture(predictions[0], 'test')
def find_namespace(self, name): assert isinstance(name, Symbol) if UT.namespace(name) is not nil: raise ValueError("Namespace names cannot have namespaces") return self._namespaces.get(self._str_for_name(name), None)
def stage(inputs): """ Setup, and optionally launch, a SORD job. """ import glob, time, getopt, shutil import setup # Save start time starttime = time.asctime() print('SORD setup') # Read defaults pm = {} f = os.path.join(os.path.dirname(__file__), 'parameters.py') exec open(f) in pm if 'machine' in inputs: cf = configure.configure(machine=inputs['machine']) else: cf = configure.configure() # Merge inputs inputs = inputs.copy() util.prune(inputs) util.prune(pm) util.prune(cf, pattern='(^_)|(^.$)') for k, v in inputs.iteritems(): if k in cf: cf[k] = v elif k in pm: pm[k] = v else: sys.exit('Unknown parameter: %s = %r' % (k, v)) cf = util.namespace(cf) cf.rundir = os.path.expanduser(cf.rundir) pm = prepare_param(util.namespace(pm), cf.itbuff) # Command line options opts = [ 'n', 'dryrun', 's', 'serial', 'm', 'mpi', 'i', 'interactive', 'q', 'queue', 'd', 'debug', 'g', 'debugging', 't', 'testing', 'p', 'profiling', 'O', 'optimized', 'f', 'force', ] options = ''.join(opts[::2]) long_options = opts[1::2] opts = getopt.getopt(sys.argv[1:], options, long_options)[0] for o, v in opts: if o in ('-n', '--dry-run'): cf.prepare = False elif o in ('-s', '--serial'): cf.mode = 's' elif o in ('-m', '--mpi'): cf.mode = 'm' elif o in ('-i', '--interactive'): cf.run = 'i' elif o in ('-q', '--queue'): cf.run = 'q' elif o in ('-d', '--debug'): cf.optimize = 'g' cf.run = 'g' elif o in ('-g', '--debugging'): cf.optimize = 'g' elif o in ('-t', '--testing'): cf.optimize = 't' elif o in ('-p', '--profiling'): cf.optimize = 'p' elif o in ('-O', '--optimized'): cf.optimize = 'O' elif o in ('-f', '--force'): if os.path.isdir(cf.rundir): shutil.rmtree(cf.rundir) else: sys.exit('Error: unknown option: ' + o) if not cf.prepare: cf.run = False # Partition for parallelization pm.nn = tuple(int(i) for i in pm.nn) maxtotalcores = cf.maxnodes * cf.maxcores if not cf.mode and maxtotalcores == 1: cf.mode = 's' np3 = pm.np3[:] if cf.mode == 's': np3 = [1, 1, 1] nl = [(pm.nn[i] - 1) / np3[i] + 1 for i in range(3)] i = abs(pm.faultnormal) - 1 if i >= 0: nl[i] = max(nl[i], 2) pm.np3 = tuple((pm.nn[i] - 1) / nl[i] + 1 for i in range(3)) cf.np = pm.np3[0] * pm.np3[1] * pm.np3[2] if not cf.mode: cf.mode = 's' if cf.np > 1: cf.mode = 'm' # Resources if cf.maxcores: cf.nodes = min(cf.maxnodes, (cf.np - 1) / cf.maxcores + 1) cf.ppn = (cf.np - 1) / cf.nodes + 1 cf.cores = min(cf.maxcores, cf.ppn) cf.totalcores = cf.nodes * cf.maxcores else: cf.nodes = 1 cf.ppn = cf.np cf.cores = cf.np cf.totalcores = cf.np # RAM and Wall time usage if pm.oplevel in (1, 2): nvars = 20 elif pm.oplevel in (3, 4, 5): nvars = 23 else: nvars = 44 nm = (nl[0] + 2) * (nl[1] + 2) * (nl[2] + 2) cf.pmem = 32 + int(1.2 * nm * nvars * int(cf.dtype[-1]) / 1024 / 1024) cf.ram = cf.pmem * cf.ppn ss = (pm.nt + 10) * cf.ppn * nm / cf.cores / cf.rate sus = int(ss / 3600 * cf.totalcores + 1) mm = ss / 60 * 3.0 + 10 if cf.maxtime: mm = min(mm, 60 * cf.maxtime[0] + cf.maxtime[1]) mm = mm * 3 hh = mm / 60 mm = mm % 60 cf.walltime = '%d:%02d:00' % (hh, mm) #v1.1.01 2 times of orignal hours cf.walltime = '1:00:00' # used in v1.1 print('Machine: ' + cf.machine) print('Cores: %s of %s' % (cf.np, maxtotalcores)) print('Nodes: %s of %s' % (cf.nodes, cf.maxnodes)) print('RAM: %sMb of %sMb per node' % (cf.ram, cf.maxram)) print('Time limit: ' + cf.walltime) print('SUs: %s' % sus) if cf.maxcores and cf.ppn > cf.maxcores: print('Warning: exceding available cores per node (%s)' % cf.maxcores) if cf.ram and cf.ram > cf.maxram: print('Warning: exceding available RAM per node (%sMb)' % cf.maxram) # Compile code if not cf.prepare: return cf setup.build(cf.mode, cf.optimize) # Create run directory print('Run directory: ' + cf.rundir) try: os.makedirs(cf.rundir) except (OSError): sys.exit('%r exists or cannot be created. Use --force to overwrite.' % cf.rundir) for f in 'in', 'out', 'prof', 'stats', 'debug', 'checkpoint': os.mkdir(os.path.join(cf.rundir, f)) # Copy files to run directory cwd = os.path.realpath(os.getcwd()) cf.rundate = time.asctime() cf.name = os.path.basename(cf.rundir) cf.rundir = os.path.realpath(cf.rundir) os.chdir(os.path.realpath(os.path.dirname(__file__))) cf.bin = os.path.join('.', 'sord-' + cf.mode + cf.optimize) path = os.path.join('bin', 'sord-' + cf.mode + cf.optimize) shutil.copy(path, cf.rundir) if os.path.isfile('sord.tgz'): shutil.copy('sord.tgz', cf.rundir) if cf.optimize == 'g': for f in glob.glob(os.path.join('src', '*.f90')): shutil.copy(f, cf.rundir) f = os.path.join('conf', cf.machine, 'templates') if not os.path.isdir(f): f = os.path.join('conf', 'default', 'templates') for d in os.path.join('conf', 'common', 'templates'), f: for f in glob.glob(os.path.join(d, '*')): ff = os.path.join(cf.rundir, os.path.basename(f)) out = open(f).read() % cf.__dict__ open(ff, 'w').write(out) shutil.copymode(f, ff) # Combine metadata meta = util.namespace(pm.__dict__) for k in 'name', 'rundate', 'rundir', 'user', 'os_', 'dtype': setattr(meta, k, getattr(cf, k)) meta.indices = {} meta.xi = {} for f in meta.fieldio: op, filename = f[0], f[8] if filename != '-': meta.indices[filename] = f[7] if 'wi' in op: meta.xi[filename] = f[4] meta.shape = {} for k in meta.indices: nn = [(i[1] - i[0]) / i[2] + 1 for i in meta.indices[k]] nn = [n for n in nn if n > 1] if nn == []: nn = [1] meta.shape[k] = nn # Write files os.chdir(cf.rundir) log = open('log', 'w') log.write(starttime + ': setup started\n') util.save('conf.py', cf, prune_pattern='(^_)|(^.$)') util.save('parameters.py', pm, expand=['fieldio']) util.save('meta.py', meta, expand=['shape', 'xi', 'indices', 'fieldio']) # Return to initial directory os.chdir(cwd) return cf
def find_var(self, name): assert isinstance(name, Symbol) if UT.namespace(name) is not nil: raise ValueError("Var names cannot have namespaces") return self._vars.get(name, self._str_for_name(name))