def unrar(filename, tmpdir): "Распаковывает архив с помощью unrar." olddir = os.path.abspath(os.path.curdir) if not os.path.isabs(filename): filename = os.path.abspath(filename) try: os.chdir(tmpdir) cmd = util.pipe([['unrar', 'e', '-p-', '-o+', '-inul', '-y', filename]]) return [os.path.join(tmpdir, f) for f in os.listdir(tmpdir)] finally: os.chdir(olddir)
def __init__(self, trial, x, y, num_steps, data_path, log_path, learning_rate=0.5, norm=False, radius=None, tensorboard=False, gpu=None): ''' trial: string, name of the trial data: tensorflow dataset iterator x, y: int, dimension of the map feat_nr: int, number of inpt_features num_steps: int, number of training steps norm: bool, if True then data will be normalized radius: int, if None use default radius max(x,y)/2 tensorboard: bool, if True, logs the graph and runs a performance test gpu: int, picks gpu from cluster by number ''' self.trial = trial self.data_path = data_path self.log_path = log_path self.x = x self.y = y self.num_steps = num_steps self.learning_rate = learning_rate self.radius = radius self.norm = norm self.tensorboard = tensorboard self.gpu = gpu if gpu != None: ########## ### PICK GPU ################### os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu) with tf.variable_scope("input"): with tf.variable_scope("data"): if norm is True: self.data = normalize( np.load(data_path).astype(np.float32)) else: self.data = np.load(data_path).astype(np.float32) self.feat_nr = len(self.data[0]) self.nr_inst = len(self.data) with tf.variable_scope("pipeline"): inpt = pipe(self.data, prefetch=20) # step count self.step = tf.placeholder(tf.float32) # create weight matrix self.weights = tf.get_variable( "weights", shape=[self.x * self.y, self.feat_nr], initializer=tf.random_uniform_initializer) #initializer=tf.random_normal_initializer) with tf.variable_scope("locations"): self.locations = tf.constant( np.array( list([ np.array([i, j]) for i in range(self.x) for j in range(self.y) ]))) if radius == None: self.radius = tf.constant(max(self.x, self.y) / 2, dtype=tf.float32) #Calculate current learning rate and radius # todo: different decay options with tf.variable_scope("decay"): decay = tf.cast(1 - (self.step / self.num_steps), dtype=tf.float32) with tf.variable_scope("current_lr"): current_lr = self.learning_rate * decay with tf.variable_scope("current_radius"): current_radius = self.radius * decay with tf.variable_scope("bmu"): # calculate Best Matching Unit (euclidean distances) distances = tf.sqrt(tf.reduce_sum((self.weights - inpt)**2, 1)) bmu = tf.reduce_min(distances) #bmu = tf.argmin(distances, 0) with tf.variable_scope("bmu_loc"): # get the location of the bmu #mask = tf.pad(tf.reshape(bmu, [1]), tf.constant(np.array([[0, 1]]))) #size = tf.cast(tf.constant(np.array([1, 2])), dtype=tf.int64) #bmu_loc = tf.reshape(tf.slice(self.locations, mask, size), [2]) mask = tf.equal(distances, bmu) bmu_locs = tf.boolean_mask(self.locations, mask) bmu_loc = tf.slice(bmu_locs, [0, 0], [1, 2])[0] # make sure its only one location with tf.variable_scope("neighborhood"): # calculate the influence on the neighborhood of the bmu bmu_dist = tf.sqrt( tf.cast(tf.reduce_sum((self.locations - bmu_loc)**2, 1), dtype=tf.float32)) nbr_func = tf.exp(-bmu_dist / (2 * (current_radius**2))) with tf.variable_scope("delta"): # get the delta of the weights delta = tf.expand_dims(nbr_func, -1) * current_lr * (inpt - self.weights) with tf.variable_scope("new_weights"): #update the new weights new_weights = self.weights + delta self.train_step = tf.assign(self.weights, new_weights) # initialize session self.sess = tf.InteractiveSession() self.sess.run(tf.global_variables_initializer()) if self.tensorboard is True: wrtr = tf.summary.FileWriter( os.path.expanduser(self.log_path + self.trial)) wrtr.add_graph(self.sess.graph) profile(self.sess, wrtr, new_weights, feed_dict=None, prerun=3, tag='flow') wrtr.flush()
def get(space, pos): return pipe(space)(*[ array_get(p, [] if (i < len(pos) - 1) else ".") for i, p in enumerate(reversed(pos)) ])
def get_next_priority_expr(expr): return pipe(expr, get_priority_scope, get_priority_simple_expr)