def unrar(filename, tmpdir): "Распаковывает архив с помощью unrar." olddir = os.path.abspath(os.path.curdir) if not os.path.isabs(filename): filename = os.path.abspath(filename) try: os.chdir(tmpdir) cmd = util.pipe([['unrar', 'e', '-p-', '-o+', '-inul', '-y', filename]]) return [os.path.join(tmpdir, f) for f in os.listdir(tmpdir)] finally: os.chdir(olddir)
def __init__(self,
trial,
x,
y,
num_steps,
data_path,
log_path,
learning_rate=0.5,
norm=False,
radius=None,
tensorboard=False,
gpu=None):
'''
trial: string, name of the trial
data: tensorflow dataset iterator
x, y: int, dimension of the map
feat_nr: int, number of inpt_features
num_steps: int, number of training steps
norm: bool, if True then data will be normalized
radius: int, if None use default radius max(x,y)/2
tensorboard: bool, if True, logs the graph and runs a performance test
gpu: int, picks gpu from cluster by number
'''
self.trial = trial
self.data_path = data_path
self.log_path = log_path
self.x = x
self.y = y
self.num_steps = num_steps
self.learning_rate = learning_rate
self.radius = radius
self.norm = norm
self.tensorboard = tensorboard
self.gpu = gpu
if gpu != None:
##########
### PICK GPU ###################
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu)
with tf.variable_scope("input"):
with tf.variable_scope("data"):
if norm is True:
self.data = normalize(
np.load(data_path).astype(np.float32))
else:
self.data = np.load(data_path).astype(np.float32)
self.feat_nr = len(self.data[0])
self.nr_inst = len(self.data)
with tf.variable_scope("pipeline"):
inpt = pipe(self.data, prefetch=20)
# step count
self.step = tf.placeholder(tf.float32)
# create weight matrix
self.weights = tf.get_variable(
"weights",
shape=[self.x * self.y, self.feat_nr],
initializer=tf.random_uniform_initializer)
#initializer=tf.random_normal_initializer)
with tf.variable_scope("locations"):
self.locations = tf.constant(
np.array(
list([
np.array([i, j]) for i in range(self.x)
for j in range(self.y)
])))
if radius == None:
self.radius = tf.constant(max(self.x, self.y) / 2,
dtype=tf.float32)
#Calculate current learning rate and radius
# todo: different decay options
with tf.variable_scope("decay"):
decay = tf.cast(1 - (self.step / self.num_steps), dtype=tf.float32)
with tf.variable_scope("current_lr"):
current_lr = self.learning_rate * decay
with tf.variable_scope("current_radius"):
current_radius = self.radius * decay
with tf.variable_scope("bmu"):
# calculate Best Matching Unit (euclidean distances)
distances = tf.sqrt(tf.reduce_sum((self.weights - inpt)**2, 1))
bmu = tf.reduce_min(distances)
#bmu = tf.argmin(distances, 0)
with tf.variable_scope("bmu_loc"):
# get the location of the bmu
#mask = tf.pad(tf.reshape(bmu, [1]), tf.constant(np.array([[0, 1]])))
#size = tf.cast(tf.constant(np.array([1, 2])), dtype=tf.int64)
#bmu_loc = tf.reshape(tf.slice(self.locations, mask, size), [2])
mask = tf.equal(distances, bmu)
bmu_locs = tf.boolean_mask(self.locations, mask)
bmu_loc = tf.slice(bmu_locs, [0, 0],
[1, 2])[0] # make sure its only one location
with tf.variable_scope("neighborhood"):
# calculate the influence on the neighborhood of the bmu
bmu_dist = tf.sqrt(
tf.cast(tf.reduce_sum((self.locations - bmu_loc)**2, 1),
dtype=tf.float32))
nbr_func = tf.exp(-bmu_dist / (2 * (current_radius**2)))
with tf.variable_scope("delta"):
# get the delta of the weights
delta = tf.expand_dims(nbr_func,
-1) * current_lr * (inpt - self.weights)
with tf.variable_scope("new_weights"):
#update the new weights
new_weights = self.weights + delta
self.train_step = tf.assign(self.weights, new_weights)
# initialize session
self.sess = tf.InteractiveSession()
self.sess.run(tf.global_variables_initializer())
if self.tensorboard is True:
wrtr = tf.summary.FileWriter(
os.path.expanduser(self.log_path + self.trial))
wrtr.add_graph(self.sess.graph)
profile(self.sess,
wrtr,
new_weights,
feed_dict=None,
prerun=3,
tag='flow')
wrtr.flush()
def get(space, pos):
return pipe(space)(*[
array_get(p, [] if (i < len(pos) - 1) else ".")
for i, p in enumerate(reversed(pos))
])
def get_next_priority_expr(expr):
return pipe(expr,
get_priority_scope,
get_priority_simple_expr)
