def __init__(self,kernels,basis,timeseries,stacks,seismo,profiles,gmtfiles,outdir, store_path='./',store=None,bounds=None,ref=[0,0]): self.kernels=flatten(kernels) self.basis=flatten(basis) self.timeseries=timeseries self.stacks=stacks self.seismo=seismo self.profiles = profiles self.store_path=store_path self.store=store self.gmtfiles=gmtfiles self.bnds=bounds self.ref=ref self.outdir=outdir self.Mker = len(self.kernels) self.Mbasis = len(self.basis) self.Nstacks=len(stacks) self.Nts=len(timeseries) self.Nwav=len(seismo) self.manifolds=flatten([stacks,timeseries,seismo]) self.Nmanif = len(self.manifolds) # load data and build model vector for each manifolds for i in xrange(self.Nmanif): self.manifolds[i].load(self) self.manifolds[i].info() self.manifolds[i].printbase() # Careful: after loading data self.N = sum(map((lambda x: getattr(x,'N')),self.manifolds)) self.Npoints = sum(map((lambda x: getattr(x,'Npoints')),self.manifolds)) # get all segments segments = [] for k in xrange(self.Mker): segments.append(map((lambda x: getattr(x,'segments')),self.kernels[k].structures)) self.segments = flatten(segments) self.Mseg = len(self.segments) # construct connecivities for k in xrange(self.Mseg): for kk in xrange(self.Mseg): # print self.segments[k].connectivity, self.segments[kk].name if self.segments[k].connectivity == self.segments[kk].name: self.segments[k].connectindex = kk self.segments[k].connect(self.segments[kk]) # sys.exit() # load local engine self.segments[k].loadEngine(self.store, self.store_path) # print self.segments[1].connectivity # print self.segments[0].name # sys.exit() # # number of parameters per patch: same for all self.Mpatch = self.segments[0].Mpatch
def get_genre(movie_id): movies = read_items('movies.dat') try: genre = movies[movie_id] return flatten(genre) except KeyError: return 0
def __init__(self, structures=[], name='', date=0., inversion_type='space', m=1., sigmam=0., prior_dist='Unif'): pattern.__init__(self, name, date, inversion_type, m, sigmam, prior_dist) self.t0 = time2dec(date)[0] self.seismo = True # segments associated to kernel self.structures = structures if len(self.structures) > 0: inversion_type = 'space' self.Mstr = len(self.structures) # each structures can have several segments self.Mseg = sum(map((lambda x: getattr(x, 'Mseg')), self.structures)) segments = [] segments.append( map((lambda x: getattr(x, 'segments')), self.structures)) self.segments = flatten(segments) # set time event for all patch map((lambda x: setattr(x, 'time', util.str_to_time(self.date))), self.segments)
def setweights(self): x = [] list = flatten(self.neurons) for neuron in list: #x.append(fn.normalize(neuron.weight)) x.append(neuron.weight) self.weights = scipy.array(x)
def get_umatrix(mymap, radius=1): umatrix = empty_list(mymap.size, 1) xmax = mymap.size[1] ymax = mymap.size[0] rad = range(-radius, radius + 1) # print rad for neuron in flatten(mymap.neurons): weight = neuron.weight position = neuron.position x = position[0] y = position[1] xrange = [] yrange = [] for i in rad: xrange.append(int((x + i) % xmax)) yrange.append(int((y + i) % ymax)) average_dist = 0 for x in xrange: for y in yrange: neighbour_weight = mymap.neurons[x][y].weight d = fn.distance(neighbour_weight, weight) average_dist += d umatrix[x][y] = average_dist return umatrix
def draw_neuron_activation(mymap, named=True, symbols=False): # iterates through EACH neuron and finds closest vector words = distances = empty_list(mymap.size, 1) if named: vectors = mymap.vectors keys = mymap.keys else: vectors = [] keys = [] idea_names = mymap.space.idea_names for item in mymap.space.table: keys.append(idea_names[item]) vectors.append(mymap.space.table[item]) if symbols: s = mymap.space.symbol_vectors keys = [] vectors = [] for item in s: keys.append(mymap.space.idea_names[item]) vectors.append(s[item]) for neuron in flatten(mymap.neurons): weight = neuron.weight match = fn.find_best_match(weight, vectors) distance = fn.distance(weight, vectors[match]) x = neuron.position x = fn.to_int(x) words[x[0]][x[1]] = keys[match] # distances[x[0]][x[1]] = distance word_plot(words) return words
def rated_genre(uid): # For this give the exact user id. This is used by the point class fname = 'ratings.dat' content = [] for line in open(fname): data = line.strip('\r\n').split('::') # Get the list of movies content.append(data) movie_info = [(x[1],x[2]) for x in content if x[0] == str(uid) ] genres = flatten([get_genre(m[0]) for m in movie_info]) genres = filter(lambda x:x!=0 ,genres) return list(set(genres))
def movies_for_user(uid,mid): movie_info = [(x[1],x[2]) for x in content if x[0] == str(uid[0]) ] avg_user_rating = sum([int(x[1]) for x in movie_info])/len(movie_info) #print movie_info # movie_info gets the list of movies rated by the user. # Get the genre of the recommended movie # Get the genres corresponding to the recommended movie and get the list of all movies in that genre recommended_genre = get_genre(str(int(mid))) #print recommended_genre #recommended_genre = gen movie_genre = flatten([genre_dict[x] for x in recommended_genre]) #print movie_genre #print movie_genre #Now get the ratings of all the movies rated by the user whose genre corresponds to the recommended movie reqd_movie = flatten([m[1] for m in movie_info if m[0] in movie_genre]) reqd_movie = map(lambda x:int(x),reqd_movie) try: print (sum(reqd_movie)/len(reqd_movie),avg_user_rating) except ZeroDivisionError: print 0,0
def plot_stations(self,nfigure): if self.Nwav>0: from mpl_toolkits.basemap import Basemap width = 22000000 lats, lons = [], [] events_lat, events_lon = [], [] names = [] for i in xrange(self.Nwav): manifold = self.seismo[i] lats.append(map((lambda x: getattr(x,'lat')),manifold.targets)) lons.append(map((lambda x: getattr(x,'lon')),manifold.targets)) events_lat.append(map((lambda x: getattr(x,'lat')),manifold.events)) events_lon.append(map((lambda x: getattr(x,'lon')),manifold.events)) names.append(manifold.names) events_lat,events_lon = np.array(flatten(events_lat)),np.array(flatten(events_lon)) names = np.array(flatten(names)) events_lat,events_lon = map(np.float,events_lat), map(np.float,events_lon) lats,lons = np.array(flatten(lats)),np.array(flatten(lons)) m = Basemap(width=width,height=width, projection='hammer', lat_0=events_lat[0],lon_0=events_lon[0]) fig = plt.figure(nfigure,figsize=(10,8)) stat_x, stat_y = m(lons,lats) event_x, event_y = m(events_lon,events_lat) m.drawmapboundary(fill_color='#99ffff') m.fillcontinents(color='lightgray',zorder=0) m.scatter(stat_x,stat_y,10,marker='^',color='k') m.scatter(event_x,event_y,30,marker='*',color='r') for label, x, y in zip(names,stat_x,stat_y): plt.text(x,y,label) plt.title('Stations (black) for Events (red)', fontsize=12) plt.savefig(self.outdir+'/wave/networkmap.eps',format = 'EPS')
def __init__(self, name, structures, date, datedec, inversion_type, T, m, sigmam, prior_dist): pattern.__init__(self, name, date, inversion_type, m, sigmam, prior_dist) self.t0 = datedec self.T = T Mstr = len(structures) # each structures can have several segments Mseg = sum(map((lambda x: getattr(x, 'Mseg')), structures)) segments = [] segments.append(map((lambda x: getattr(x, 'segments')), structures)) segments = flatten(segments) map((lambda x: setattr(x, 'time', util.str_to_time(self.date))), self.segments)
def get_distances_to_nearest(mymap): distances = empty_list(mymap.size, 1) vectors = mymap.vectors matches = [] for neuron in flatten(mymap.neurons): weight = neuron.weight match = fn.find_best_match(weight, vectors) matches.append(match) distance = fn.distance(weight, vectors[match]) x = neuron.position x = fn.to_int(x) distances[x[0]][x[1]] = distance c = Counter(matches) print c print "items mapped : " + str(len(sorted(c))) return distances
def draw_clusters_per_item(mymap, clusters): cluster_map = empty_list(mymap.size, 1) vectors = mymap.vectors keys = mymap.keys for neuron in flatten(mymap.neurons): weight = neuron.weight match = fn.find_best_match(weight, vectors) key = keys[match] cluster = clusters[key] x = neuron.position x = fn.to_int(x) cluster_map[x[0]][x[1]] = key # cluster_map[x[0]][x[1]] = cluster return cluster_map
def get_initial(self): initial = super(UserDetailView, self).get_initial() contact = self._get_bind_contact() if contact: # Guess some sort of usable username username = flatten(contact, ".") if len(username) < 3: username = getattr(contact, "email", "").split("@")[0] if len(username) < 3: username = "******" % random.randint(0, 99999999) initial.update( username=username, email=getattr(contact, "email", ""), first_name=getattr(contact, "first_name", ""), last_name=getattr(contact, "last_name", ""), ) return initial
def LeNet(x): # Arguments used for tf.truncated_normal, randomly defines variables for the weights and biases for each layer mu = 0 sigma = 0.1 # Layer 1: Convolutional. Input = 32x32x3. Output = 28x28x32. conv1_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 3, 32), mean = mu, stddev = sigma)) conv1_b = tf.Variable(tf.zeros(32)) conv1 = tf.nn.conv2d(x, conv1_W, strides=[1, 1, 1, 1], padding='VALID') + conv1_b # Activation. conv1 = tf.nn.relu(conv1) # Pooling. Input = 28x28x32. Output = 14x14x32. conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID') # Layer 2: Convolutional. Output = 12x12x64. conv2_W = tf.Variable(tf.truncated_normal(shape=(3, 3, 32, 64), mean = mu, stddev = sigma)) conv2_b = tf.Variable(tf.zeros(64)) conv2 = tf.nn.conv2d(conv1, conv2_W, strides=[1, 1, 1, 1], padding='VALID') + conv2_b # Activation. conv2 = tf.nn.relu(conv2) # Layer 3: Convolutional. Output = 10x10x128. conv3_W = tf.Variable(tf.truncated_normal(shape=(3, 3, 64, 128), mean = mu, stddev = sigma)) conv3_b = tf.Variable(tf.zeros(128)) conv3 = tf.nn.conv2d(conv2, conv3_W, strides=[1, 1, 1, 1], padding='VALID') + conv3_b # Activation. #conv2 = tf.nn.relu(conv2) conv3 = tf.nn.relu(conv3) # Pooling Input = 10x10x128. Output = 5x5x128. #conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID') conv3 = tf.nn.max_pool(conv3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID') # Flatten. Input = 5x5x128. Output = 3200. #fc0 = flatten(conv2) fc0 = flatten(conv3) # Layer 3: Fully Connected. Input = 3200. Output = 2400. fc1_W = tf.Variable(tf.truncated_normal(shape=(3200, 2400), mean = mu, stddev = sigma)) fc1_b = tf.Variable(tf.zeros(2400)) fc1 = tf.matmul(fc0, fc1_W) + fc1_b # Activation. fc1 = tf.nn.relu(fc1) # Layer 4: Fully Connected. Input = 2400. Output = 1600. fc2_W = tf.Variable(tf.truncated_normal(shape=(2400, 1600), mean = mu, stddev = sigma)) fc2_b = tf.Variable(tf.zeros(1600)) fc2 = tf.matmul(fc1, fc2_W) + fc2_b # Activation. fc2 = tf.nn.relu(fc2) # Layer 5: Fully Connected. Input = 1600. Output = 43. fc3_W = tf.Variable(tf.truncated_normal(shape=(1600, 43), mean = mu, stddev = sigma)) fc3_b = tf.Variable(tf.zeros(43)) logits = tf.matmul(fc2, fc3_W) + fc3_b return logits, conv1, conv2
import flatten import tensorflow as tf '''import time start=time.time() flatten() end=time.time() def dot_product_flatten(mat_list1, mat_list2): (list1, dims) = flatten(mat_list1) (list2, _) = flatten(mat_list2) list3 = [] for n,m in zip(list1, list2): list3.append(n*m) return unflatten((list3, dims)) def dot_product_no_flatten(mat_list1, mat_list2): for i in range(num_elems(mat_list1)): ''' # returns the number of elements in a variable's n-dim matrix given its shape def tuple_product(t): product = 1 for i in t: product *= i return product # number of individual trainable numbers def num_elems(mat_list):
def build_prior(self): # vector m = [ [m_ref, [m_basis for each point and each dim] for each manifolds], # m_faults for each segments ] self.priors = [] self.sampled = [] self.fixed = [] self.minit = [] self.name = [] self.mmin, self.mmax = [], [] for i in xrange(self.Nstacks): manifold = self.stacks[i] # Baseline for ii in xrange(manifold.Mbase): name = '{} Baseline {} {}'.format(manifold.reduction,ii,manifold.reduction) self.minit.append(manifold.base[ii]) self.name.append(name) if manifold.sig_base[ii] > 0.: m, sig = manifold.base[ii], manifold.sig_base[ii] self.mmin.append(manifold.base[ii]-manifold.sig_base[ii]) self.mmax.append(manifold.base[ii]+manifold.sig_base[ii]) if manifold.dist == 'Normal': p = pymc.Normal(name, mu=m, sd=sig) elif manifold.dist == 'Unif': p = pymc.Uniform(name, lower=m-sig, upper=m+sig, value=m) else: print('Problem with prior distribution difinition of parameter {}'.format(name)) sys.exit(1) self.sampled.append(name) self.priors.append(p) elif manifold.sig_base[ii] == 0: self.fixed.append(name) else: print('Problem with prior difinition of parameter {}'.format(name)) sys.exit(1) self.Mstacks = len(np.array(flatten(self.minit))) # print self.Mstacks # print self.Nstacks for i in xrange(self.Nts): manifold = self.timeseries[i] # Baseline for ii in xrange(manifold.Mbase): name = '{} Baseline {}'.format(manifold.reduction,ii) self.minit.append(manifold.base[ii]) self.name.append(name) if manifold.sig_base[ii] > 0.: m, sig = manifold.base[ii], manifold.sig_base[ii] self.mmin.append(manifold.base[ii]-manifold.sig_base[ii]) self.mmax.append(manifold.base[ii]+manifold.sig_base[ii]) if self.dist == 'Normal': p = pymc.Normal(name, mu=m, sd=sig) elif self.dist == 'Unif': p = pymc.Uniform(name, lower=m-sig, upper=m+sig, value=m) else: print('Problem with prior distribution difinition of parameter {}'.format(name)) sys.exit(1) self.sampled.append(name) self.priors.append(p) elif manifold.sig_base[ii] == 0: self.fixed.append(name) else: print('Problem with prior difinition of parameter {}'.format(name)) sys.exit(1) # Basis functions parameter for each dim of each manifolds for k in xrange(self.Mbasis): # name = 'basis:{}'.format(self.basis[k].name) # if self.basis[k].sigmam > 0.: # p = pymc.Uniform(name, lower=m-sig, upper=m+sig, shape=manifold.Npoints*manifold.dim) # self.sampled.append(name) # self.priors.append(p) # elif self.basis[k].sigmam == 0: # self.fixed.append(name) for j in xrange(manifold.Npoints): point = manifold.points[j] for ii in xrange(manifold.dim): name = 'Point:{}{}, dim:{}, basis:{}'.format(point.name,j,ii,self.basis[k].name) self.minit.append(self.basis[k].m) self.name.append(name) # print self.basis[k].sigmam if self.basis[k].sigmam > 0.: m, sig = self.basis[k].m, self.basis[k].sigmam self.mmin.append(self.basis[k].m-self.basis[k].sigmam) self.mmax.append(self.basis[k].m+self.basis[k].sigmam) # print name, m, sig if self.basis[k].dist == 'Normal': p = pymc.Normal(name, mu=m, sd=sig) elif self.basis[k].dist == 'Unif': p = pymc.Uniform(name, lower=m-sig, upper=m+sig, value=m) else: print('Problem with prior distribution difinition of parameter {}'.format(name)) sys.exit(1) self.sampled.append(name) self.priors.append(p) elif self.basis[k].sigmam == 0: self.fixed.append(name) else: print('Problem with prior difinition of parameter {}'.format(name)) sys.exit(1) # number of basis parameters self.Msurface = len(np.array(flatten(self.minit))) # print self.Msurface # sys.exit() # Faults parameters list_of_lists=map((lambda x: getattr(x,'fixed')),self.segments) flattened_list = [y for x in list_of_lists for y in x] for item in flattened_list: self.fixed.append(item) list_of_lists=map((lambda x: getattr(x,'priors')),self.segments) flattened_list = [y for x in list_of_lists for y in x] for item in flattened_list: self.priors.append(item) list_of_lists=map((lambda x: getattr(x,'mmin')),self.segments) flattened_list = [y for x in list_of_lists for y in x] for item in flattened_list: self.mmin.append(item) list_of_lists=map((lambda x: getattr(x,'mmax')),self.segments) flattened_list = [y for x in list_of_lists for y in x] for item in flattened_list: self.mmax.append(item) list_of_lists=map((lambda x: getattr(x,'sampled')),self.segments) flattened_list = [y for x in list_of_lists for y in x] for item in flattened_list: self.sampled.append(item) list_of_lists=map((lambda x: getattr(x,'m')),self.segments) flattened_list = [y for x in list_of_lists for y in x] for item in flattened_list: self.minit.append(item) list_of_lists=map((lambda x: getattr(x,'param')),self.segments) flattened_list = [y for x in list_of_lists for y in x] for item in flattened_list: self.name.append(item) self.faults = [] list_of_lists=map((lambda x: getattr(x,'sampled')),self.segments) flattened_list = [y for x in list_of_lists for y in x] for item in flattened_list: self.faults.append(item) # print # convert to array self.fixed = np.array(flatten(self.fixed)) # print 'fixed:', self.fixed self.priors = np.array(self.priors).flatten() # print 'priors:', self.priors self.sampled = np.array(flatten(self.sampled)) # print 'sampled:', self.sampled self.minit = np.array(flatten(self.minit)) # print 'minit:', self.minit self.name = np.array(flatten(self.name)) # print 'name:', self.name # print self.faults = np.array(flatten(self.faults)) # initialize m self.m = np.copy(self.minit) self.M = len(self.m) # sys.exit() return self.minit
def flush_map(self): list = flatten(self.neurons) for n, neuron in enumerate(list): neuron.weight = self.weights[n]
def setpositions(self): x = [] list = flatten(self.neurons) for neuron in list: x.append(fn.to_int(neuron.position)) self.positions = scipy.array(x)
def save(self, *args, **kwargs): if not self.identifier: raise ValueError(u"Attribute with null identifier not allowed") self.identifier = flatten(("%s" % self.identifier).lower()) return super(Attribute, self).save(*args, **kwargs)
print() print( "---------------------------------------------------------------------------" ) print(' PRIOR MODEL ') print( "---------------------------------------------------------------------------" ) print() logger.debug('Read Structures and load Fault model') # fault segment model fmodel = [] fmodel.append(map((lambda x: getattr(x, 'segments')), inv.structures)) inv.fmodel = flatten(fmodel) # get size of the green function # number of structures inv.Mstruc = len(inv.structures) # number of fault segments inv.Mseg = sum(map((lambda x: getattr(x, 'Mseg')), inv.structures)) # profile parameters logger.debug('Read profile parameters') try: profile = inv.profile except: # old version profile = inv.profiles # profile.xp0 = (profile.x-inv.fmodel[0].x)*inv.s[0]+(profile.y-inv.fmodel[0].y)*inv.s[1]
def load(self, inv): # load the data as a pyrocko pile and reform them into an array of traces data = pile.make_pile([self.wdir + self.reduction]) self.traces = data.all() # load station file fname = self.wdir + self.network stations_list = model.load_stations(fname) for s in stations_list: s.set_channels_by_name(*self.component.split()) self.targets = [] self.tmin, self.tmax = [], [] self.arrivals = [] self.names = [] for station, tr in zip(stations_list, self.traces): # iterate over all stations # print station.lat, station.lon target = Target( lat=np.float(station.lat), # station lat. lon=np.float(station.lon), # station lon. store_id=inv.store, # The gf-store to be used for this target, # we can also employ different gf-stores for different targets. interpolation='multilinear', # interp. method between gf cells quantity='displacement', # wanted retrieved quantity codes=station.nsl() + ('BH' + self.component, )) # Station and network code # Next we extract the expected arrival time for this station from the the store, # so we can use this later to define a cut-out window for the optimization: self.targets.append(target) self.names.append(station.nsl()[1]) # print len(self.traces), len(self.targets) for station, tr, target in zip(stations_list, self.traces, self.targets): engine = LocalEngine(store_superdirs=inv.store_path) store = engine.get_store(inv.store) # trace.snuffle(tr, events=self.events) arrival = store.t(self.phase, self.base_source, target) # expected P-wave arrival # print arrival tmin = self.base_source.time + arrival - 15 # start 15s before theor. arrival tmax = self.base_source.time + arrival + 15 # end 15s after theor. arrival # # print self.tmin,self.tmax tr.chop(tmin=tmin, tmax=tmax) self.tmin.append(tmin) self.tmax.append(tmax) self.arrivals.append(self.base_source.time + arrival) self.Npoints = len(self.targets) # data vector self.d = [] self.d.append(map((lambda x: getattr(x, 'ydata')), self.traces)) self.d = flatten(self.d) # time vector t = [] for i in xrange(self.Npoints): t.append(self.traces[i].get_xdata()) # self.t.append(map((lambda x: getattr(x,'get_xdata()')),self.traces)) # convert time self.t = time2dec(map(util.time_to_str, flatten(t))) # print self.t self.N = len(self.d)
import flatten import tensorflow as tf '''import time start=time.time() flatten() end=time.time() def dot_product_flatten(mat_list1, mat_list2): (list1, dims) = flatten(mat_list1) (list2, _) = flatten(mat_list2) list3 = [] for n,m in zip(list1, list2): list3.append(n*m) return unflatten((list3, dims)) def dot_product_no_flatten(mat_list1, mat_list2): for i in range(num_elems(mat_list1)): ''' # returns the number of elements in a variable's n-dim matrix given its shape def tuple_product(t): product = 1 for i in t: product *= i return product # number of individual trainable numbers
def test_flatten(): text = "Whät is Löve? Bäby Don't Hurt Me" assert flatten(text) == "what-is-love?-baby-don't-hurt-me" assert flatten(text, "/") == "what/is/love?/baby/don't/hurt/me"