def _distance_to_W(self, ids=None): allneighbors = {} weights = {} if ids: ids = np.array(ids) else: ids = np.arange(len(self._nmat)) if self.binary: for i, neighbors in enumerate(self._nmat): ns = [ni for ni in neighbors if ni != i] neigh = list(ids[ns]) if len(neigh) == 0: allneighbors[ids[i]] = [] weights[ids[i]] = [] else: allneighbors[ids[i]] = neigh weights[ids[i]] = [1] * len(ns) else: self.dmat = self.kd.sparse_distance_matrix( self.kd, max_distance=self.threshold) for i, neighbors in enumerate(self._nmat): ns = [ni for ni in neighbors if ni != i] neigh = list(ids[ns]) if len(neigh) == 0: allneighbors[ids[i]] = [] weights[ids[i]] = [] else: try: allneighbors[ids[i]] = neigh weights[ids[i]] = [self.dmat[( i, j)] ** self.alpha for j in ns] except ZeroDivisionError: raise Exception, "Cannot compute inverse distance for elements at same location (distance=0)." return allneighbors, weights
def generate_pdb(self,pdb_name,shift=0,traduction={"1":"bead","2":"telo","3":"ribo","4":"cent","5":"spbb","6":"rcut","7":"scut"}): """ Generate a pdb file according to the molecules given to the LSimu object traduction must contain as key the type of bead and as item the pdb name (should be 4 letters name) (Very ugly code) """ #Initial stuff for pdb: towrite = [] atomid = 0 for n,molecule in enumerate(self.molecules): for rn,((x,y,z),type_bead) in enumerate(zip(molecule.coords,molecule.types_beads)): atomid+= 1 name="bead" residue_name="bea" chain_id=string.ascii_letters[(n + shift) % len(string.ascii_letters)] residue_number=rn name = traduction["%i" % type_bead] towrite.append(self.one_pdb_atom(x,y,z,atomid,name,residue_name,residue_number,chain_id)) with open(pdb_name,"w") as f: f.write("".join(towrite))
def initializeGUI(self): self.d = {"Switches": {}, "Triggers": {}} # set layout layout = QtGui.QGridLayout() self.setLayout(layout) # get switch names and add them to the layout, and connect their function layout.addWidget(QtGui.QLabel("Switches"), 0, 0) switchNames = yield self.server.get_switching_channels() for order, name in enumerate(switchNames): button = QtGui.QPushButton(name) self.d["Switches"][name] = button button.setCheckable(True) initstate = yield self.server.get_state(name) button.setChecked(initstate) self.setButtonText(button, name) button.clicked.connect(self.buttonConnection(name, button)) layout.addWidget(button, 0, 1 + order) # do same for trigger channels layout.addWidget(QtGui.QLabel("Triggers"), 1, 0) triggerNames = yield self.server.get_trigger_channels() for order, name in enumerate(triggerNames): button = QtGui.QPushButton(name) button.clicked.connect(self.triggerConnection(name)) self.d["Triggers"][name] = button layout.addWidget(button, 1, 1 + order)
def restore_sources(self, pubkey, tx): """ Restore the sources of a cancelled tx :param sakia.entities.Transaction tx: """ txdoc = TransactionDoc.from_signed_raw(tx.raw) for offset, output in enumerate(txdoc.outputs): if output.conditions.left.pubkey == pubkey: source = Source(currency=self.currency, pubkey=pubkey, identifier=txdoc.sha_hash, type='T', noffset=offset, amount=output.amount, base=output.base) self._sources_processor.drop(source) for index, input in enumerate(txdoc.inputs): source = Source(currency=self.currency, pubkey=txdoc.issuers[0], identifier=input.origin_id, type=input.source, noffset=input.index, amount=input.amount, base=input.base) if source.pubkey == pubkey: self._sources_processor.insert(source)
def calculate_zernikes(self, workspace): zernike_indexes = cpmz.get_zernike_indexes(self.zernike_degree.value + 1) meas = workspace.measurements for o in self.objects: object_name = o.object_name.value objects = workspace.object_set.get_objects(object_name) # # First, get a table of centers and radii of minimum enclosing # circles per object # ij = np.zeros((objects.count + 1, 2)) r = np.zeros(objects.count + 1) for labels, indexes in objects.get_labels(): ij_, r_ = minimum_enclosing_circle(labels, indexes) ij[indexes] = ij_ r[indexes] = r_ # # Then compute x and y, the position of each labeled pixel # within a unit circle around the object # ijv = objects.ijv l = ijv[:, 2] yx = (ijv[:, :2] - ij[l, :]) / r[l, np.newaxis] z = cpmz.construct_zernike_polynomials( yx[:, 1], yx[:, 0], zernike_indexes) for image_group in self.images: image_name = image_group.image_name.value image = workspace.image_set.get_image( image_name, must_be_grayscale=True) pixels = image.pixel_data mask = (ijv[:, 0] < pixels.shape[0]) & \ (ijv[:, 1] < pixels.shape[1]) mask[mask] = image.mask[ijv[mask, 0], ijv[mask, 1]] yx_ = yx[mask, :] l_ = l[mask] z_ = z[mask, :] if len(l_) == 0: for i, (n, m) in enumerate(zernike_indexes): ftr = self.get_zernike_magnitude_name(image_name, n, m) meas[object_name, ftr] = np.zeros(0) if self.wants_zernikes == Z_MAGNITUDES_AND_PHASE: ftr = self.get_zernike_phase_name(image_name, n, m) meas[object_name, ftr] = np.zeros(0) continue areas = scind.sum( np.ones(l_.shape, int), labels=l_, index=objects.indices) for i, (n, m) in enumerate(zernike_indexes): vr = scind.sum( pixels[ijv[mask, 0], ijv[mask, 1]] * z_[:, i].real, labels=l_, index=objects.indices) vi = scind.sum( pixels[ijv[mask, 0], ijv[mask, 1]] * z_[:, i].imag, labels=l_, index=objects.indices) magnitude = np.sqrt(vr * vr + vi * vi) / areas ftr = self.get_zernike_magnitude_name(image_name, n, m) meas[object_name, ftr] = magnitude if self.wants_zernikes == Z_MAGNITUDES_AND_PHASE: phase = np.arctan2(vr, vi) ftr = self.get_zernike_phase_name(image_name, n, m) meas[object_name, ftr] = phase
def load_text(self): ''' The text of instances are not stored in the prediction result file, so you need to call this function to load texts from testing data. >>> from libshorttext.analyzer import * >>> insts = InstanceSet('prediction_result_path') >>> insts.load_text() This method also load the extra svm features if extra svm files are used when training. ''' EMPTY_MESSAGE = '**None**' sorted_insts = sorted(self.insts, key = lambda inst: inst.idx) i = 0 for idx, lines in enumerate(izip(*([open(self.filepath, 'r')] + [open(f, 'r') for f in self.extra_svm_files]))): line = lines[0] extra_svm_feats = lines[1:] nr_extra_svm_feats = len(extra_svm_feats) if idx > sorted_insts[-1].idx: break if idx == sorted_insts[i].idx: try: sorted_insts[i].text = line.split('\t',1)[1].strip() except: sorted_insts[i].text = EMPTY_MESSAGE sorted_insts[i].extra_svm_feats = [None] * nr_extra_svm_feats for j, extra_svm_feat in enumerate(extra_svm_feats): try: sorted_insts[i].extra_svm_feats[j] = dict(map(lambda t: (int(t[0]), float(t[1])), [feat.split(':') for feat in extra_svm_feat.split(None, 1)[1].split()])) except: sorted_insts[i].extra_svm_feats[j] = EMPTY_MESSAGE i += 1
def _eval_kernel(self): # get points within bandwidth distance of each point if not hasattr(self, 'neigh'): kdtq = self.kdt.query_ball_point neighbors = [kdtq(self.data[i], r=bwi[0]) for i, bwi in enumerate(self.bandwidth)] self.neigh = neighbors # get distances for neighbors data = np.array(self.data) bw = self.bandwidth kdtq = self.kdt.query z = [] for i, nids in enumerate(self.neigh): di, ni = kdtq(self.data[i], k=len(nids)) zi = np.array([dict(zip(ni, di))[nid] for nid in nids]) / bw[i] z.append(zi) zs = z # functions follow Anselin and Rey (2010) table 5.4 if self.function == 'triangular': self.kernel = [1 - z for z in zs] elif self.function == 'uniform': self.kernel = [np.ones(z.shape) * 0.5 for z in zs] elif self.function == 'quadratic': self.kernel = [(3. / 4) * (1 - z ** 2) for z in zs] elif self.function == 'quartic': self.kernel = [(15. / 16) * (1 - z ** 2) ** 2 for z in zs] elif self.function == 'gaussian': c = np.pi * 2 c = c ** (-0.5) self.kernel = [c * np.exp(-(z ** 2) / 2.) for z in zs] else: print 'Unsupported kernel function', self.function
def load_records(self, records): """Load in a database and interpret it as a network First column must be unique keys which define the instance units. Each column is a pool (names, gangs, ages, etc). Every row is mutually excitory. """ self.units[:] = [] self.pools[:] = [] for counter,record in enumerate(records): relatedunits = ['%s:%s' % (k,v) for (k,v) in record.items()] relatedunits.insert(0,'_:%s'% counter) if not len(relatedunits): continue key = len(self.units) for poolnum, name in enumerate(relatedunits): if poolnum >= len(self.pools): self.pools.append(Pool(self.unitbyname)) pool = self.pools[poolnum] if name in self.unitbyname: unit = self.unitbyname[name] else: unit = Unit(name, pool, self.unitbyname) self.units.append(unit) pool.addmember(unit) if poolnum > 0: self.units[key].addexciter(unit) unit.addexciter(self.units[key])
def plot_overtime(data_file): data = performance.load_score_dict(data_file) avg_sim = [] std_sim = [] # Lets compute the average fraction of matching paths for each case for index, time_step in enumerate(data): if index == 0: continue prev_step = data[index - 1] sim_list = [] for pair_index, pair in enumerate(time_step): curr_chain = set([x[0] for x in pair]) print curr_chain prev_chain = set([x[0] for x in prev_step[pair_index]]) if len(curr_chain) == 0 or len(prev_chain) == 0: continue sim = float(len(curr_chain & prev_chain)) / len(curr_chain) sim_list.append(sim) avg_sim.append(np.mean(sim_list)) std_sim.append(np.std(sim_list)) print "Next Time Step!" plotting.overtime_plot(avg_sim, std_sim)
def recompute_unread(min_date = None): from r2.models import Inbox, Account, Comment, Message from r2.lib.db import queries def load_accounts(inbox_rel): accounts = set() q = inbox_rel._query(eager_load = False, data = False, sort = desc("_date")) if min_date: q._filter(inbox_rel.c._date > min_date) for i in fetch_things2(q): accounts.add(i._thing1_id) return accounts accounts_m = load_accounts(Inbox.rel(Account, Message)) for i, a in enumerate(accounts_m): a = Account._byID(a) print "%s / %s : %s" % (i, len(accounts_m), a) queries.get_unread_messages(a).update() queries.get_unread_comments(a).update() queries.get_unread_selfreply(a).update() accounts = load_accounts(Inbox.rel(Account, Comment)) - accounts_m for i, a in enumerate(accounts): a = Account._byID(a) print "%s / %s : %s" % (i, len(accounts), a) queries.get_unread_comments(a).update() queries.get_unread_selfreply(a).update()
def load_level(filename): """ Loads a level from a given text file. Args: filename: The name of the txt file containing the maze. Returns: The loaded level (dict) containing the locations of walls (set), the locations of spaces (dict), and a mapping of locations to waypoints (dict). """ walls = set() spaces = {} waypoints = {} with open(filename, "r") as f: for j, line in enumerate(f.readlines()): for i, char in enumerate(line): if char == '\n': continue elif char == WALL: walls.add((i, j)) elif char.isnumeric(): spaces[(i, j)] = float(char) elif char.islower(): spaces[(i, j)] = 1. waypoints[char] = (i, j) level = {'walls': walls, 'spaces': spaces, 'waypoints': waypoints} return level
def links(self, data_matrix): data_size = data_matrix.shape[0] kernel_matrix = pairwise_kernels(data_matrix, metric=self.metric, **self.kwds) # compute instance density as average pairwise similarity density = np.sum(kernel_matrix, 0) / data_size # compute list of nearest neighbors kernel_matrix_sorted = np.argsort(-kernel_matrix) # make matrix of densities ordered by nearest neighbor density_matrix = density[kernel_matrix_sorted] # if a denser neighbor cannot be found then assign link to the instance itself link_ids = list(range(density_matrix.shape[0])) # for all instances determine link link for i, row in enumerate(density_matrix): i_density = row[0] # for all neighbors from the closest to the furthest for jj, d in enumerate(row): # proceed until n_nearest_neighbors have been explored if self.n_nearest_neighbors is not None and jj > self.n_nearest_neighbors: break j = kernel_matrix_sorted[i, jj] if jj > 0: j_density = d # if the density of the neighbor is higher than the density of the instance assign link if j_density > i_density: link_ids[i] = j break return link_ids
def flux_matrix(self, geo): """Returns a sparse matrix which can be used to multiply a vector of connection table values for underground blocks, to give approximate average fluxes of those values at the block centres.""" natm = geo.num_atmosphere_blocks nele = geo.num_underground_blocks conindex = dict([((c.block[0].name, c.block[1].name), i) for i, c in enumerate(self.connectionlist)]) from scipy import sparse A = sparse.lil_matrix((3 * nele, self.num_connections)) if not self.block_centres_defined: self.calculate_block_centres(geo) for iblk, blk in enumerate(self.blocklist[natm:]): ncons = blk.num_connections for conname in blk.connection_name: otherindex, sgn = [(0, -1), (1, 1)][conname[0] == blk.name] blk2name = conname[otherindex] icon = conindex[conname] centre2 = self.block[blk2name].centre if centre2 <> None: n = centre2 - blk.centre n /= np.linalg.norm(n) else: n = np.array([0, 0, 1]) # assumed connection to atmosphere for i, ni in enumerate(n): A[3 * iblk + i, icon] = -sgn * ni / (ncons * self.connection[conname].area) return A
def rx_oversampled(frames, ref_frame, modulated_frame, x_preamble, data, rx_kernel, demapper, timeslots, fft_len, cp_len, cs_len): ref_frame_os = signal.resample(ref_frame, 2 * len(ref_frame)) x_preamble_os = signal.resample(x_preamble, 2 * len(x_preamble)) nyquist_frame_len = cp_len + 2 * fft_len + cs_len + cp_len + timeslots * fft_len + cs_len n_frames = np.shape(frames)[0] sync_frames = np.zeros((n_frames, nyquist_frame_len), dtype=np.complex) print('nyquist sampled frame len', nyquist_frame_len, 'with n_frames', n_frames) f_start = cp_len + 2 * fft_len + cs_len d_start = f_start + cp_len print('data start: ', d_start) for i, f in enumerate(frames[0:2]): tf = np.roll(f, 1) tf[0] = 0 ff = signal.resample(tf, len(f) // 2) sframe = synchronize_time(ff, ref_frame_os, x_preamble_os, 2 * fft_len, 2 * cp_len) sframe = signal.resample(sframe, len(sframe) // 2) sframe = synchronize_freq_offsets(sframe, modulated_frame, x_preamble, fft_len, cp_len, samp_rate=3.125e6) print(len(sframe), len(ref_frame)) rx_preamble = sframe[cp_len:cp_len + 2 * fft_len] avg_phase = calculate_avg_phase(rx_preamble, x_preamble) # m, c = calculate_avg_phase(rx_preamble, x_preamble) # avg_phase = calculate_avg_phase(sframe, ref_frame) # phase_eqs = m * np.arange(-cp_len, len(sframe) - cp_len) + c # sframe *= np.exp(-1j * phase_eqs) # sframe *= np.exp(-1j * avg_phase) sync_frames[i] = sframe rx_data_frame = sframe[d_start:d_start + fft_len * timeslots] # # rx_data_frame *= np.exp(-1j * avg_phase) # demodulate_frame(rx_data_frame, modulated_frame, rx_kernel, demapper, data, timeslots, fft_len) for i, f in enumerate(sync_frames[0:3]): rx_data_frame = f[d_start:d_start + fft_len * timeslots] demodulate_frame(rx_data_frame, modulated_frame, rx_kernel, demapper, data, timeslots, fft_len)
def model(self): model = { "hostname": "", "dns[0]": "", "dns[1]": "", "ntp[0]": "", "ntp[1]": "", "bond.name": "", "bond.slaves.selected": "", "bond.options": defaults.NicBonding.default_options } model["hostname"] = defaults.Hostname().retrieve()["hostname"] or \ network.hostname() # Pull name-/timeservers from config files (not defaults) nameservers = config.network.nameservers() if nameservers: for idx, nameserver in enumerate(nameservers): model["dns[%d]" % idx] = nameserver timeservers = config.network.timeservers() if timeservers: for idx, timeserver in enumerate(timeservers): model["ntp[%d]" % idx] = timeserver model.update(self._model_extra) return model
def _make_scalar_compound_controller(self, fcurves, keyframes, bez_chans, default_xform): ctrl = plCompoundController() subctrls = ("X", "Y", "Z") for i in subctrls: setattr(ctrl, i, plLeafController()) exported_frames = ([], [], []) ctrl_fcurves = { i.array_index: i for i in fcurves } for keyframe in keyframes: for i, subctrl in enumerate(subctrls): fval = keyframe.values.get(i, None) if fval is not None: keyframe_type = hsKeyFrame.kBezScalarKeyFrame if i in bez_chans else hsKeyFrame.kScalarKeyFrame exported = hsScalarKey() exported.frame = keyframe.frame_num exported.frameTime = keyframe.frame_time exported.inTan = keyframe.in_tans[i] exported.outTan = keyframe.out_tans[i] exported.type = keyframe_type exported.value = fval exported_frames[i].append(exported) for i, subctrl in enumerate(subctrls): my_keyframes = exported_frames[i] # ensure this controller has at least ONE keyframe if not my_keyframes: hack_frame = hsScalarKey() hack_frame.frame = 0 hack_frame.frameTime = 0.0 hack_frame.type = hsKeyFrame.kScalarKeyFrame hack_frame.value = default_xform[i] my_keyframes.append(hack_frame) getattr(ctrl, subctrl).keys = (my_keyframes, my_keyframes[0].type) return ctrl
def extractNames(li): finList = [] ## Loop through the list that has the HTML page content for a in li: ## Tokenize the HTML text into smaller blocks of text for send in nltk.sent_tokenize(str(a)): smLi = [] ## Tokenize the smaller blocks of text in individual words and then add a Part-of-Speech(POS) tag for index, chunk in enumerate(nltk.pos_tag(nltk.word_tokenize(send))): ## If the POS tag is NNP (noun) if 'NNP' in chunk[1]: ## If the each character in the word is an alphanumeric character and there are more than 2 characters in the word if(len(' '.join(e for e in chunk[0] if e.isalnum())) > 2): ## Append the list with the index of the word, chunk that has the POS tag and the link smLi.append([index, chunk, a[1]]) finList.append(smLi) nameLi = [] for f in finList: if len(f) > 0: strName = '' for index, i in enumerate(f): ## If strName is blank, declare it with the current word in the list if strName == '': strName = i[1][0] ## If index+1 is not at the end of the list, continue if (index + 1) < len(f): ## If the index is a consecutive index, add to the strName if i[0] + 1 == f[index + 1][0]: strName = strName + ' ' + f[index + 1][1][0] ## If the index is not a consecutive index, append strName to the nameLi list with the article link and make the strName blank else: if ' ' in strName: nameLi.append([strName, i[2]]) strName = '' return nameLi
def loadTrajectoryData(inFile = UJILocDataFile): with open(UJILocDataFile, 'r') as dataFile: data = dataFile.read() # 9-axis IMU data # trajectory: dictionary with three elements # N is number of samples in the trajectory (data taken at 10Hz) # mag: Nx3 numpy array where each line has XYZ mag data # gyro: Nx3 numpy array where each line has XYZ gyro vel data # accel: Nx3 numpy array where each line has XYZ lin accelerometer data segments = data.split("<", 2) IMUDataStr = segments[0].split('\n')[:-1] magArr = [] oriArr = [] accelArr = [] for i, lineStr in enumerate(IMUDataStr): lineStr = lineStr.split(' ', 10)[:-1] lineStr = [float(x) for x in lineStr] magArr.append(lineStr[1:4]) # xyz mag data for sample accelArr.append(lineStr[4:7]) # xyz accelerometer data for single samp oriArr.append(lineStr[7:10]) # xyz gyro data for sample # values initially are given as euler angles which are not good for imu-type calculations. # so we fix em! gyroArr = rawSensorStateProc.orientationToGyro(oriArr) initOrientationMatrix = rawSensorStateProc.calcInitialOrientation(oriArr[0]) # IMUData = [{'mag': magArr, 'gyro': gyroArr, 'accel': accelArr}] # process waypoint data # each waypoint consists of a latitude coordinate, longitude coordinate, # and index (what IMU dataopoint it represents) waypoints = [] waypointStr = segments[1].split(">", 2) numWaypoints = int(waypointStr[0]) waypointLns = waypointStr[1].lstrip().split('\n') for i, lineStr in enumerate(waypointLns): line = lineStr.split(' ', WAYPOINTS_ELEMS_PER_LINE) line = [float(x) for x in line] if i == 0: waypoints.append({'lat': line[0], 'long': line[1], 'index': line[4]}) waypoints.append({'lat': line[2], 'long': line[3], 'index': line[5]}) seqLen = line[5] traj = ({'waypoints': np.array(waypoints), 'mag': np.array(magArr), 'gyro': np.array(gyroArr), 'accel': np.array(accelArr), 'orientSensed': np.array(oriArr), 'initOrient': initOrientationMatrix, 'seqLen': seqLen}) return traj # loadTrajectoryData()
def run_image_find(self): if self.ss_labels is None or self.s_labels is None: return featured_labels = self.tree.get_featured( VARIABLE['edge_mode'] ) # print( featured_labels ) object_color = { } color = 1 label_img = numpy.zeros( (len( self.ss_labels), len( self.s_labels[0])) ) for object_name, object_featured in featured_labels.items(): object_color[ object_name ] = color for i, ss_label in enumerate( self.ss_labels ): print(i, ss_label, object_featured.keys()) if ss_label in object_featured.keys(): for j, label in enumerate( self.s_labels[i] ): if label in object_featured[ ss_label ]: label_img[i,j] = color color += 1 for row in label_img: print(row) print( self.ss_labels ) print( object_color ) self.piximage.color_objects(label_img)
def get_unigram_probs(vocab, counts, smooth_constant): special_symbol_ids = [vocab[x] for x in SPECIAL_SYMBOLS] vocab_size = len(vocab) - len(SPECIAL_SYMBOLS) num_words_with_non_zero_counts = 0 for word_id, count in enumerate(counts): if word_id in special_symbol_ids: continue if counts[word_id] > 0: num_words_with_non_zero_counts += 1 if num_words_with_non_zero_counts < vocab_size and smooth_constant == 0.0: sys.exit(sys.argv[0] + ": --smooth-unigram-counts should not be zero, " "since there are words with zero-counts") smooth_count = smooth_constant * num_words_with_non_zero_counts / vocab_size total_counts = 0.0 for word_id, count in enumerate(counts): if word_id in special_symbol_ids: continue counts[word_id] += smooth_count total_counts += counts[word_id] probs = [] for count in counts: probs.append(count / total_counts) return probs
def rectangle_plot(self, xparams, yparams, yroots, filled=True, ymarkers=None, xmarkers=None, **kwargs): self.make_figure(nx=len(xparams), ny=len(yparams)) # f, plots = subplots(len(yparams), len(xparams), sharex='col', sharey='row') sharey = None yshares = [] for x, xparam in enumerate(xparams): sharex = None for y, (yparam, roots) in enumerate(zip(yparams, yroots)): # f.sca(plots[y, x]) if x > 0: sharey = yshares[y] ax = self.subplot(x + 1, y + 1, sharex=sharex, sharey=sharey) if y == 0: sharex = ax self.plot_2d(roots, param_pair=[xparam, yparam], filled=filled, do_xlabel=y == len(yparams) - 1, do_ylabel=x == 0, add_legend_proxy=x == 0 and y == 0) if ymarkers is not None and ymarkers[y] is not None: self.add_y_marker(ymarkers[y], **kwargs) if xmarkers is not None and xmarkers[x] is not None: self.add_x_marker(xmarkers[x], **kwargs) if y == 0: lims = xlim() else: lims = (min(xlim()[0], lims[0]), max(xlim()[1], lims[1])) if y != len(yparams) - 1: setp(ax.get_xticklabels(), visible=False) if x != 0: setp(ax.get_yticklabels(), visible=False) if x == 0: yshares.append(ax) sharex.set_xlim(lims) self.spaceTicks(sharex.xaxis) sharex.set_xlim(sharex.xaxis.get_view_interval()) for ax in yshares: self.spaceTicks(ax.yaxis) ax.set_ylim(ax.yaxis.get_view_interval()) subplots_adjust(wspace=0, hspace=0) self.finish_plot(no_gap=True)
def classListBox2(self, id): pageNumber = self.noteBook.GetCurrent() cname = self.classBox[pageNumber].GetEntry(id).GetText().Data() self.methodBox[pageNumber].RemoveAll() self.labelBox[pageNumber].RemoveAll() methods = self.cmap[cname]['methods'] labels= self.cmap[cname]['labels'] self.cmap[cname]['selected'] = True # List methods names = self.cmap[cname]['sortedmethods'] for index, name in enumerate(names): if not methods[name]: continue self.methodBox[pageNumber].AddEntry(name, index) self.methodBox[pageNumber].Layout() # List getByLabels names = labels.keys() names.sort() for index, name in enumerate(names): if not labels[name]: continue self.labelBox[pageNumber].AddEntry(name, index) self.labelBox[pageNumber].Layout()
def WorkBook_writeSheet(self, filename): columns = ['Date', 'Month', 'ID #', 'Contact ID #', \ 'Talked to Person X?', 'Closeness/Trust with X', \ 'Connecting ID', 'Connector ID'] # Writes to csv file with open(filename, 'w') as f: writer = csv.writer(f) writer.writerow(columns) for row in self.sheet: date = row[DATE_COLUMN] month = row[MONTH_COLUMN] connecting = row[CONNECTING_COLUMN] connector = row[CONNECTOR_COLUMN] talkVal = row[TALKED_WEIGHT] closeVal = row[CLOSENESS_WEIGHT] connectingID = row[CONNECTING_ID_COLUMN] connectorID = row[CONNECTOR_ID_COLUMN] row = [date, month, connecting, connector, talkVal, \ closeVal, connectingID, connectorID] writer.writerow(row) # Converts from the written csv file to xlsx for csvfile in glob.glob(os.path.join('.', '*.csv')): workbook = Workbook(csvfile[0:-4] + '.xlsx') worksheet = workbook.add_worksheet() with open(csvfile, 'rb') as f: reader = csv.reader(f) for r, row in enumerate(reader): for c, col in enumerate(row): worksheet.write(r, c, col) workbook.close() sys.exit()
def validate_label_generation(): mals1_df = pd.read_csv('data/sorted-train-labels-vs251-252.csv') mals2_df = pd.read_csv('data/sorted-train-labels-vs263-264-apt.csv') counter = 0 m1_x = np.array(mals1_df['malware_type_x']) m1_f = np.array(mals1_df['family_name']) m1_sl = np.array(mals1_df['sample_label']) m1_fl = np.array(mals1_df['family_label']) m2_x = np.array(mals2_df['malware_type_x']) m21_f = np.array(mals2_df['family_name']) m2_sl = np.array(mals2_df['sample_label']) m2_fl = np.array(mals2_df['family_label']) for idx1, mname1 in enumerate(m1_x): for idx2, mname2 in enumerate(m2_x): if mname1 == mname2: if m1_sl[idx1] != m2_sl[idx2]: print("Sample label incongruence: {:d} {:d}".format(m1_sl[idx1], m2_sl[idx2])) counter += 1 if (m1_fl[idx1] != m2_fl[idx2]): print("Family label incongruence: {:d} {:d}".format(m1_fl[idx1], m2_fl[idx2])) counter += 1 if (idx1 % 1000) == 0: print("Processed {:d} malware names.".format(idx1)) print("Total Incongruence Errors: {:d}".format(counter)) return
def knapsack_unbounded_dp(items, C): # order by max value per item size items = sorted(items, key=lambda item: item[VALUE]/float(item[SIZE]), reverse=True) # Sack keeps track of max value so far as well as the count of each item in the sack print('!') sack = [(0, [0 for i in items]) for i in range(0, C+1)] # value, [item counts] print('!') for i,item in enumerate(items): name, size, value = item for c in range(size, C+1): print(sack) sackwithout = sack[c-size] # previous max sack to try adding this item to trial = sackwithout[0] + value used = sackwithout[1][i] if sack[c][0] < trial: # old max sack with this added item is better sack[c] = (trial, sackwithout[1][:]) sack[c][1][i] +=1 # use one more value, bagged = sack[C] numbagged = sum(bagged) size = sum(items[i][1]*n for i,n in enumerate(bagged)) # convert to (iten, count) pairs) in name order bagged = sorted((items[i][NAME], n) for i,n in enumerate(bagged) if n) return value, size, numbagged, bagged
def testPeriodsMonths(self): """ Test iteration over periods (months) """ dt = datetime.datetime ef = S3TimePlotEventFrame(dt(2011, 1, 5), dt(2011, 4, 28), slots="months") expected = [(dt(2011, 1, 5), dt(2011, 2, 5)), (dt(2011, 2, 5), dt(2011, 3, 5)), (dt(2011, 3, 5), dt(2011, 4, 5)), (dt(2011, 4, 5), dt(2011, 4, 28))] for i, period in enumerate(ef): self.assertEqual(period.start, expected[i][0]) self.assertEqual(period.end, expected[i][1]) ef = S3TimePlotEventFrame(dt(2011, 1, 5), dt(2011, 8, 16), slots="3 months") expected = [(dt(2011, 1, 5), dt(2011, 4, 5)), (dt(2011, 4, 5), dt(2011, 7, 5)), (dt(2011, 7, 5), dt(2011, 8, 16))] for i, period in enumerate(ef): self.assertEqual(period.start, expected[i][0]) self.assertEqual(period.end, expected[i][1])
def categorize(data, colnum, missingvals, ranges=[]): categories = set() for row in data: if row[colnum] not in missingvals: categories.add(row[colnum]) catlist = list(categories) catlist.sort() # print(', '.join(['%i: %s' % (n, catlist[n]) for n in xrange(len(catlist))]), "(with missing vals:", missingvals, ")") missing_indices = [] for index, row in enumerate(data): if row[colnum] in missingvals: # missing data row[colnum] = 0 missing_indices.append(index) else: # this row doesn't have missing data. if len(ranges) > 0: # find val in ranges and use that index. found = False for i, r in enumerate(ranges): if isinstance(r, basestring): # compare strings. if r in row[colnum]: row[colnum] = i found = True break elif isinstance(r, ( int, long )) and not re.search('[a-zA-Z]', row[colnum]): # ref : http://stackoverflow.com/questions/3501382/checking-whether-a-variable-is-an-integer-or-not if float(row[colnum]) >= r and len(ranges) > i+1 and isinstance(ranges[i+1], ( int, long )) and float(row[colnum]) < ranges[i+1]: row[colnum] = i found = True break if not found: print(row[colnum]) # error here else: # no ranges given, so just set category of appearance. row[colnum] = catlist.index(row[colnum])+1 return missing_indices
def lcs_dy_prog(s1, s2): table = np.zeros((len(s1), len(s2)), dtype=np.int) def lookup(i, j): if i < 0 or j < 0: return 0 else: return table[i, j] # find length of the lcs for i, c1 in enumerate(s1): for j, c2 in enumerate(s2): if c1 == c2: table[i, j] = lookup(i - 1, j - 1) + 1 else: table[i, j] = max(lookup(i - 1, j), lookup(i, j - 1)) # backtrac to find lcs (not unique) i = len(s1) - 1 j = len(s2) - 1 res = "" while i >= 0 and j >= 0: if s1[i] == s2[j]: res += s1[i] i -= 1 j -= 1 else: if lookup(i - 1,j) > lookup(i, j - 1): i -= 1 else: j -= 1 res = res[::-1] return res
def testPeriodsWeeks(self): """ Test iteration over periods (weeks) """ dt = datetime.datetime ef = S3TimePlotEventFrame(dt(2011, 1, 5), dt(2011, 1, 28), slots="weeks") expected = [(dt(2011, 1, 5), dt(2011, 1, 12)), (dt(2011, 1, 12), dt(2011, 1, 19)), (dt(2011, 1, 19), dt(2011, 1, 26)), (dt(2011, 1, 26), dt(2011, 1, 28))] for i, period in enumerate(ef): self.assertEqual(period.start, expected[i][0]) self.assertEqual(period.end, expected[i][1]) ef = S3TimePlotEventFrame(dt(2011, 1, 5), dt(2011, 2, 16), slots="2 weeks") expected = [(dt(2011, 1, 5), dt(2011, 1, 19)), (dt(2011, 1, 19), dt(2011, 2, 2)), (dt(2011, 2, 2), dt(2011, 2, 16))] for i, period in enumerate(ef): self.assertEqual(period.start, expected[i][0]) self.assertEqual(period.end, expected[i][1])
def plots_1d(self, roots, params=None, legend_labels=None, legend_ncol=None, nx=None, paramList=None, roots_per_param=False, share_y=None, markers=None, xlims=None): if roots_per_param: params = [self.check_param(roots[i][0], param) for i, param in enumerate(params)] else: params = self.get_param_array(roots[0], params) if paramList is not None: wantedParams = self.paramNameListFromFile(paramList) params = [param for param in params if param.name in wantedParams] nparam = len(params) if share_y is None: share_y = self.settings.prob_label is not None and nparam > 1 plot_col, plot_row = self.make_figure(nparam, nx=nx) plot_roots = roots for i, param in enumerate(params): subplot(plot_row, plot_col, i + 1) if roots_per_param: plot_roots = roots[i] if markers is not None and i < len(markers): marker = markers[i] else: marker = None # self.plot_1d(plot_roots, param, no_ylabel=share_y and i % self.plot_col > 0, marker=marker, prune=(None, 'both')[share_y]) self.plot_1d(plot_roots, param, no_ylabel=share_y and i % self.plot_col > 0, marker=marker) if xlims is not None: xlim(xlims[i][0], xlims[i][1]) if share_y: self.spaceTicks(gca().xaxis, expand=True) self.finish_plot([legend_labels, roots][legend_labels is None], legend_ncol=legend_ncol) if share_y: subplots_adjust(wspace=0) return plot_col, plot_row
def dt_freq(dt, ignore_M=False): ''' Detect the (highest) frequency in the passed date time index, and return it in a format the pandas can interpret. Also tries to detect whether the found freq is monthly or higher (slow) Parameters: ------- dt : pd.DatetimeIndex The index that is explored Returns: -------- unit : str Unit string of the used resolution ignore_M : bool Do not attempt to find out if the daily freq is monthly or lower (slow) ''' if dt.freq is not None: # if we have the info, it's trivial return dt.freq.n, dt.freq.name months, years = dt.month.values, dt.year.values first_month, first_year = months[0], years[0] last_month, last_year = months[-1], years[-1] dm = days_in_month(months, years, astype=float) # days in month sm = np.array([int(timedelta(days=d).total_seconds()) for d in dm]) # seconds in month seconds = [] for ns, sec_in_m in zip(np.diff(dt.values), sm): # month, day, hour, minute i = ns.astype('timedelta64[s]').astype(int) seconds.append(i) sampled = np.array(seconds) # now check if we can resample to months: resample = [('m', 60.), ('H', 60.), ('D', 24.)] highest_freq = 's' for freq, m in resample: if all(sampled % m == 0.): # can be resampled, up to daily from seconds sampled = sampled / m highest_freq = freq if highest_freq == 'D' and not any(sampled < 28) and not ignore_M: # it could be monthly, check df_month = pd.DataFrame(index=pd.period_range('{}-{}'.format(first_year, first_month), '{}-{}'.format(last_year, last_month), freq='M')) df_month['passed'] = np.nan df_month['days'] = np.nan for i, (y, m) in enumerate(zip(years, months)): df_month.loc['{}-{:02d}'.format(y,m),'passed'] = i df_month['passed'] = df_month['passed'].bfill() for y, m in zip(df_month.index.year, df_month.index.month): df_month.loc['{}-{:02d}'.format(y,m),'days'] = days_in_month(m, y, float) df_month = df_month[1:] m_should_days = df_month.groupby(['passed']).sum()['days'].values if len(m_should_days) == len(sampled) and all(m_should_days == sampled): # it is monthly or lower m_freq = min(df_month.groupby(['passed']).count()['days'].values) return m_freq, 'M' else: return min(sampled), highest_freq else: return min(sampled), highest_freq
albu.HorizontalFlip(p=.5), # Rotate(5, p=.5) ]) # image_augmenter = albu.Compose([albu.GaussNoise(p=.5), # albu.RandomBrightnessContrast(p=.5)]) image_augmenter = None dataset = CityscapesDataset(split='train', net_type='deeplab', ignore_index=19, debug=True, affine_augmenter=affine_augmenter, image_augmenter=image_augmenter) dataloader = DataLoader(dataset, batch_size=8, shuffle=True) print(len(dataset)) for i, batched in enumerate(dataloader): images, labels = batched if i == 0: fig, axes = plt.subplots(8, 2, figsize=(20, 48)) plt.tight_layout() for j in range(8): axes[j][0].imshow( minmax_normalize(images[j], norm_range=(0, 1), orig_range=(-1, 1))) axes[j][1].imshow(labels[j]) axes[j][0].set_xticks([]) axes[j][0].set_yticks([]) axes[j][1].set_xticks([]) axes[j][1].set_yticks([]) plt.savefig('dataset/cityscapes.png')
################ LASSO ################ # Parameter grid to go through. param_grid_lasso = {'alpha': [0.001, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.2, 0.3, 0.4, 0.5, 1]} # Construct (random) combinations of the parameters (in this case just all of them) N_ITER = len(param_grid_lasso['alpha']) param_sampler = ParameterSampler(param_grid_lasso, n_iter=N_ITER) # Preallocate space for the results lasso_validation_results = pd.DataFrame(np.zeros((N_ITER, len(param_grid_lasso)+2)), columns=['alpha', 'coeff', 'score']) # Loop over the different parameters. for idx, param in enumerate(param_sampler): lasso_obj = LassoEstimator(**param) lasso_obj.fit(features_train, target_train) predict = lasso_obj.predict(features_validation) score = lasso_obj.score(target_validation, predict) lasso_validation_results.iloc[idx, :] = [param['alpha'], lasso_obj.coef_.astype('object'), score] print(lasso_validation_results) # Get parameters of the best performing model idx_lasso_model = np.argmin(lasso_validation_results['score']) lasso_model = lasso_validation_results['alpha'][idx_lasso_model] ################ NN ################
x_test, x_validation, y_test, y_validation = train_test_split(x_test, y_test, random_state=0, train_size=0.3) #数据归一化 scaler = preprocessing.StandardScaler().fit(x_train) #保存训练集的标准差和均值 x_train = scaler.transform(x_train) #训练集数据归一化 x_test = scaler.transform(x_test) #测试集使用训练集数据归一化 x_validation = scaler.transform(x_validation) #验证集使用训练集数据归一化 #从2^-5-2^15区间中寻找最优C值,从2^-15-2^3区间寻找最优γ值 CScale = [-5, -3, -1, 1, 3, 5, 7, 9, 11, 13, 15] #gammaScale = [-15, -13, -11, -9, -7, -5,-3,-1,1,3] score = {'c': 0.0, 'g': 0.0, 'score': 0.0} for Cindex, c in enumerate(CScale): # for Gindex,g in enumerate(gammaScale): #clf = svm.SVC(C=pow(2,c), kernel='rbf', gamma=pow(2,g), decision_function_shape='ovo') clf = svm.SVC(C=pow(2, c), kernel='rbf', gamma=7.464263932294464, decision_function_shape='ovo') clf.fit(x_train, y_train.ravel()) #result=clf.score(x_train, y_train) result = clf.score(x_test, y_test) if result > score['score']: score['c'] = Cindex #score['g']=Gindex score['score'] = result n = 10
def detectCharsInPlates(listOfPossiblePlates, filePath): intPlateCounter = 0 imgContours = None contours = [] if len(listOfPossiblePlates) == 0: # if list of possible plates is empty return listOfPossiblePlates # return # end if # at this point we can be sure the list of possible plates has at least one plate for index, possiblePlate in enumerate( listOfPossiblePlates ): # for each possible plate, this is a big for loop that takes up most of the function possiblePlate.imgGrayscale, possiblePlate.imgThresh = Preprocess.preprocess( possiblePlate.imgPlate ) # preprocess to get grayscale and threshold images adaptivePlate = cv2.adaptiveThreshold(possiblePlate.imgGrayscale, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 11, 2) blurPlate = cv2.GaussianBlur(adaptivePlate, (5, 5), 0) ret, processedPlate = cv2.threshold( blurPlate, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) if Main.showSteps == True: # show steps ################################################### cv2.imshow("5a", possiblePlate.imgPlate) cv2.imshow("5b", possiblePlate.imgGrayscale) cv2.imshow("5c.adaptive", adaptivePlate) cv2.imshow("5d.blur", blurPlate) cv2.imshow("5e.otsu", processedPlate) # end if # show steps ##################################################################### # increase size of plate image for easier viewing and char detection possiblePlate.imgThresh = cv2.resize(possiblePlate.imgThresh, (0, 0), fx=1.6, fy=1.6) # find all possible chars in the plate, # this function first finds all contours, then only includes contours that could be chars (without comparison to other chars yet) listOfPossibleCharsInPlate = findPossibleCharsInPlate(adaptivePlate) listOfPossibleCharsInPlate.sort(key=lambda Char: Char.intCenterX) if Main.showSteps == True: # show steps ################################################### showContours(possiblePlate, listOfPossibleCharsInPlate) listOfListsOfChars = findListOfListsOfMatchingChars( listOfPossibleCharsInPlate, minChars=3, maxAngle=10) if len(listOfListsOfChars) == 0: continue # find chars that have same heights listOfListsOfChars1 = [ getEqualHeightList(x) for x in listOfListsOfChars ] listOfListsOfChars2 = getEqualHeightList(listOfListsOfChars1, mode=1) # remove Distance Char listOfListsOfChars3 = [ removeDistanceChar(x) for x in listOfListsOfChars2 ] # flatten list listOfCharsInPlate = [ char for listChars in listOfListsOfChars3 for char in listChars ] # remove inner Chars listOfCharsInPlate = removeInnerChars(listOfCharsInPlate) # number of plate elements must be > 6 if len(listOfCharsInPlate) >= 6: possiblePlate.isPlate = True if Main.showSteps == True: # show steps ####################################################### showListOfLists(possiblePlate, listOfCharsInPlate) # end of big for loop that takes up most of the function possiblePlate.strChars = recognizeCharsInPlate( possiblePlate.imgGrayscale, listOfCharsInPlate, index, filePath) print("predict: ", possiblePlate.strChars) else: continue listOfPlates = [plate for plate in listOfPossiblePlates if plate.isPlate] if Main.showSteps == True: print( "\nchar detection complete, click on any image and press a key to continue . . .\n" ) cv2.waitKey(0) # end if return listOfPlates
def recognizeCharsInPlate(imgThresh, listOfMatchingChars, order, filePath): height, width = imgThresh.shape imgThreshColor = np.zeros((height, width, 3), np.uint8) listOfMatchingChars.sort(key=charPlace) # sort chars from left to right cv2.cvtColor( imgThresh, cv2.COLOR_GRAY2BGR, imgThreshColor ) # make color version of threshold image so we can draw contours in color on it charImages = [] for i, currentChar in enumerate( listOfMatchingChars): # for each char in plate pt1 = (currentChar.intBoundingRectX, currentChar.intBoundingRectY) pt2 = ((currentChar.intBoundingRectX + currentChar.intBoundingRectWidth), (currentChar.intBoundingRectY + currentChar.intBoundingRectHeight)) cv2.rectangle(imgThreshColor, pt1, pt2, Main.SCALAR_GREEN, 2) # draw green box around the char # crop char out of threshold image imgROI = imgThresh[pt1[1]:pt2[1], pt1[0]:pt2[0]] imgROIResized = cv2.resize( imgROI, RESIZED_CHAR ) # resize image, this is necessary for char recognition # retreive binary image from the char images adaptivePlate = cv2.adaptiveThreshold(imgROIResized, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 11, 2) blurPlate = cv2.GaussianBlur(adaptivePlate, (5, 5), 0) ret, im = cv2.threshold(blurPlate, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) if Main.showSteps == True: cv2.imshow("resize_" + str(i), im) cv2.imshow("img_thresh", imgThresh) charImages.append(im) # end for if Main.showSteps == True: # show steps ####################################################### cv2.imshow("8", imgThreshColor) # end if # show steps ######################################################################### cv2.waitKey(0) cv2.destroyAllWindows() chars = recognizeLetter(charImages) strChars = "".join(chars) # Gen Plate Letter # you can write all chars found in a folder by changing the following code if Main.sourceFolder: for i, im in enumerate(charImages): fileName = filePath.split("/")[-1].split(".") plateFolder = join(Main.targetFolder, fileName[0]) extractedPlateName = "_" + str(order) + "_" + str( i) + "_" + chars[i] + "." + fileName[1] resizedChar = cv2.resize(im, RESIZED_CHAR) borderChar = cv2.copyMakeBorder(resizedChar, 5, 5, 5, 5, cv2.BORDER_CONSTANT, value=0) extractedChar = cv2.resize(borderChar, RESIZED_CHAR) cv2.imwrite(plateFolder + extractedPlateName, extractedChar) return strChars
def get_stats(username): ghuser = username.lower() firstname = flask.request.args.get("firstname", username) eventscores = flask.g.redis.zrevrangebyscore("gh:user:{0}:event" .format(ghuser), "+inf", 5, 0, 10, withscores=True) events = [e[0] for e in eventscores] evtcounts = [int(e[1]) for e in eventscores] # Get the user histogram. pipe = flask.g.redis.pipeline() # Get the time zone. pipe.get("gh:user:{0}:tz".format(ghuser)) # Get the total number of events. pipe.zscore("gh:user", ghuser) # Get full commit schedule. pipe.hgetall("gh:user:{0}:date".format(ghuser)) # Get the daily schedule for each type of event. [pipe.hgetall("gh:user:{0}:event:{1}:day".format(ghuser, e)) for e in events] # Get the hourly schedule for each type of event. [pipe.hgetall("gh:user:{0}:event:{1}:hour".format(ghuser, e)) for e in events] # Get the distribution of languages contributed to. pipe.zrevrange("gh:user:{0}:lang".format(ghuser), 0, -1, withscores=True) # Get the vulgarity (and vulgar rank) of the user. pipe.zrevrange("gh:user:{0}:curse".format(ghuser), 0, -1, withscores=True) pipe.zcount("gh:curse:user", 4, "+inf") pipe.zrevrank("gh:curse:user", ghuser) # Get connected users. pipe.zrevrangebyscore("gh:user:{0}:connection".format(ghuser), "+inf", 5, 0, 5) # Fetch the data from the database. raw = pipe.execute() # Get the general stats. tz = int(raw[0]) if raw[0] is not None and raw[0] != "None" else None total = int(raw[1]) if raw[1] is not None else 0 if total == 0: return json.dumps({"message": "Couldn't find any stats for this user."}), 404 # Get the schedule histograms. n, m = 3, len(events) week = zip(*[make_hist(d.iteritems(), 7) for k, d in zip(events, raw[n:n + m])]) offset = tz + 8 if tz is not None else 0 day = zip(*[make_hist(d.iteritems(), 24, offset=offset) for k, d in zip(events, raw[n + m:n + 2 * m])]) # If there's no weekly schedule, we don't have enough info. if not len(week): return json.dumps({"message": "Couldn't find any stats for this user."}), 404 # Get the language proportions. n = n + 2 * m langs = raw[n] curses = raw[n + 1] # Parse the vulgarity factor. vulgarity = None try: vulgarity = int(100 * float(raw[n + 3]) / float(raw[n + 2])) + 1 except: pass # Get the connected users. connections = [c for c in raw[n + 4] if c.lower() != ghuser] # Get language rank. langrank = None langname = None if len(langs): lang = langs[0][0] langname = language_users.get(lang, "{0} expert".format(lang)) # Made up number. How many contributions count as enough? 20? Sure. pipe.zcount("gh:lang:{0}:user".format(lang), 50, "+inf") pipe.zrevrank("gh:lang:{0}:user".format(lang), ghuser) ltot, lrank = pipe.execute() # This user is in the top N percent of users of language "lang". try: langrank = (lang, int(100 * float(lrank) / float(ltot)) + 1) except: pass # Get neighbors. neighbors = get_neighbors(ghuser) # Figure out the representative weekly schedule. hacker_type = "a pretty inconsistent hacker" if len(week): mu = np.sum(week, axis=1) mu /= np.sum(mu) hacker_type = mean_desc[np.argmin(np.sum(np.abs(means - mu[None, :]), axis=1))] # Build a human readable summary. summary = "<p>" if langname: adj = np.random.choice(["a high caliber", "a heavy hitting", "a serious", "an awesome", "a top notch", "a trend setting", "a champion"]) summary += ("{0} is {2} <a href=\"#languages\">{1}</a>" .format(firstname, langname, adj)) if langrank and langrank[1] < 50: summary += (" (one of the top {0}% most active {1} users)" .format(langrank[1], langrank[0])) if len(events): if events[0] in evtactions: summary += (" who <a href=\"#events\">would rather be {0} " "instead of pushing code</a>").format( evtactions[events[0]]) elif events[0] == "PushEvent": summary += " who <a href=\"#events\">loves pushing code</a>" summary += ". " summary += "{0} is <a href=\"#week\">{1}</a>".format(firstname, hacker_type) if len(day): best_hour = np.argmax(np.sum(day, axis=1)) if 0 <= best_hour < 7: tod = "wee hours" elif 7 <= best_hour < 12: tod = "morning" elif 12 <= best_hour < 18: tod = "mid-afternoon" elif 18 <= best_hour < 21: tod = "evening" else: tod = "late evening" summary += " who seems to <a href=\"#day\">work best in the {0}</a>" \ .format(tod) summary += ". " if vulgarity: if vulgarity < 50: summary += ("I hate to say it but {0} does seem—as one of " "the top {1}% most vulgar users on GitHub—to " "be <a href=\"#swearing\">a tad foul-mouthed</a> " "(with a particular affinity " "for filthy words like '{2}').").format(firstname, vulgarity, curses[0][0]) elif vulgarity < 100: summary += ("I hate to say it but {0} is becoming—as one of " "the top {1}% most vulgar users on GitHub—" "<a href=\"#swearing\">a tad foul-mouthed</a> " "(with a particular affinity " "for filthy words like '{2}').").format(firstname, vulgarity, curses[0][0]) summary += "</p>" # Add similar and connected users to summary. if len(week) and (len(neighbors) or len(connections)): summary += "<p>" if len(neighbors): ind = np.random.randint(len(neighbors)) summary += ("{0}'s behavior is quite similar to <a " "href=\"{2}\">{1}</a>'s but <span " "class=\"comparison\" data-url=\"{3}\"></span>. ") \ .format(firstname, neighbors[ind], flask.url_for(".user", username=neighbors[ind]), flask.url_for(".compare", username=ghuser, other=neighbors[ind])) if len(neighbors) == 2: ind = (ind + 1) % 2 summary += ("<a href=\"{1}\">{0}</a>'s activity stream also " "shows remarkable similarities to {2}'s " "behavior. ").format(neighbors[ind], flask.url_for(".user", username= neighbors[ind]), firstname) elif len(neighbors) > 2: summary += ("It would also be impossible to look at {0}'s " "activity stream and not compare it to those " "of ").format(firstname) cus = [] for i in range(len(neighbors)): if i != ind: cus.append("<a href=\"{1}\">{0}</a>" .format(neighbors[i], flask.url_for(".user", username= neighbors[i]))) summary += ", ".join(cus[:-1]) summary += " and " + cus[-1] + ". " if len(connections): ind = 0 summary += ("It seems—from their activity streams—" "that {0} and <a href=\"{2}\">{1}</a> are probably " "friends or at least virtual friends. With this in " "mind, it's worth noting that <span " "class=\"comparison\" data-url=\"{3}\"></span>. ") \ .format(firstname, connections[ind], flask.url_for(".user", username=connections[ind]), flask.url_for(".compare", username=ghuser, other=connections[ind])) if len(connections) > 2: summary += ("There is also an obvious connection between " "{0} and ").format(firstname) cus = [] for i in range(len(connections)): if i != ind: cus.append("<a href=\"{1}\">{0}</a>" .format(connections[i], flask.url_for(".user", username= connections[i]))) summary += ", ".join(cus[:-1]) summary += " and " + cus[-1] + ". " summary += "</p>" # Summary text for schedule graphs. sctxt = "" if len(events): sctxt = ("<p>The two following graphs show {0}'s average weekly and " "daily schedules. These charts give significant insight " "into {0}'s character as a developer. ").format(firstname) if len(events) == 1: sctxt += "All of the events in {0}'s activity stream are {1}. " \ .format(firstname, evtverbs.get(events[0])) else: sctxt += ("The colors in the charts indicate the fraction of " "events that are ") for i, e in enumerate(events): if i == len(events) - 1: sctxt += "and " sctxt += ("<span class=\"evttype\" data-ind=\"{1}\">{0}" "</span>").format(evtverbs.get(e), i) if i < len(events) - 1: sctxt += ", " sctxt += ". " sctxt += """</p> <div class="hist-block"> <div id="week" class="hist"></div> <div id="day" class="hist"></div> </div> <p>""" sctxt += ("Based on this average weekly schedule, we can " "describe {0} as " "<strong>{1}</strong>. ").format(firstname, hacker_type) if len(day): if best_hour == 0: tm = "midnight" elif best_hour == 12: tm = "noon" else: tm = "{0}{1}".format(best_hour % 12, "am" if best_hour < 12 else "pm") sctxt += ("Since {0}'s most active time is around {1}, I would " "conclude that {0} works best in the " "<strong>{2}</strong>. ").format(firstname, tm, tod) sctxt += ("It is important to note that an attempt has been made " "to show the daily schedule in the correct time zone " "but this procedure is imperfect at best. ") sctxt += "</p>" if len(events) > 1: sctxt += ("<p>The following chart shows number of events of " "different types in {0}'s activity stream. In the " "time frame included in this analysis, {0}'s event " "stream included " "a total of {1} events and they are all ") \ .format(firstname, sum(evtcounts)) for i, e in enumerate(events): if i == len(events) - 1: sctxt += "or " sctxt += ("<span class=\"evttype\" data-ind=\"{1}\">{0}" "</span>").format(evtverbs.get(e), i) if i < len(events) - 1: sctxt += ", " sctxt += ". " sctxt += """</p><div class="hist-block"> <div id="events"></div> </div>""" if langs and len(langs) > 1: sctxt += ("<p>{0} has contributed to repositories in {1} " "different languages. ").format(firstname, len(langs)) sctxt += ("In particular, {0} is a serious <strong>{1}</strong> " "expert").format(firstname, langs[0][0]) ls = [float(l[1]) for l in langs] if (ls[0] - ls[1]) / sum(ls) < 0.25: sctxt += (" with a surprisingly broad knowledge of {0} " "as well").format(langs[1][0]) sctxt += ". " sctxt += ("The following chart shows the number of contributions " "made by {0} to repositories where the main " "language is listed as ").format(firstname) for i, l in enumerate(langs): if i == len(langs) - 1: sctxt += "or " sctxt += ("<span class=\"evttype\" data-ind=\"{1}\">{0}" "</span>").format(l[0], i) if i < len(langs) - 1: sctxt += ", " sctxt += "." sctxt += """</p><div class="hist-block"> <div id="languages"></div> </div>""" if langs and len(langs) == 1: sctxt += ("<p>{0} seems to speak only one programming language: " "<strong>{1}</strong>. Maybe it's about time to branch " "out a bit.</p>").format(firstname, langs[0][0]) # Format the results. results = {"summary": summary} results["events"] = [" ".join(re.findall("([A-Z][a-z]+)", e)) for e in events] results["event_counts"] = evtcounts results["tz"] = tz results["total"] = total results["week"] = week results["hacker_type"] = hacker_type results["day"] = day results["schedule_text"] = sctxt # results["activity"] = raw[2].items() results["languages"] = langs results["lang_user"] = langname results["language_rank"] = langrank results["curses"] = curses results["vulgarity"] = vulgarity results["similar_users"] = neighbors results["connected_users"] = connections return json.dumps(results)
exit(1) if opts.store_file: vcd_store = VCDStore(fs_db_path=opts.store_file) else: vcd_store = VCDStore() overwrite = opts.overwrite ### Loading up clip IDs and numpy file, pairing parallel rows print "Extracting clip IDs and features..." clip_ids = map(int, (l.strip(' \t\n') for l in open(opts.clip_id_file).readlines())) npy_matrix = np.load(opts.npy_file) assert len(clip_ids) == npy_matrix.shape[0], \ "ERROR: Number of clip ids found did not match the numpy matrix size! " \ "(clip ids: %d) (np arrays: %d)" \ % (len(clip_ids), npy_matrix.shape[0]) print "-->", len(clip_ids) ### Creating VCDStoreElements elems = [None] * len(clip_ids) for i, (cid, np_array) in enumerate(zip(clip_ids, npy_matrix)): elems[i] = VCDStoreElement(descr_name, cid, np_array) print "VCD Elements generated:", len(elems) ### Storage into VCDStore vcd_store.store_feature(elems, overwrite=overwrite)
def _compare_document(self, system, reference): """Compute the F1 score for a single document, given a system output and a reference. This is done by computing a precision according to the best possible matching of annotations from the system's perspective, and a recall according to the best possible matching of annotations from the reference perspective. Gives some partial credit to annotations that match with the wrong severity. Args: system: dictionary mapping doc id's to lists of annotations reference: dictionary mapping doc id's to lists of annotations Returns: The F1 score of a single document. """ num_f1_scores = 8 all_matched = np.zeros((num_f1_scores, len(system), len(reference))) annotation_labels = defaultdict(list) for i, system_annotation in enumerate(system): for j, reference_annotation in enumerate(reference): ( matched, severity_exact_matched, category_exact_matched, both_exact_matched, category_exact_top_matched, severity_discount_matched, category_discount_matched, both_discount_matched, ) = reference_annotation.count_overlap( system_annotation, severity_match=self.severity_match, category_match=self.category_match ) all_matched[:, i, j] = [ matched, severity_exact_matched, category_exact_matched, both_exact_matched, category_exact_top_matched, severity_discount_matched, category_discount_matched, both_discount_matched, ] # only consider a label as a 'prediction' when there is _some_ overlap if matched > 0: annotation_labels['system_severities'].append( system_annotation.severity) annotation_labels['system_categories'].append( system_annotation.category) annotation_labels['reference_severities'].append( reference_annotation.severity) annotation_labels['reference_categories'].append( reference_annotation.category) lengths_sys = np.array([len(annotation) for annotation in system]) lengths_ref = np.array([len(annotation) for annotation in reference]) if lengths_sys.sum() == 0: # no system annotations precision = np.ones(num_f1_scores) elif lengths_ref.sum() == 0: # there were no references precision = np.zeros(num_f1_scores) else: # normalize by annotation length precision_by_annotation = all_matched.max(2) / lengths_sys precision = precision_by_annotation.mean(1) # same as above, for recall now if lengths_ref.sum() == 0: recall = np.ones(num_f1_scores) elif lengths_sys.sum() == 0: recall = np.zeros(num_f1_scores) else: recall_by_annotation = all_matched.max(1) / lengths_ref recall = recall_by_annotation.mean(1) # simultaneous division of all the values, with division by zero defaulting to 0 f1 = np.divide( 2*precision*recall, precision + recall, out=np.zeros_like(precision), where=precision + recall != 0 ) assert 0. <= f1.min() and f1.max() <= 1. return tuple(f1), dict(annotation_labels)
def main(): global tmpSettingsDir, availableBuildSystems qtVersionsForQuick = ["5.3"] availableBuildSystems = ["qmake", "Qbs"] if platform.system() != 'Darwin': qtVersionsForQuick.append("5.4") if which("cmake"): availableBuildSystems.append("CMake") else: test.warning("Could not find cmake in PATH - several tests won't run without.") startApplication("qtcreator" + SettingsPath) if not startedWithoutPluginError(): return kits = getConfiguredKits() test.log("Collecting potential project types...") availableProjectTypes = [] invokeMenuItem("File", "New File or Project...") categoriesView = waitForObject(":New.templateCategoryView_QTreeView") catModel = categoriesView.model() projects = catModel.index(0, 0) test.compare("Projects", str(projects.data())) comboBox = findObject(":New.comboBox_QComboBox") targets = zip(*kits.values())[0] test.verify(comboBox.enabled, "Verifying whether combobox is enabled.") test.compare(comboBox.currentText, "All Templates") try: selectFromCombo(comboBox, "All Templates") except: test.warning("Could not explicitly select 'All Templates' from combobox.") for category in [item.replace(".", "\\.") for item in dumpItems(catModel, projects)]: # skip non-configurable if "Import" in category: continue clickItem(categoriesView, "Projects." + category, 5, 5, 0, Qt.LeftButton) templatesView = waitForObject("{name='templatesView' type='QListView' visible='1'}") # needed because categoriesView and templatesView using same model for template in dumpItems(templatesView.model(), templatesView.rootIndex()): template = template.replace(".", "\\.") # skip non-configurable if not template in ["Qt Quick UI Prototype", "Qt Canvas 3D Application", "Auto Test Project"]: # FIXME availableProjectTypes.append({category:template}) safeClickButton("Cancel") for current in availableProjectTypes: category = current.keys()[0] template = current.values()[0] displayedPlatforms = __createProject__(category, template) if template == "Qt Quick Application" or template == "Qt Quick Controls Application": for counter, qtVersion in enumerate(qtVersionsForQuick): def additionalFunc(displayedPlatforms, qtVersion): requiredQtVersion = __createProjectHandleQtQuickSelection__(qtVersion) __modifyAvailableTargets__(displayedPlatforms, requiredQtVersion, True) handleBuildSystemVerifyKits(category, template, kits, displayedPlatforms, additionalFunc, qtVersion) # are there more Quick combinations - then recreate this project if counter < len(qtVersionsForQuick) - 1: displayedPlatforms = __createProject__(category, template) continue elif template == "Qt Quick Controls 2 Application": # needs a Qt5.7 def additionalFunc(displayedPlatforms): clickButton(waitForObject(":Next_QPushButton")) # ignore this details page for now handleBuildSystemVerifyKits(category, template, kits, displayedPlatforms, additionalFunc) continue elif template.startswith("Plain C"): handleBuildSystemVerifyKits(category, template, kits, displayedPlatforms) continue handleBuildSystemVerifyKits(category, template, kits, displayedPlatforms) invokeMenuItem("File", "Exit")
import calendar import sys, traceback import logging import os logger = logging.getLogger('PricesPaidTrans') hdlr = logging.FileHandler('../logs/PricesPaidTrans.log') formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(logging.ERROR) # This works in 2.7, which I don't have on the AWS instance yet! # monthLookup = {v.upper(): k for k,v in enumerate(calendar.month_abbr)} monthLookup = dict((v.upper(), k) for k, v in enumerate(calendar.month_abbr)) # This are magic numbers that are simply based on the RevAuc files # as Laura presented them to me. def findMonthFromAbbrev(a): global monthLookup try: m = monthLookup[a] return m except KeyError as e: logger.error('Caught error ' + repr(e) + a) return 0 # Need to test this month lookup stuff, right now
def draw_trefoil3d(x=None, y=None, z=None, c=None, s='z', ax=None, fig=None, view=(90, -90), **kwargs): """Plot trefoil knot. """ if c is None: c = np.copy(z) cmap = plt.get_cmap(kwargs.get('cmap', "brg"), 3) norm = mpl.colors.Normalize(c.min(), c.max()) # extract x, y, z if s == 'z': zbins = np.linspace(z.min(), z.max(), num=10) zbins = np.digitize(z, zbins) s = zbins**2 # combine features X = np.c_[x, y, z] # plot data fig, axes = plt.subplots(1, 3, figsize=(15,5),subplot_kw=dict(projection='3d')) # 3 views for ax_i, ax in enumerate(axes): if ax_i == 0: xcol, ycol, zcol = 0, 1, 2 elif ax_i == 1: xcol, ycol, zcol = 0, 2, 1 elif ax_i == 2: xcol, ycol, zcol = 1, 2, 0 zbins = np.linspace(z.min(), z.max(), num=10) zbins = np.digitize(z, zbins) ax.scatter(X[:,xcol], X[:,ycol], X[:,zcol], c=c, s=s, alpha=0.8, cmap=cmap, norm=norm) ax.set_xlabel(list('xyz')[xcol], fontweight='bold') ax.set_ylabel(list('xyz')[ycol], fontweight='bold') ax.set_zlabel(list('xyz')[zcol], fontweight='bold') ax.set_xticks([]) ax.set_yticks([]) ax.set_zticks([]) # Get rid of colored axes planes # First remove fill ax.xaxis.pane.fill = False ax.yaxis.pane.fill = False ax.zaxis.pane.fill = False # Now set color to white (or whatever is "invisible") ax.xaxis.pane.set_edgecolor('w') ax.yaxis.pane.set_edgecolor('w') ax.zaxis.pane.set_edgecolor('w') # Bonus: To get rid of the grid as well: ax.grid(False) #if ax_i == 1: # view = (view[0]+0, view[1]+90) #if ax_i == 2: # view = (view[0]+90, view[1]+0) ax.set_title("view = {}".format(view)) ax.view_init(*view) return axes
plt.ioff() currFolder = os.path.dirname(os.path.realpath(__file__)) os.chdir(currFolder) patchesToShow = 'finalPatches' #patchesToShow = 'finalPatchesMarked' columnNum = 4 trialList = [ '160208_M193206_Trial1.pkl', ] for k, trialName in enumerate(trialList): trialPath = os.path.join(currFolder,trialName) trial, _ = rm.loadTrial(trialPath) finalPatches = getattr(trial,patchesToShow) numOfPatches = len(finalPatches.keys()) rowNum = numOfPatches // columnNum + 1 f = plt.figure(figsize=(10,10)) f.suptitle(trialName) ax = f.add_subplot(111) rm.plotPatches(finalPatches,plotaxis=ax,markersize=0) for key,patch in finalPatches.iteritems(): center = patch.getCenter()
db = config_dict['数据库名称'].strip() connect = pymysql.connect(host='localhost', user=user, passwd=passwd, db=db, charset='utf8') # 捕获异常信息 try: cursor = connect.cursor() cursor.execute("truncate table jd_help") connect.commit() coupon = [] with open('coupon.txt', 'r') as f: coupons = f.readlines() for k, cou in enumerate(coupons): coupon = 'http://coupon.m.jd.com/coupons/show.action?{}&to=m.jd.com'.format( cou.strip()) res = requests.get( url=coupon, headers={ 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' }) ur_yuan = res.content.decode('utf-8') limitStr = re.search('limitStr":"(.*?)"', ur_yuan, re.S).group(1) discount = re.search('discount":"(.*?)"', ur_yuan, re.S).group(1) quota = re.search('quota":"(.*?)"', ur_yuan, re.S).group(1) batchid = re.search('batchId":"(.*?)"', ur_yuan, re.S).group(1) batchcount = re.search('batchCount":(\d+),', ur_yuan, re.S).group(1) des = limitStr + quota + '-' + discount
def to_one_hot(labels, dimension=3): results = np.zeros((len(labels), dimension)) for i, label in enumerate(labels): results[i, label] = 1. return results
def flush(self): if not len(self.incoming_stats): logger.info("Nothing to report") return logger.info("Loss on {} batches".format(len(self.incoming_stats))) batch_stats = merge_tensor_namedtuple_list(self.incoming_stats, BatchStats) batch_stats.write_summary(self.action_names) print_details = "Loss:\n" td_loss_mean = float(batch_stats.td_loss.mean()) self.td_loss.append(td_loss_mean) print_details = print_details + "TD LOSS: {0:.3f}\n".format(td_loss_mean) if batch_stats.logged_rewards is not None: flattened_rewards = torch.flatten(batch_stats.logged_rewards).tolist() self.running_reward.extend(flattened_rewards) if batch_stats.reward_loss is not None: reward_loss_mean = float(batch_stats.reward_loss.mean()) self.reward_loss.append(reward_loss_mean) print_details = print_details + "REWARD LOSS: {0:.3f}\n".format( reward_loss_mean ) if batch_stats.imitator_loss is not None: imitator_loss_mean = float(batch_stats.imitator_loss.mean()) self.imitator_loss.append(imitator_loss_mean) print_details = print_details + "IMITATOR LOSS: {0:.3f}\n".format( imitator_loss_mean ) if batch_stats.model_values is not None and self.action_names: self.model_values.append( dict(zip(self.action_names, batch_stats.model_values.mean(dim=0))) ) self.model_value_stds.append( dict(zip(self.action_names, batch_stats.model_values.std(dim=0))) ) if batch_stats.model_values_on_logged_actions is not None: self.logged_action_q_value.append( batch_stats.model_values_on_logged_actions.mean().item() ) if ( batch_stats.logged_actions is not None and batch_stats.model_action_idxs is not None ): logged_action_counts = { action: (batch_stats.logged_actions == i).sum().item() for i, action in enumerate(self.action_names) } model_action_counts = { action: (batch_stats.model_action_idxs == i).sum().item() for i, action in enumerate(self.action_names) } print_details += "The distribution of logged actions : {}\n".format( logged_action_counts ) print_details += "The distribution of model actions : {}\n".format( model_action_counts ) for action, count in logged_action_counts.items(): self.logged_action_counts[action] += count self.model_action_counts.append(model_action_counts) for action, count in model_action_counts.items(): self.model_action_counts_cumulative[action] += count total = float(sum(model_action_counts.values())) self.model_action_distr.append( {action: count / total for action, count in model_action_counts.items()} ) print_details += "Batch Evaluator Finished" for print_detail in print_details.split("\n"): logger.info(print_detail) self.incoming_stats.clear()
def _base_iterator(self, circuit: circuits.Circuit, qubit_order: ops.QubitOrderOrList, initial_state: Union[int, np.ndarray], all_measurements_are_terminal=False) -> Iterator: qubits = ops.QubitOrder.as_qubit_order(qubit_order).order_for( circuit.all_qubits()) qid_shape = protocols.qid_shape(qubits) qubit_map = {q: i for i, q in enumerate(qubits)} initial_matrix = qis.to_valid_density_matrix(initial_state, len(qid_shape), qid_shape=qid_shape, dtype=self._dtype) measured = collections.defaultdict( bool) # type: Dict[Tuple[cirq.Qid, ...], bool] if len(circuit) == 0: yield DensityMatrixStepResult(initial_matrix, {}, qubit_map, self._dtype) return state = _StateAndBuffers(len(qid_shape), initial_matrix.reshape(qid_shape * 2)) def on_stuck(bad_op: ops.Operation): return TypeError( "Can't simulate operations that don't implement " "SupportsUnitary, SupportsConsistentApplyUnitary, " "SupportsMixture, SupportsChannel or is a measurement: {!r}". format(bad_op)) def keep(potential_op: ops.Operation) -> bool: return (protocols.has_channel(potential_op) or isinstance(potential_op.gate, ops.MeasurementGate)) noisy_moments = self.noise.noisy_moments(circuit, sorted(circuit.all_qubits())) for moment in noisy_moments: measurements = collections.defaultdict( list) # type: Dict[str, List[int]] channel_ops_and_measurements = protocols.decompose( moment, keep=keep, on_stuck_raise=on_stuck) for op in channel_ops_and_measurements: indices = [qubit_map[qubit] for qubit in op.qubits] # TODO: support more general measurements. # Github issue: https://github.com/quantumlib/Cirq/issues/1357 if all_measurements_are_terminal and measured[op.qubits]: continue if isinstance(op.gate, ops.MeasurementGate): measured[op.qubits] = True meas = op.gate if all_measurements_are_terminal: continue if self._ignore_measurement_results: for i, q in enumerate(op.qubits): self._apply_op_channel( ops.phase_damp(1).on(q), state, [indices[i]]) else: invert_mask = meas.full_invert_mask() # Measure updates inline. bits, _ = (density_matrix_utils.measure_density_matrix( state.tensor, indices, qid_shape=qid_shape, out=state.tensor, seed=self._prng)) corrected = [ bit ^ (bit < 2 and mask) for bit, mask in zip(bits, invert_mask) ] key = protocols.measurement_key(meas) measurements[key].extend(corrected) else: self._apply_op_channel(op, state, indices) yield DensityMatrixStepResult(density_matrix=state.tensor, measurements=measurements, qubit_map=qubit_map, dtype=self._dtype)
cfg = Config.Config(options.config) box = options.box inputData = cfg.getVariables(box, "inputDataNtu") treeName = cfg.getVariables(box, "treeName") outFile = cfg.getVariables(box, "outFile") loose = cfg.getVariables(box, "loose") medium = cfg.getVariables(box, "medium") dR = cfg.getVariables(box, "deltaR") ################################################################### print treeName tchain = rt.TChain(treeName) for i, line in enumerate(inputData): tchain.Add(line) nEntries = tchain.GetEntries() print 'Number of entries: ', nEntries # Book histos #bins,min,max ptRange = [100, 0, 1000] mjjRange = [150, 500, 2000] etaRange = [100, -3.5, 3.5] csvRange = [20, 0, 1] allHistos = [] ##calo
# set path to ground truth str_gt = os.path.join(str_gt_path, 'derivatives', str_subj, 'labeled', str_subj + '_labels_v01.nii.gz') # %% # Prepare lists for different metrics resultsAcc = [] resultsF1 = [] # Load ground truth image tns_gt = func_nii_to_tf_tensor(str_gt, dtype=np.int32) # loop over models for indModel, sg_model in enumerate(lst_sg_models): model_name = lst_model_headers[indModel] print('---Working on model: ' + model_name) # Append model name to lists for row header resultsAcc.append([model_name]) resultsF1.append([model_name]) # loop over augmentations for aug_stride in lst_aug_strides: print('---Working on augmentation: ' + aug_stride) # derive path to predicted segmentations str_ps = os.path.join(str_sg_path, sg_model, aug_stride, 'pred.nii.gz') # derive path to predicted probabilities
def grader(e, ans): state = json.loads(ans) #['answer'] response = state['response'] colors = state['colors'] #return {'ok': False, 'msg': '%s' % str(response)} Numeric = 'NUMERIC_RESPONSE' questionTable = [ [ 'Mass of Falling Object<br>M(kg)', 'Distance Object Fall<br>D(m)', 'Times for Fall<br>t(s)', 'Calculated Average Acceleration of Falling Object<br>a(m/s^2)', 'Calculated Tension in String<br>T(N)', 'Calculated Torque Exerted on Post of Apparatus<br>T*D(m*N)' ], ['0.100', '0.500', '9.7, 10.0, 10.3', '0.010', Numeric, Numeric], ['0.150', '0.500', '6.9, 7.1, 7.3', '0.020', Numeric, Numeric], ['0.200', '0.500', '5.8, 5.7, 5.6', '0.031', Numeric, Numeric], ['0.250', '0.500', '4.9, 4.9, 4.8', '0.042', Numeric, Numeric], ] answerTable = [ [ 'Mass of Falling Object<br>M(kg)', 'Distance Object Fall<br>D(m)', 'Times for Fall<br>t(s)', 'Calculated Average Acceleration of Falling Object<br>a(m/s^2)', 'Calculated Tension in String<br>T(N)', 'Calculated Torque Exerted on Post of Apparatus<br>T*D(m*N)' ], ['0.100', '0.500', '9.7, 10.0, 10.3', '0.010', 0.96, 0.0011], ['0.150', '0.500', '6.9, 7.1, 7.3', '0.020', 1.47, 0.0016], ['0.200', '0.500', '5.8, 5.7, 5.6', '0.031', 1.95, 0.0021], ['0.250', '0.500', '4.9, 4.9, 4.8', '0.042', 2.44, 0.0027], ] ### Must remove header for comparison in loop below ### JQuery takes cells without a header, so i,j values must only be fore table body. answerTable = answerTable[1::] def stringComparison(R, K): ''' R: Response from student K: Answer from (K)ey ''' if str(R) == str(K): return True else: return False def testNumeric(X): ''' X: entry taken from Active Table ''' try: float(X) return True except: return False def numericalComparison(R, K, **kwargs): ''' R: Response from student K: Answer from (K)ey tolerance: percentage tolerance allowed in numerical response *** Implement edX tolerance: https://github.com/edx/edx-platform/blob/dbc465a51871bd685dd925c23bf73b981e07abe6/common/lib/capa/capa/util.py#L14 Currently Calculated: return 100.*abs(student - instructor)/instructor <= tolerance ''' percent_tolerance = kwargs.get('percent_tolerance', 1.0) if testNumeric(R) and testNumeric(K): return 100. * abs(float(R) - float(K)) / float(K) <= percent_tolerance else: #print "Issue with grading rubric. Could not convert Response to float." return False def returnID(row, column): return str(row) + '___' + str(column).replace(' ', '') def feedback(cellType, R, K): ''' cellType: type of cell input, options are [NUMERIC_RESPONSE,STRING_RESPONSE] R: short-hand for student Response Value K: short-hand for answer Key Value ''' if cellType == 'NUMERIC_RESPONSE': return numericalComparison(R, K, percent_tolerance=5.0) #return stringComparison(R,K) elif cellType == 'STRING_RESPONSE': return stringComparison(R, K) ### Loop through question Table and use indices to check answer table header = questionTable[0] numCorrects = 0 for i, row in enumerate(questionTable[1::]): for j, cellType in enumerate(row): ID = returnID(i, header[j]) #print i,j,header[j],ID,answerTable[i][j] ### Check for Existent Active Cell if ID in response: ### Compare answer key and contents of cell if feedback(cellType, response[ID], answerTable[i][j]): numCorrects = numCorrects + 1 else: state['colors'][ID] = 'LightPink' # ### Check for Active Cell # if cellType == 'NUMERIC_RESPONSE': # ID = returnID(i,header[j]) # ### Test that ID is in passed answer from edX # if ID in response: # #print i,j,header[j],ID,answerTable[i][j] # ### Compare Responses to Key # if not stringComparison(response[ID],answerTable[i][j]): # #print "Incorrect" # state['colors'][ID] = 'LightPink' # else: # numCorrects = numCorrects + 1 if numCorrects == len(response.keys()): return {'ok': True, 'msg': 'Great job!'} else: return { 'ok': False, 'msg': 'You have %s cells out of %s correct.' % (str(numCorrects), str(len(response.keys()))) }
# coding: utf-8 import string import numpy as np task_time = {letter: 61 + i for i, letter in enumerate(string.ascii_uppercase)} def next_task(tasklist): for task in string.ascii_uppercase: try: if len(tasklist[task]) == 0: return task except KeyError: pass return None def set_task_done(task, tasklist): for dependencies in tasklist.values(): try: dependencies.remove(task) except ValueError: pass todo = set(string.ascii_uppercase) tasks = {letter: [] for letter in string.ascii_uppercase} with open('input7.txt') as f: for line in f: _, pre, _, _, _, _, _, post, _, _ = line.split() tasks[post].append(pre)
def search(query, docTermIndex): print('\nRetrieving documents for query \'{}\'\n'.format(query)) qlist = query.strip().split() #Remove stop words modified = [ term for term in qlist if term not in stopwords.words('english') ] EXPAND = False '''Expansion EXPAND = True for term in modified: syns = wordnet.synsets(term) for i in range(len(syns)): #print(syns[i].lemmas()[0].name()) new.append(syns[i].lemmas()[0].name()) ''' '''''' if EXPAND == False: new = modified before_stem = np.unique(new) ps = stem.PorterStemmer() after_stem = [ps.stem(word) for word in before_stem] mQuery = np.unique(after_stem) mQuery #read potentially relevant docs into matrix using tf*idf weights qMatrix = pd.DataFrame(np.zeros((0, len(mQuery))), columns=mQuery) for term in mQuery: if term in docTermIndex.keys(): termInfo = docTermIndex.get(term) for occurence in termInfo.occList: if occurence.docID not in qMatrix.index: toAppend = pd.Series(np.zeros(len(qMatrix.columns)), index=qMatrix.columns, name=occurence.docID) toAppend[term] = occurence.count * termInfo.idf qMatrix = qMatrix.append(toAppend) else: qMatrix.loc[occurence.docID, term] = occurence.count * termInfo.idf #compute tfxidf vector of query #print(qMatrix.columns) q_vect = [docTermIndex.get(term).idf for term in qMatrix.columns] #Get cosine similarities for query matrix_norm = np.array( [np.linalg.norm(qMatrix.iloc[i]) for i in range(len(qMatrix))]) q_norm = np.linalg.norm(q_vect) sims = np.dot(qMatrix, q_vect) / (matrix_norm * q_norm) dists = 1 - sims idx = np.argsort(dists) user_docs = qMatrix.iloc[idx[:10]].index classes = pd.read_csv('classes.csv', index_col=1) for i, path in enumerate(user_docs): parts = path.split('/') group = parts[1] file = parts[2] print('----{}: File {} in folder {}-----\n'.format(i + 1, file, group)) with open(path, 'r', errors='ignore') as myfile: data = myfile.read() art = data ind = art.find('\n\n') art = art[ind + 2:] #If article(and not post), re-index again to get rid of tags if art[0:10].find('archive') != -1: ind = art.find('\n\n') art = art[ind + 2:] mid = len(art) // 2 midmid = mid // 2 print('---------------------------------------------\n') print(art[mid:mid + 200]) print('---------------------------------------------\n') return user_docs
data = [111, 444, 555, 888, 412, 647, 951, 789, 654, 258, 5, 4, 234, 543, 345, 123235245, 213, 435, 2334, 345 ] min_valid_amount = 200 max_valid_amount = 400 for index in range(len(data) - 1, -1, -1): if data[index] > max_valid_amount or data[index] < min_valid_amount: print(index, data) del data[index] print(data) # another backwards/reverse way top_index = len(data) - 1 for index, number in enumerate(reversed(data)): if number < min_valid_amount or number > min_valid_amount: print(top_index - index, number) del data[top_index - index] print(data)
def tempo_polyco_table_writer(polycoTable, filename='polyco.dat'): """ Write tempo style polyco file from an astropy table Tempo style polyco file: The polynomial ephemerides are written to file 'polyco.dat'. Entries are listed sequentially within the file. The file format is:: Line Columns Item ---- ------- ----------------------------------- 1 1-10 Pulsar Name 11-19 Date (dd-mmm-yy) 20-31 UTC (hhmmss.ss) 32-51 TMID (MJD) 52-72 DM 74-79 Doppler shift due to earth motion (10^-4) 80-86 Log_10 of fit rms residual in periods 2 1-20 Reference Phase (RPHASE) 21-38 Reference rotation frequency (F0) 39-43 Observatory number 44-49 Data span (minutes) 50-54 Number of coefficients 55-75 Observing frequency (MHz) 76-80 Binary phase 3* 1-25 Coefficient 1 (COEFF(1)) 26-50 Coefficient 2 (COEFF(2)) 51-75 Coefficient 3 (COEFF(3)) * Subsequent lines have three coefficients each, up to NCOEFF One polyco file could include more then one entrie The pulse phase and frequency at time T are then calculated as:: DT = (T-TMID)*1440 PHASE = RPHASE + DT*60*F0 + COEFF(1) + DT*COEFF(2) + DT^2*COEFF(3) + .... FREQ(Hz) = F0 + (1/60)*(COEFF(2) + 2*DT*COEFF(3) + 3*DT^2*COEFF(4) + ....) Parameters --------- polycoTable: astropy table Polycos style table filename : str Name of the output poloco file. References ---------- http://tempo.sourceforge.net/ref_man_sections/tz-polyco.txt """ f = open(filename, 'w') lenTable = len(polycoTable) if lenTable == 0: errorMssg = ("Insufficent polyco data." + " Please make sure polycoTable has data.") raise AttributeError(errorMssg) for i in range(lenTable): entry = polycoTable['entry'][i] psrname = polycoTable['psr'][i].ljust(10) dateDMY = polycoTable['date'][i].ljust(10) utcHMS = polycoTable['utc'][i][0:9].ljust(10) tmid_mjd = utils.longdouble2str(entry.tmid.value) + ' ' dm = str(polycoTable['dm'][i]).ljust(72 - 52 + 1) dshift = str(polycoTable['doppler'][i]).ljust(79 - 74 + 1) logrms = str(polycoTable['logrms'][i]).ljust(80 - 86 + 1) line1 = psrname + dateDMY + utcHMS + tmid_mjd + dm + dshift + logrms + '\n' # Get the reference phase rph = (entry.rphase.int + entry.rphase.frac).value[0] # FIXME: sometimes raises error, sometimes loses precision! rphase = utils.longdouble2str(rph)[0:19].ljust(20) f0 = ('%.12lf' % entry.f0).ljust(38 - 21 + 1) obs = entry.obs.ljust(43 - 39 + 1) tspan = str(round(entry.mjdspan.to('min').value, 4))[0].ljust(49 - 44 + 1) if len(tspan) >= (49 - 44 + 1): # Hack to fix read errors in python tspan = tspan + ' ' ncoeff = str(entry.ncoeff).ljust(54 - 50 + 1) obsfreq = str(polycoTable['obsfreq'][i]).ljust(75 - 55 + 1) binPhase = str(polycoTable['binary_phase'][i]).ljust(80 - 76 + 1) line2 = rphase + f0 + obs + tspan + ncoeff + obsfreq + binPhase + '\n' coeffBlock = "" for j, coeff in enumerate(entry.coeffs): coeffBlock += ('%.17e' % coeff).ljust(25) if (j + 1) % 3 == 0: coeffBlock += '\n' f.write(line1 + line2 + coeffBlock) f.close()
raw_imgs = crop_imgs raw_masks = crop_masks height, width = raw_imgs.shape[1:3] zoomed_imgs = zoom(raw_imgs, (slice_resolution, new_height/height, new_width/width)) np.save(os.path.join(des_home, x+".npy"), zoomed_imgs) zoomed_masks = zoom(raw_masks, (slice_resolution, new_height/height, new_width/width)) np.save(os.path.join(des_home, x+"-dlmask.npy"), zoomed_masks) immasks = np.concatenate([zoomed_imgs, zoomed_masks*255], axis=2)[length//2] cv2.imwrite(f"debug/{x}.png", immasks) #imgs2vid(immasks, "debug/{}.avi".format(x)) #for x in pe_list: # resize_cta_images(x) # #exit() #for x in orcale_normal_list: # resize_cta_images(x, "orcale_normal") from concurrent import futures num_threads=10 with futures.ProcessPoolExecutor(max_workers=num_threads) as executor: fs = [executor.submit(resize_cta_images, x, ) for x in pe_list[::-1]] for i, f in enumerate(futures.as_completed(fs)): print ("{}/{} done...".format(i, len(fs)))
def expand(self, epsF, wfs): if wfs.nspins == 1: epsF = [epsF] elif not self.fixmom: epsF = [epsF, epsF] if self.nos == None: self.nos = wfs.nbands # Check dimension of lists if len(self.wf_u) == len(wfs.kpt_u): wf_u = self.wf_u p_uai = self.p_uai else: raise RuntimeError('List of wavefunctions has wrong size') c_un = [] p_u = [] for u, kpt in enumerate(wfs.kpt_u): # Inner product of pseudowavefunctions wf = np.reshape(wf_u[u], -1) Wf_n = kpt.psit_nG Wf_n = np.reshape(Wf_n, (len(kpt.f_n), -1)) Porb_n = np.dot(Wf_n.conj(), wf) * wfs.gd.dv # Correction to obtain inner product of AE wavefunctions for a, p_i in p_uai[u].items(): for n in range(wfs.nbands): for i in range(len(p_i)): for j in range(len(p_i)): Porb_n[n] += (kpt.P_ani[a][n][i].conj() * wfs.setups[a].dO_ii[i][j] * p_i[j]) wfs.gd.comm.sum(Porb_n) #print 'Kpt:', kpt.k, ' Spin:', kpt.s, \ # ' Sum_n|<orb|nks>|^2:', sum(abs(Porb_n)**2) p_u.append(np.array([sum(abs(Porb_n)**2)], dtype=float)) # Starting from KS orbitals with largest overlap, # fill in the expansion coeffients c_n = np.zeros(wfs.nbands, dtype=complex) nos = 0 bandpriority = np.argsort(abs(Porb_n)**2)[::-1] for n in bandpriority: if (kpt.eps_n[n] > epsF[kpt.s] + self.Estart and kpt.eps_n[n] < epsF[kpt.s] + self.Eend): c_n[n] = Porb_n[n] nos += 1 if nos == self.nos: break # Normalize expansion coefficients c_n /= np.sqrt(sum(abs(c_n)**2)) c_un.append(c_n) for s in range(wfs.nspins): for k in range(wfs.nibzkpts): p = wfs.collect_auxiliary(p_u, k, s) if wfs.world.rank == 0: self.txt.write('Kpt: %d, Spin: %d, ' \ 'Sum_n|<orb|nks>|^2: %f\n' % (k, s, p)) return c_un
def _train(path_to_train_lmdb_dir, path_to_val_lmdb_dir, path_to_log_dir, path_to_restore_checkpoint_file, training_options, max_steps): batch_size = training_options['batch_size'] initial_learning_rate = training_options['learning_rate'] initial_patience = training_options['patience'] num_steps_to_show_loss = 100 num_steps_to_check = training_options["validation_interval"] step = 0 patience = initial_patience best_accuracy = 0.0 duration = 0.0 model = Model() model.cuda() transform = transforms.Compose([ transforms.RandomCrop([54, 54]), transforms.ToTensor(), transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]) ]) train_loader = torch.utils.data.DataLoader(Dataset(path_to_train_lmdb_dir, transform), batch_size=batch_size, shuffle=True, num_workers=0, pin_memory=True) evaluator = Evaluator(path_to_val_lmdb_dir) optimizer = optim.SGD(model.parameters(), lr=initial_learning_rate, momentum=0.9, weight_decay=0.0005) scheduler = StepLR(optimizer, step_size=training_options['decay_steps'], gamma=training_options['decay_rate']) if path_to_restore_checkpoint_file is not None: assert os.path.isfile(path_to_restore_checkpoint_file), '%s not found' % path_to_restore_checkpoint_file step = model.restore(path_to_restore_checkpoint_file) scheduler.last_epoch = step print('Model restored from file: %s' % path_to_restore_checkpoint_file) path_to_losses_npy_file = os.path.join(path_to_log_dir, 'losses.npy') if os.path.isfile(path_to_losses_npy_file): losses = np.load(path_to_losses_npy_file) else: losses = np.empty([0], dtype=np.float32) path_to_test_losses_npy_file = os.path.join(path_to_log_dir, 'test_losses.npy') if os.path.isfile(path_to_test_losses_npy_file): test_losses = np.load(path_to_test_losses_npy_file) else: test_losses = np.empty([0], dtype=np.float32) train_loss_array = [] val_loss_array = [] model_checkpoints = [] model_saved = False # Used to save model (checkpoint) every 2 epochs model_save_counter = 0 while True: for batch_idx, (images, length_labels, digits_labels, _) in enumerate(train_loader): start_time = time.time() images, length_labels, digits_labels = images.cuda(), length_labels.cuda(), [digit_labels.cuda() for digit_labels in digits_labels] length_logits, digit1_logits, digit2_logits, digit3_logits, digit4_logits, digit5_logits = model.train()(images) loss = _loss(length_logits, digit1_logits, digit2_logits, digit3_logits, digit4_logits, digit5_logits, length_labels, digits_labels) optimizer.zero_grad() loss.backward() optimizer.step() scheduler.step() step += 1 duration += time.time() - start_time if step % num_steps_to_show_loss == 0: examples_per_sec = batch_size * num_steps_to_show_loss / duration duration = 0.0 print('=> %s: step %d, loss = %f, learning_rate = %f (%.1f examples/sec)' % ( datetime.now(), step, loss.item(), scheduler.get_lr()[0], examples_per_sec)) if step % num_steps_to_check != 0: continue model_save_counter += 1 losses = np.append(losses, loss.item()) np.save(path_to_losses_npy_file, losses) train_loss_array.append((step, loss.item())) print('=> Evaluating on validation dataset...') accuracy, test_loss_args = evaluator.evaluate(model) test_loss = _loss(*test_loss_args) val_loss_array.append((step, test_loss.item())) print('==> accuracy = %f, best accuracy %f' % (accuracy, best_accuracy)) # print(f'==> loss = {test_loss}') # Save model every 2 epochs if model_save_counter >= 2 or step in [1000, 2000, 3000, 4000, 5000]: path_to_checkpoint_file = model.store(path_to_log_dir, step=step) print('=> Model saved to file: %s' % path_to_checkpoint_file) model_save_counter = 0 model_saved = True model_checkpoints.append((step, f"model-{step}.pth")) if accuracy > best_accuracy: patience = initial_patience best_accuracy = accuracy else: patience -= 1 print("Train losses: ", train_loss_array) print("Saved Model Checkpoints: ", model_checkpoints) print('=> patience = %d' % patience) if patience == 0 or step >= max_steps: if not model_saved: path_to_checkpoint_file = model.store(path_to_log_dir, step=step) print('=> Model MANUALLY saved to file: %s' % path_to_checkpoint_file) model_checkpoints.append((step, f"model-{step}.pth")) training_output = { "model_checkpoints": model_checkpoints, "train_loss": train_loss_array, "val_loss": val_loss_array, } print("TRAINING OUTPUT -----------------------------") print(training_output) return training_output
# Сами минимальный и максимальный элементы в сумму не включать. # Примечание: попытайтесь решить задания без использования функций max, min, sum, sorted и их аналогов, в том числе # написанных самостоятельно. from random import randint SIZE = 20 MIN_ITEM = 0 MAX_ITEM = 100 summ = 0 arr = [randint(MIN_ITEM, MAX_ITEM) for _ in range(SIZE)] print('Исходный массив') print(arr) min_el = max_el = arr[0] min_i = max_i = 0 for i, el in enumerate(arr): if min_el > el: min_el = el min_i = i if max_el < el: max_el = el max_i = i if min_i > max_i: min_i, max_i = max_i, min_i print('Полученный массив') print(arr[min_i+1:max_i]) for i in range(min_i+1, max_i): summ += arr[i] print(f'Сумма элементов {summ}')
import csv import ast import pickle import utilFunc import pprint with open('C:/Users/이재원/Documents/code/Sodict.txt', 'r', encoding="UTF-8") as inf: aa = inf.read() dicList = [] tokenNum = 0 newData = "" for i in aa: if i == '{': newData = "" tokenNum = tokenNum + 1 newData = newData + i elif i == '}': tokenNum = tokenNum - 1 newData = newData + i dicList.append(newData) else: newData = newData + i for num, value in enumerate(dicList): dicData = ast.literal_eval(value) dicList[num] = dicData print(dicData['DEPTH 0'])
# init graph and train som.init_graph() som.train(colors) # get output grid image_grid = som.get_centroids() # map colours to their closest neurons mapped = som.map_vects(colors) # plot plt.imshow(image_grid) plt.title('Colour SOM (' + str(som._m) + ' x ' + str(som._n) + '), ' + str(som._n_iterations) + ' iterations') for i, m in enumerate(mapped): plt.text(m[1], m[0], color_names[i], ha='center', va='center', bbox=dict(facecolor='white', alpha=0.5, lw=0)) plt.show() # By running the session created in the method [init_graph()](#init_graph), we can evaluate the tensors and display the SOM data, for example the **weight vectors**: # In[10]: #np.set_printoptions(threshold=np.inf) # <--- with this option, the entire array is printed (we do not wish it here) print(som._sess.run(som._weightage_vects))