def testChunks(self): arr = [1, 2, 3, 4, 5, 6, 7, 8, 9] chunked = tools.chunks(arr, 3) i = 0 total = 0 for chunk in chunked: total += sum(chunk) i += 1 self.assertEqual(total, 45) self.assertEqual(i, 3)
def testChunks(self): arr = [1, 2, 3, 4, 5, 6, 7, 8, 9] chunked = tools.chunks(arr, 3) i = 0 total = 0 for chunk in chunked: total += sum(chunk) i += 1 self.assertEqual(total, 45) self.assertEqual(i, 3)
def get_school_holidays(self): range_name = "school_data!X2:X" range_values = self.service.spreadsheets().values().get( spreadsheetId=self.spreadsheet_id, range=range_name).execute() values = [ datetime.datetime.strptime(x[0], "%d/%m/%Y") for x in range_values.get('values', []) ] grouped = tools.chunks(values, 2) r = list(grouped) return r
def iter_parts(self, parts=range(4)): '''Iterate over parts in self.trials part 0: trials 1-30 part 1: trials 31-60 part 2: trials 61-90 part 3: trials 91-120''' if type(parts) is not list: parts = [parts] assert all([0 <= t < 4 for t in parts]) trial_chunks = list(tls.chunks(self.trials, 30)) for i in parts: yield i, trial_chunks[i]
def __construct_data(self): indexed_content = [] for block in self.block_image.blocks: itt = ImageToText(block) char_index = self.char_database.get(str(itt), 32) indexed_content.append(str(char_index)) line_count = DATA_FIRST_LINE data_lines = [] for data in chunks(indexed_content, 16): line = DATA_TEMPLATE.format(line=line_count, content=",".join(data)) data_lines.append(line) line_count += 10 return data_lines
def run(self, circuits, meas_qubits=None, measure=None, count_chunks=False): """ :param circuits: list of QuantumCircuit or QuantumCircuit :param meas_qubits: list of qubits that will be measured. :param measure: optional. By default after channel transformation we do tomography. Not implemented now :return: depend on measure parameter. By default, it's list of density matrix """ if measure is not None: raise NotImplementedError if isinstance(circuits, QuantumCircuit): circuits = [circuits] meas_qubits, s_matrix = sort_list_and_transformation_matrix( meas_qubits) number_measure_experiments = 3**len(meas_qubits) jobs = [] for qc in circuits: qr = qc.qregs[0] tomo_circuits = state_tomography_circuits( qc, [qr[qubit_number] for qubit_number in meas_qubits]) jobs.extend(tomo_circuits) res = None for i, chunk_jobs in enumerate(chunks(jobs, self.max_jobs_per_one)): if count_chunks: print(f'chunk number: {i + 1}') execute_kwargs = { 'experiments': chunk_jobs, 'backend': self.backend, 'shots': self.shots, 'max_credits': 15 } new_res = execute(**execute_kwargs).result() if res is None: res = new_res else: res += new_res matrices = [] for i in range(int(len(res.results) / number_measure_experiments)): res_matrix = copy(res) res_matrix.results = res.results[i * number_measure_experiments:(i + 1) * number_measure_experiments] rho = StateTomographyFitter( res_matrix, jobs[i * number_measure_experiments:(i + 1) * number_measure_experiments]).fit() rho = np.linalg.inv(s_matrix) @ rho @ s_matrix matrices.append(rho) return matrices
max_colm_norm=False, max_norm=15.0, norm_gsup=norm_gsup, norm_gh=norm_gh) eval_fn, eval_fn_tr = fns["eval_fn"], fns["eval_fn_tr"] # Things to track during training: epoch and minibatch train_stats = {"tr_error_ep": [], "vl_error_ep": [], "tr_cost_ep": [], "tr_error_mn": [], "vl_error_mn": [], "tr_cost_mn": [], "current_nb_mb": 0, "best_epoch": 0, "best_mn": 0} names = [] for l, i in zip(layers, range(len(layers))): if l["hint"] is not None: names.append(i) debug = {"grad_sup": [], "grad_hint": [], "penalty": [], "names": names} # Eval before start training l_vl = chunks(range(validx.shape[0]), valid_batch_size) l_tr = chunks(range(trainx.shape[0]), valid_batch_size) vl_err_start = np.mean( [eval_fn(np.array(l_vl[kk])) for kk in range(len(l_vl))]) tr_err_start = np.mean( [eval_fn_tr(np.array(l_tr[kk])) for kk in range(len(l_tr))]) print vl_err_start, tr_err_start # Exp stamp time_exp = DT.datetime.now().strftime('%m_%d_%Y_%H_%M_%s') tag_text = "_".join([str(l["hint"]) for l in layers]) h_exp = "_".join([str(e) for e in h_ind]) fold_exp = "exps/" + tag + "_" + str(nbr_sup) + "_" + h_exp + "_" +\ size_model + "_" + time_exp if not os.path.exists(fold_exp): os.makedirs(fold_exp)
def get(self): results = self.credentials.user_playlist(self.user.user_id, self.id, fields="tracks,next") for i, item in enumerate(results['tracks']['items']): dtc = {} dtc['Title'] = item['track']['name'] dtc['TrackId'] = item['track']['id'] dtc['Album'] = item['track']['album']['name'] dtc['AlbumId'] = item['track']['album']['id'] dtc['Comments'] = '' try: dtc['Artist'] = item['track']['album']['artists'][0]['name'] except IndexError: dtc['Artist'] = 'none' try: # only take first artist ID dtc['ArtistId'] = item['track']['album']['artists'][0]['id'] except IndexError: dtc['ArtistId'] = 'none' self.tracks_json[i] = dtc outside_track_list = chunks( self.tracks_json, 20) # this is to avoid hitting the API id count limit for j, tracks in enumerate(outside_track_list): track_ids = [] for track in tracks: track_ids.append(track['TrackId']) artist_ids = [] for track in tracks: artist_ids.append(track['ArtistId']) album_ids = [] for track in tracks: album_ids.append(track['AlbumId']) # get album details album_meta = self.credentials.albums(albums=album_ids) for i, track in enumerate(tracks): track['ReleaseDate'] = album_meta['albums'][i]['release_date'] track['RecordLabel'] = album_meta['albums'][i]['label'] # Get artist details artist_meta = self.credentials.artists(artists=artist_ids) for i, track in enumerate(tracks): try: # tags will always refer to first artist listed tags = '; '.join(artist_meta['artists'][i]['genres'][:4]) track['ArtistTags'] = tags except IndexError: track['ArtistTags'] = 'none' # get track features data feature_meta = self.credentials.audio_features(tracks=track_ids) for i, track in enumerate(tracks): track['danceability'] = feature_meta[i]['danceability'] track['energy'] = feature_meta[i]['energy'] track['key'] = feature_meta[i]['key'] track['loudness'] = feature_meta[i]['loudness'] track['mode'] = feature_meta[i]['mode'] track['speechiness'] = feature_meta[i]['speechiness'] track['acousticness'] = feature_meta[i]['acousticness'] track['instrumentalness'] = feature_meta[i]['instrumentalness'] track['liveness'] = feature_meta[i]['liveness'] track['valence'] = feature_meta[i]['valence'] track['tempo'] = feature_meta[i]['tempo'] if j == 0: main_df = pd.DataFrame.from_dict(tracks) else: sub_df = pd.DataFrame.from_dict(tracks) main_df = main_df.append(sub_df) column_order = ('Title', 'Album', 'Artist', 'ArtistTags', 'ReleaseDate', 'RecordLabel', 'Comments', 'TrackId', 'AlbumId', 'ArtistId', 'acousticness', 'danceability', 'energy', 'instrumentalness', 'key', 'liveness', 'loudness', 'mode', 'speechiness', 'tempo', 'valence') main_df = main_df.ix[:, column_order] return main_df
def test(nbr_sampler, config): data = nbr_sampler.data multilabel, device = config.multilabel, config.device model, _ = load_model(config, data) model.eval() # _, prior = torch.unique(data.y[data.train_mask == 1], return_counts=True) # prior = prior.to(torch.float32)/prior.sum() # prior = prior.to(device) city2ix = {c: i for i, c in enumerate(data.cities)} if multilabel: test_cities = [ 'co_Russia.r1_CentralRussia', 'co_Russia.r1_CentralRussia.r2_MosObl', 'co_Russia.r1_CentralRussia.r2_MosObl.ci_Moscow', 'co_Russia.r1_NorthWest', 'co_Russia.r1_NorthWest.r2_SpbObl', 'co_Russia.r1_NorthWest.r2_SpbObl.ci_SaintPetersburg', 'co_Russia.r1_Urals', 'co_Russia.r1_Urals.r2_Sverdlovsk', 'co_Russia.r1_Urals.r2_Sverdlovsk.ci_Yekaterinburg', 'co_Russia.r1_FarEast', 'co_Russia.r1_FarEast.r2_Yakutia', 'co_Russia.r1_FarEast.r2_Yakutia.ci_Yakutsk', 'co_Russia.r1_Siberia', 'co_Russia.r1_Siberia.r2_Chita', 'co_Russia.r1_Siberia.r2_Chita.ci_Chita', ] else: test_cities = [ 'p_Europe.co_Russia.r1_CentralRussia.r2_MosObl.ci_Moscow', 'p_Europe.co_Russia.r1_NorthWest.r2_SpbObl.ci_SaintPetersburg', 'p_Europe.co_Russia.r1_Urals.r2_Sverdlovsk.ci_Yekaterinburg', 'p_Europe.co_Russia.r1_Siberia.r2_Novosibirsk.ci_Novosibirsk', 'p_Europe.co_Belorussia.r1_MinskObl.ci_Minsk', 'p_Europe.co_Russia.r1_CentralRussia.r2_Tula.ci_Tula', 'p_Europe.co_Russia.r1_Siberia.r2_Chita.ci_Chita', 'p_Europe.co_Russia.r1_CentralRussia.r2_Bryansk.ci_Bryansk', 'p_Europe.co_Russia.r1_FarEast.r2_Yakutia.ci_Yakutsk', # 'p_Europe.co_Russia.r1_CentralRussia.r2_Orel', ] ixs = [city2ix[c] for c in test_cities] if multilabel: plt_ixs = list(chunks(ixs, 3)) probs, preds, y = [], [], [] with torch.no_grad(): for data_flow in nbr_sampler(data.test_mask): if multilabel: y_cur = data.y[data_flow.n_id] else: y_cur = data.y[data_flow.n_id].numpy() x = slice_data_flow(config, data, data_flow) logits = model(x, data_flow.to(device)) probs.append(torch.exp(logits).cpu().detach().numpy()) pred = logits.max(1)[1] preds.append(pred.cpu().detach().numpy()) y.append(y_cur) probs = np.concatenate(probs) preds = np.concatenate(preds) y = np.concatenate(y) if not multilabel: ntest = data.test_mask.sum().item() accuracy = (preds == y).sum() / ntest print(accuracy) Y = label_binarize(y, classes=ixs) # ids = load_vector("../data/vk/nodes.bin", 'I') # df = pd.DataFrame(probs) # df.columns = data.cities # df.index = [ids[i] for i in data_flow.n_id] # df.to_csv('../data/vk/preds_example.csv') else: Y = y df = pd.DataFrame() df['geo'] = data.cities # choose thresholds suche that precision >= precision_min for th in [config.precision_min1, config.precision_min2]: recalls, thresholds = [], [] for i in range(Y.shape[1]): precision, recall, _thresholds = precision_recall_curve( Y[:, i], probs[:, i]) wix = np.where(precision >= th)[0][0] if wix == len(_thresholds): thresholds.append(1) recalls.append(0) else: thresholds.append(_thresholds[wix]) recalls.append(recall[wix]) df['recall_%.2f' % th] = recalls df['threshold_%.2f' % th] = thresholds df.to_csv(config.join('thresholds.csv'), index=False) def plot_precision_recall(Y, probs, ixs, fname): plt.clf() for i in ixs: precision, recall, _ = precision_recall_curve(Y[:, i], probs[:, i]) plt.plot(recall, precision, lw=1, label=data.cities[i].split('.')[-1]) plt.legend(fontsize='small', loc='lower left') plt.xlabel("recall") plt.ylabel("precision") plt.title("precision vs. recall curve") plt.savefig(fname, dpi=900) fname = config.join('precision_recall%s_all.pdf' % config.postfix) plot_precision_recall(Y, probs, ixs, fname) if multilabel: for i, sub_ixs in enumerate(plt_ixs): fname = config.join('precision_recall%s_%s.pdf' % (config.postfix, i)) plot_precision_recall(Y, probs, sub_ixs, fname)
def pullFeatureForSong(self, feature_name, song_id, pack_size): """ Returns an array with elements of type <feature_name>. The song is split into n number of sample packs. Each sample pack is of size pack_size. The feature is evaluated on each sample pack and filled into the returned array (in same order). If no data is already present in the database for the given pack_size, the feature will be evaluated for the whole song and added to the database. @param feature_name: the feature to get @param song_id: the song to get the feature for @param pack_size: the size of the packets to split the song into. (-1 to apply the feature to the whole song)dw """ logging.debug("song_id " + str(song_id) + ": " + feature_name + " with pack size " + str(pack_size) + " searching...") # PULL IF FEATURE THAT ALREADY EXISTS self.cur.execute("SELECT data FROM FeatureData WHERE song_id=%s AND feature_name=%s AND pack_size=%s", (song_id, feature_name, pack_size)) class_ = getattr(features, feature_name) # prepare the class for instantiation packList = [] row = self.cur.fetchone() if row: # RETURN THE LIST OF FEATURES # in the case that class_.requireFullSong == True, their should only be one row. But is returned in the same # way. while row: feature = class_.unserialize(row[0]) packList.append(feature) row = self.cur.fetchone() else: # NO ELEMENTS FOUND, COMPUTE raw_data = self.fetchSongData(song_id) # fetch raw data from file on disk raw_chunks = [] if pack_size == -1: raise "pack_size == -1 not yet supported." else: raw_chunks = tools.chunks(raw_data, pack_size) # Split the song into chunks of size pack_size. this will be processed by the feature logging.debug("Feature Data not found. Generating feature data for " + str(len(raw_data) / pack_size) + " packs...") if class_.requireFullSong: # single feature creation for full song feature = class_(raw_chunks) packList.append(feature) serialized = feature.serialize() logging.debug(feature_name + ".value = " + str(feature.value)) self.cur.execute("INSERT INTO FeatureData (song_id, pack_index, feature_name, pack_size, data) VALUES (%s, %s, %s, %s, %s)", (song_id, 0, feature_name, pack_size, serialized)) else: # feature creation for single sample pack # iterate and create a feature for each one to save to the DB pack_index = 0 for dataPack in raw_chunks: feature = class_(dataPack) packList.append(feature) serialized = feature.serialize() self.cur.execute("INSERT INTO FeatureData (song_id, pack_index, feature_name, pack_size, data) VALUES (%s, %s, %s, %s, %s)", (song_id, pack_index, feature_name, pack_size, serialized)) pack_index += pack_size self.commit() # commit the insertions return packList