def compute_elo_by_goals(data_df, players, all_teams, elo, initial_score=100): """ This function is used to compute the elo ratings of teams based on goals scored depending on wins and losses. :param data_df: :param players: :param elo: :return: """ n_games = data_df.shape[0] elo_table = np.zeros((n_games, len(players))) # set initial score for all teams elo_table[0, :] = initial_score bar = progressbar.ProgressBar(widgets=[ ' [', progressbar.Timer(), '] ', progressbar.Bar(), ' (', progressbar.ETA(), ') ', ]) for i in bar(range(n_games)): match = data_df.iloc[i] player_home = match.home_team player_away = match.away_team hid = np.where(all_teams == player_home)[0][0] aid = np.where(all_teams == player_away)[0][0] pair = [players[hid], players[aid]] res = match.result_final home_goals = match.home_goals away_goals = match.away_goals if res == 0: for goal_difference in range(int(abs(home_goals - away_goals))): a, b = elo.match_algo_strict(pair[0], pair[1]) elo_table[i, hid] = a.score elo_table[i, aid] = b.score elif res == 2: for goal_difference in range(int(abs(home_goals - away_goals))): a, b = elo.match(pair[1], pair[0]) elo_table[i, aid] = a.score elo_table[i, hid] = b.score else: pass # make continuous bar = progressbar.ProgressBar(widgets=[ ' [', progressbar.Timer(), '] ', progressbar.Bar(), ' (', progressbar.ETA(), ') ', ]) print('[C] Making continuous') for p in bar(range(len(players))): for g in range(n_games): if elo_table[g, p] == 0: elo_table[g, p] = elo_table[g - 1, p] else: pass return elo_table
def train_model(model, encoder_frnn, encoder_rrnn, decoder_rnn, train_lemmas, train_feat_dicts, train_words, dev_lemmas, dev_feat_dicts, dev_words, alphabet_index, inverse_alphabet_index, epochs, optimization, results_file_path, morph_index, train_aligned_pairs, dev_aligned_pairs, feat_index, feature_types, plot): print 'training...' np.random.seed(17) random.seed(17) if optimization == 'ADAM': trainer = pc.AdamTrainer(model, lam=REGULARIZATION, alpha=LEARNING_RATE, beta_1=0.9, beta_2=0.999, eps=1e-8) elif optimization == 'MOMENTUM': trainer = pc.MomentumSGDTrainer(model) elif optimization == 'SGD': trainer = pc.SimpleSGDTrainer(model) elif optimization == 'ADAGRAD': trainer = pc.AdagradTrainer(model) elif optimization == 'ADADELTA': trainer = pc.AdadeltaTrainer(model) else: trainer = pc.SimpleSGDTrainer(model) total_loss = 0 best_avg_dev_loss = 999 best_dev_accuracy = -1 best_train_accuracy = -1 patience = 0 train_len = len(train_words) epochs_x = [] train_loss_y = [] dev_loss_y = [] train_accuracy_y = [] dev_accuracy_y = [] e = -1 # progress bar init widgets = [progressbar.Bar('>'), ' ', progressbar.ETA()] train_progress_bar = progressbar.ProgressBar(widgets=widgets, maxval=epochs).start() avg_loss = -1 for e in xrange(epochs): # randomize the training set indices = range(train_len) random.shuffle(indices) train_set = zip(train_lemmas, train_feat_dicts, train_words, train_aligned_pairs) train_set = [train_set[i] for i in indices] # compute loss for each example and update for i, example in enumerate(train_set): lemma, feats, word, alignment = example loss = one_word_loss(model, encoder_frnn, encoder_rrnn, decoder_rnn, lemma, feats, word, alphabet_index, alignment, feat_index, feature_types) loss_value = loss.value() total_loss += loss_value loss.backward() trainer.update() if i > 0: avg_loss = total_loss / float(i + e * train_len) else: avg_loss = total_loss if EARLY_STOPPING: # get train accuracy train_predictions = predict_templates( model, decoder_rnn, encoder_frnn, encoder_rrnn, alphabet_index, inverse_alphabet_index, train_lemmas, train_feat_dicts, feat_index, feature_types) print 'evaluating on train...' train_accuracy = evaluate_model(train_predictions, train_lemmas, train_feat_dicts, train_words, feature_types, print_results=False)[1] if train_accuracy > best_train_accuracy: best_train_accuracy = train_accuracy dev_accuracy = 0 avg_dev_loss = 0 if len(dev_lemmas) > 0: # get dev accuracy dev_predictions = predict_templates(model, decoder_rnn, encoder_frnn, encoder_rrnn, alphabet_index, inverse_alphabet_index, dev_lemmas, dev_feat_dicts, feat_index, feature_types) print 'evaluating on dev...' # get dev accuracy dev_accuracy = evaluate_model(dev_predictions, dev_lemmas, dev_feat_dicts, dev_words, feature_types, print_results=False)[1] if dev_accuracy > best_dev_accuracy: best_dev_accuracy = dev_accuracy # save best model to disk save_pycnn_model(model, results_file_path, morph_index) print 'saved new best model' patience = 0 else: patience += 1 # found "perfect" model if dev_accuracy == 1: train_progress_bar.finish() if plot: plt.cla() return model, e # get dev loss total_dev_loss = 0 for i in xrange(len(dev_lemmas)): total_dev_loss += one_word_loss( model, encoder_frnn, encoder_rrnn, decoder_rnn, dev_lemmas[i], dev_feat_dicts[i], dev_words[i], alphabet_index, dev_aligned_pairs[i], feat_index, feature_types).value() avg_dev_loss = total_dev_loss / float(len(dev_lemmas)) if avg_dev_loss < best_avg_dev_loss: best_avg_dev_loss = avg_dev_loss print 'epoch: {0} train loss: {1:.4f} dev loss: {2:.4f} dev accuracy: {3:.4f} train accuracy = {4:.4f} \ best dev accuracy {5:.4f} best train accuracy: {6:.4f} patience = {7}'.format( e, avg_loss, avg_dev_loss, dev_accuracy, train_accuracy, best_dev_accuracy, best_train_accuracy, patience) if patience == MAX_PATIENCE: print 'out of patience after {0} epochs'.format(str(e)) # TODO: would like to return best model but pycnn has a bug with save and load. Maybe copy via code? # return best_model[0] train_progress_bar.finish() if plot: plt.cla() return model, e else: # if no dev set is present, optimize on train set print 'no dev set for early stopping, running all epochs until perfectly fitting or patience was \ reached on the train set' if train_accuracy > best_train_accuracy: best_train_accuracy = train_accuracy # save best model to disk save_pycnn_model(model, results_file_path, morph_index) print 'saved new best model' patience = 0 else: patience += 1 print 'epoch: {0} train loss: {1:.4f} train accuracy = {2:.4f} best train accuracy: {3:.4f} \ patience = {4}'.format(e, avg_loss, train_accuracy, best_train_accuracy, patience) # found "perfect" model on train set or patience has reached if train_accuracy == 1 or patience == MAX_PATIENCE: train_progress_bar.finish() if plot: plt.cla() return model, e # update lists for plotting train_accuracy_y.append(train_accuracy) epochs_x.append(e) train_loss_y.append(avg_loss) dev_loss_y.append(avg_dev_loss) dev_accuracy_y.append(dev_accuracy) # finished epoch train_progress_bar.update(e) if plot: with plt.style.context('fivethirtyeight'): p1, = plt.plot(epochs_x, dev_loss_y, label='dev loss') p2, = plt.plot(epochs_x, train_loss_y, label='train loss') p3, = plt.plot(epochs_x, dev_accuracy_y, label='dev acc.') p4, = plt.plot(epochs_x, train_accuracy_y, label='train acc.') plt.legend(loc='upper left', handles=[p1, p2, p3, p4]) plt.savefig(results_file_path + '_' + morph_index + '.png') train_progress_bar.finish() if plot: plt.cla() print 'finished training. average loss: ' + str(avg_loss) return model, e
def calculate_score(means_path, label_encoder_path, best_weight_path, test_hdf5_path, cross_val=None, preds_cross=None, labels_cross=None, is_mapped=False): # load RGB means for training set means = json.loads(open(means_path).read()) # load LabelEncoder le = pickle.loads(open(label_encoder_path, 'rb').read()) # initialize image preprocessors sp, mp, cp, iap = SimplePreprocessor( config.IMAGE_SIZE, config.IMAGE_SIZE), MeanPreprocessor( means['R'], means['G'], means['B']), CropPreprocessor( config.IMAGE_SIZE, config.IMAGE_SIZE), ImageToArrayPreprocessor() custom_objects = None agh = AgeGenderHelper(config, deploy) if config.DATASET_TYPE == 'age': one_off_mappings = agh.build_oneoff_mappings(le) one_off = OneOffAccuracy(one_off_mappings) custom_objects = {'one_off_accuracy': one_off.one_off_accuracy} # load model print(f'[INFO] loading {best_weight_path}...') model = load_model(best_weight_path, custom_objects=custom_objects) # initialize testing dataset generator, then predict if cross_val is None: print( f'[INFO] predicting in testing data (no crops){config.SALIENCY_INFO}...' ) else: print( f'[INFO] predicting in testing data (no crops) for cross validation {cross_val}{config.SALIENCY_INFO}...' ) test_gen = HDF5DatasetGenerator(test_hdf5_path, batch_size=config.BATCH_SIZE, preprocessors=[sp, mp, iap], classes=config.NUM_CLASSES) preds = model.predict_generator(test_gen.generator(), steps=test_gen.num_images // config.BATCH_SIZE) # compute rank-1 and one-off accuracies labels = to_categorical( test_gen.db['labels'][0:config.BATCH_SIZE * (test_gen.num_images // config.BATCH_SIZE)], num_classes=config.NUM_CLASSES) preds_mapped = preds.argmax(axis=1) if is_mapped == True: preds_mapped = agh.build_mapping_to_iog_labels()[preds_mapped] if cross_val is None: print( '[INFO] serializing all images classified incorrectly for testing dataset...' ) prefix_path = os.path.sep.join( [config.WRONG_BASE, config.DATASET_TYPE]) agh.plot_confusion_matrix_from_data(config, labels.argmax(axis=1), preds_mapped, le=le, save_path=os.path.sep.join([ config.OUTPUT_BASE, f'cm_{config.DATASET_TYPE}.png' ])) else: print( f'[INFO] serializing all images classified incorrectly for cross validation {cross_val} of testing dataset...' ) prefix_path = os.path.sep.join( [config.WRONG_BASE, f'Cross{cross_val}', config.DATASET_TYPE]) preds_cross.extend(preds_mapped.tolist()) labels_cross.extend(labels.argmax(axis=1).tolist()) if os.path.exists(prefix_path): shutil.rmtree(prefix_path) os.makedirs(prefix_path) for i, (pred, label) in enumerate(zip(preds_mapped, labels.argmax(axis=1))): if pred != label: image = test_gen.db['images'][i] if config.DATASET_TYPE == 'age': real_label, real_pred = le.classes_[label], le.classes_[pred] real_label = real_label.replace('_', '-') real_label = real_label.replace('-inf', '+') real_pred = real_pred.replace('_', '-') real_pred = real_pred.replace('-inf', '+') elif config.DATASET_TYPE == 'gender': real_label = 'Male' if label == 0 else 'Female' real_pred = 'Male' if pred == 0 else 'Female' cv2.putText(image, f'Actual: {real_label}, Predict: {real_pred}', (15, 15), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 0), 2) cv2.imwrite(os.path.sep.join([prefix_path, f'{i:05d}.jpg']), image) score = accuracy_score(labels.argmax(axis=1), preds_mapped) print(f'[INFO] rank-1: {score:.4f}') score_one_off = None if config.DATASET_TYPE == 'age': score_one_off = one_off.one_off_compute( labels, to_categorical(preds_mapped, num_classes=config.NUM_CLASSES)) print(f'[INFO] one-off: {score_one_off:.4f}') test_gen.close() # re-initialize testing generator, now excluding SimplePreprocessor test_gen = HDF5DatasetGenerator(test_hdf5_path, config.BATCH_SIZE, preprocessors=[mp], classes=config.NUM_CLASSES) preds = [] labels = to_categorical(test_gen.db['labels'], num_classes=config.NUM_CLASSES) print('[INFO] predicting in testing data (with crops)...') # initialize progress bar widgets = [ 'Evaluating: ', progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA() ] pbar = progressbar.ProgressBar(maxval=math.ceil(test_gen.num_images / config.BATCH_SIZE), widgets=widgets).start() for i, (images, _) in enumerate(test_gen.generator(passes=1)): for image in images: crops = cp.preprocess(image) crops = np.array([iap.preprocess(c) for c in crops]) pred = model.predict(crops) preds.append(pred.mean(axis=0)) pbar.update(i) pbar.finish() test_gen.close() # compute rank-1 accuracy preds_mapped = np.argmax(preds, axis=1) if is_mapped == True: preds_mapped = agh.build_mapping_to_iog_labels()[preds_mapped] score_crops = accuracy_score(labels.argmax(axis=1), preds_mapped) print(f'[INFO] rank-1: {score_crops:.4f}') score_one_off_crops = None if config.DATASET_TYPE == 'age': score_one_off_crops = one_off.one_off_compute( labels, to_categorical(preds_mapped, num_classes=config.NUM_CLASSES)) print(f'[INFO] one-off: {score_one_off_crops:.4f}') return score, score_one_off, score_crops, score_one_off_crops
datasets = [ ('train', train_paths, train_labels, config.TRAIN_HDF5), ('val', val_paths, val_labels, config.VAL_HDF5), ('test', test_paths, test_labels, config.TEST_HDF5) ] R = [] G = [] B = [] for dataset_type, paths, labels, output_path in datasets: print(f'[INFO] Building {output_path}...') writer = HDF5DatasetWriter((len(paths), 64, 64, 3), output_path) widgets = ['Building dataset: ', progressbar.Percentage(), ' ', progressbar.Bar(), progressbar.ETA()] progress_bar = progressbar.ProgressBar(maxval=len(paths), widgets=widgets).start() for i, (path, label) in enumerate(zip(paths, labels)): image = cv2.imread(path) if dataset_type == 'train': (b, g, r) = cv2.mean(image)[:3] R.append(r) G.append(g) B.append(b) writer.add([image], [label]) progress_bar.update(i) progress_bar.finish()
def generate_fom_data(self, nb_seeds=10, new_data_path=None, length=None, new_save_path=None, save_images=False, no_duration=False, verbose=1): """ Generate Midi file from the seed and the trained model :param nb_seeds: number of seeds for the generation :param new_data_path: The path of the seed :param length: Length of th generation :param new_save_path: :param save_images: To save the pianoroll of the generation (.jpg images) :param no_duration: if True : all notes will be the shortest length possible :param verbose: Level of verbose :return: """ # ---------- Verify the inputs ---------- # ----- Create the seed ----- if (new_data_path is not None) and (new_data_path != self.data_transformed_path.as_posix()): self.load_data(new_data_path) if self.data_transformed_path is None: raise Exception('Some data need to be loaded before generating') self.sequence = Sequences.AllInstSequence( path=str(self.data_transformed_path), nb_steps=self.nb_steps, batch_size=1, work_on=self.work_on) nb_instruments = self.sequence.nb_instruments seeds_indexes = random.sample(range(len(self.sequence)), nb_seeds) # -- Length -- length = length if length is not None else 20 # -- For save Midi path -- if type(new_save_path) is str or ( type(new_save_path) is bool and new_save_path) or ( new_save_path is None and self.save_midis_path is None): self.get_new_save_midis_path(path=new_save_path) # --- Done Verifying the inputs --- mask = self.get_mask(nb_instruments) self.save_midis_path.mkdir(parents=True, exist_ok=True) cprint('Start generating from data ...', 'blue') for s in range(nb_seeds): cprint('Generation {0}/{1}'.format(s + 1, nb_seeds), 'blue') generated = np.array( self.sequence[seeds_indexes[s]][0]) # (nb_instruments, 1, nb_steps, step_size, inputs_size, 2) bar = progressbar.ProgressBar(maxval=length, widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage(), ' ', progressbar.ETA()]) bar.start() # To see it working for l in range(length): samples = generated[:, :, l:] # (nb_instruments, 1, nb_steps, length, 88, 2) # 1 = batch # expanded_samples = np.expand_dims(samples, axis=0) preds = self.keras_nn.generate( input=list(samples) + mask) # (nb_instruments, batch=1 , nb_steps=1, length, 88, 2) preds = np.asarray(preds).astype('float64') # (nb_instruments, 1, 1, step_size, input_size, 2) if len(preds.shape) == 4: # Only one instrument : output of nn not a list preds = preds[np.newaxis] next_array = midi.create.normalize_activation(preds) # Normalize the activation part generated = np.concatenate((generated, next_array), axis=2) # (nb_instruments, nb_steps, length, 88, 2) bar.update(l + 1) bar.finish() self.ensure_save_midis_path() generated_midi_final = self.reshape_generated_array(generated) self.compute_generated_array( generated_array=generated_midi_final, file_name=self.save_midis_path / f'generated_{s}', no_duration=no_duration, verbose=verbose, save_images=save_images ) if self.batch is not None: self.sequence.change_batch_size(self.batch) summary.summarize_generation(str(self.save_midis_path), **{ 'full_name': self.full_name, 'epochs': self.total_epochs, 'input_param': self.input_param, 'instruments': self.instruments, 'notes_range': self.notes_range }) cprint('Done generating', 'green')
def _need_update(self): cur_val = self.get_current_value() if cur_val < 100: return cur_val - self.last_update > 10 elif cur_val < 500: return cur_val - self.last_update > 50 else: return cur_val - self.last_update > 100 default_widgets = [ progressbar.Percentage(), ' ', progressbar.Bar(marker='=', left='[', right=']'), ' ', progressbar.CounterWidget(), ' ', progressbar.GenericSpeed(format='%.2ft/s'), ' ', progressbar.ETA(prefix='eta ') ] class ProgressBarObserver(progressbar.ProgressBar, Observer): """Display progress through a progressbar. """ # the progress bar is only updated in increments of this for performance UPDATE_INTERVAL = 25 def __init__(self, widgets=default_widgets, term_width=None, fd=sys.stderr): super(ProgressBarObserver, self).__init__(widgets=widgets,
# from disk print("[INFO] loading trained models...") faceNet = cv2.dnn.readNet(config.FACE_PROTOTXT, config.FACE_WEIGHTS) ageNet = cv2.dnn.readNet(config.AGE_PROTOTXT, config.AGE_WEIGHTS) camoNet = load_model(config.CAMO_MODEL) # grab the paths to all images in our dataset imagePaths = sorted(list(paths.list_images(args["dataset"]))) print("[INFO] processing {} images".format(len(imagePaths))) # initialize the progress bar widgets = [ "Processing Images: ", progressbar.Percentage(), " ", progressbar.Bar(), " ", progressbar.ETA() ] pbar = progressbar.ProgressBar(maxval=len(imagePaths), widgets=widgets).start() # loop over the image paths for (i, imagePath) in enumerate(imagePaths): # load the image from disk image = cv2.imread(imagePath) # if the image is 'None', then it could not be properly read from # disk (so we should just skip it) if image is None: continue # detect all faces in the input image and then predict their # perceived age based on the face ROI
def main(): ''' Parse command line arguments and execute the code ''' parser = argparse.ArgumentParser() parser.add_argument('--dataset_path', required=True, type=str) parser.add_argument('--newext', default='.PNG', type=str) parser.add_argument('--oldext', default='.JPEG', type=str) args = parser.parse_args() start = time.time() image_list= Utils_Image.get_Image_List(args.dataset_path, args.oldext) progress = progressbar.ProgressBar(widgets=[progressbar.Bar('=', '[', ']'), ' ',progressbar.Percentage(), ' ',progressbar.ETA()]) print "Start Processing... May take a while..." for image_path in progress(image_list): Utils_Image.resizeImage(image_path) Utils_Image.change_extension(image_path,args.oldext,args.newext) end = time.time() print("Parsed: %d Image of the Dataset"%(len(image_list))) print("Elapsed Time:%d Seconds"%(end-start)) print("Running Completed with Success!!!")
def runSteppedTestCase( testcase, name, keep_cuda_code=False, printTimings=True, showProgress=True, solver=None, ): params = getTestCaseParams(testcase, keep_cuda_code=keep_cuda_code) epsilon = 1e-6 bar = None result = { "testcase": testcase, "name": name, "field_states": None, "runtime": 0.0, "error": None, } try: if solver is None: solver = coss.CUDAODESystemSolver( testcase.num_nodes, testcase.ode, init_field_parameters=testcase.field_parameter_values, params=params, ) field_states_fn = testcase.field_states_fn if field_states_fn is None: field_states_fn = lambda _: None if testcase.field_states is not None and testcase.field_states != [""]: field_states = np.zeros( testcase.num_nodes * len(testcase.field_states), dtype=getFloatPrecisionDtypeStr(testcase.double), ) solver.get_field_states(field_states) field_states_fn(field_states) else: field_states = None field_states_fn(solver.field_states) t = testcase.t0 dt = testcase.dt * testcase.ode_substeps tstop = testcase.tstop if progressbar is not None and showProgress: bar = progressbar.ProgressBar( maxval=tstop + epsilon, widgets=[ progressbar.Bar("=", "[", "]"), " ", progressbar.Percentage(), " ", progressbar.ETA(), ], ) while t < tstop + epsilon: if testcase.update_field_states: solver.set_field_states(field_states) solver.forward( t, dt, update_simulation_runtimes=True, update_host_states=testcase.update_host_states, ) if testcase.update_field_states: solver.get_field_states(field_states) if field_states is not None: field_states_fn(field_states) else: field_states_fn(solver.field_states) if progressbar is not None and showProgress: bar.update(t) t += dt if progressbar is not None and showProgress: bar.finish() if printTimings: list_timings() clear_timings() solver.get_field_states(field_states) result["field_states"] = field_states result["runtime"] = solver.simulation_runtime solver.reset() print("Completed test '{0}' in {1:.2f}s".format( name, result["runtime"])) except Exception: if bar is not None: bar.finish() f = StringIO() traceback.print_exc(file=f) f.read() result["error"] = f.buf f.close() print("FAILED test '{0}'.".format(name)) print() return result
def run_mnist(use_tensorboard=False, use_trace=False, arithmetic_type='wf'): ''' Run MNIST using the Wave custom matmul operator. ''' tf.reset_default_graph() waveflow.waveflow_arithmetic = arithmetic_type # Import data mnist = input_data.read_data_sets('./mnist_data') # Create the model x = tf.placeholder(tf.float32, [None, 784]) xx = tf.zeros([784, 10]) W = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) y_l1 = tf.matmul(x, W) y = tf.nn.bias_add(y_l1, b) # Define loss and optimizer y_ = tf.placeholder(tf.int64, [None]) # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.losses.sparse_softmax_cross_entropy on the raw # outputs of 'y', and then average across the batch. with tf.name_scope('loss'): cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_, logits=y) # cross_entropy = -tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), reduction_indices=[1]) avg_entropy = tf.reduce_mean(cross_entropy) train_step = tf.train.GradientDescentOptimizer(0.5).minimize( avg_entropy) if use_tensorboard: tf.summary.scalar("avg_loss", avg_entropy, family='Accuracy') with tf.name_scope('accuracy'): correct_prediction = tf.equal(tf.argmax(y, 1), y_) correct_prediction = tf.cast(correct_prediction, tf.float32) accuracy = tf.reduce_mean(correct_prediction) if use_tensorboard: tf.summary.scalar("accuracy", accuracy, family='Accuracy') tb_log = waveflow.TF_TBLogger(log_dir='./tb_mnist_log/', enable_tb=use_tensorboard, enable_trace=use_trace, unified_trace=use_trace, arith_type=arithmetic_type) total_batches = 100 widgets = ["Training: ", pb.Percentage(), ' ', pb.Bar(), ' ', pb.ETA()] pbar = pb.ProgressBar(widgets=widgets, maxval=total_batches) pbar.start() # for shape debugging # shape_op = tf.shape(y) print("Got op list: %s" % (waveflow.op_list(tf.get_default_graph()))) # dev = '/device:GPU:0' dev = '/cpu:0' with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: with sess.graph.device(dev): print("Running model ...") tb_log.init_session(sess) tf.global_variables_initializer().run() # Train for i in range(total_batches): pbar.update(i) batch_xs, batch_ys = mnist.train.next_batch(100) tb_log.run_session(iter=i, ops=train_step, feed={ x: batch_xs, y_: batch_ys }) pbar.finish() tb_log.close() # Compute training set accuracy print("Evaluating full training set ...") sys.stdout.flush() train_accuracy = sess.run(accuracy, feed_dict={ x: mnist.train.images, y_: mnist.train.labels }) print("Train set accuracy: %s" % (train_accuracy)) # Compute test set accuracy print("Evaluating full test set ...") sys.stdout.flush() test_accuracy = sess.run(accuracy, feed_dict={ x: mnist.test.images, y_: mnist.test.labels }) print("Test set accuracy: %s" % (test_accuracy)) return test_accuracy
import datetime import keras import numpy as np import os import progressbar from collections import defaultdict from settings import FLOAT_TYPE, COMMENT_DELIMITER, PARAMETERS_FILE_NAME_SUFFIX widgets_progressbar = [ ' [', progressbar.Timer(), '] ', progressbar.Bar("░", fill="⋅"), ' (', progressbar.ETA(), ') ', ] def get_parameters_dict(job_folder): """ :param job_folder: :return: """ parameters = defaultdict(str) try: parameters_file = list(filter(lambda file: PARAMETERS_FILE_NAME_SUFFIX in file, os.listdir(job_folder)))[0] with open(os.path.join(job_folder, parameters_file), "r") as f: lines = f.readlines() for line in lines: words = line.replace("\n", "").split("=")
def update_embedding(self, session, batch_size, model_path, loaded_emb=None, loaded_obj_bbs=None): # model_name = os.path.basename(model_path).split('.')[0] model_name = self._get_codebook_name(model_path) self._dataset._kw['model_path'] = list([str(model_path)]) self._dataset._kw[ 'model'] = 'cad' if 'cad' in model_path else self._dataset._kw[ 'model'] self._dataset._kw[ 'model'] = 'reconst' if 'reconst' in model_path else self._dataset._kw[ 'model'] if loaded_emb is None: embedding_size = self._dataset.embedding_size J = self._encoder.latent_space_size embedding_z = np.empty((embedding_size, J)) obj_bbs = np.empty((embedding_size, 4)) widgets = [ 'Creating embedding: ', progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.Counter(), ' / %s' % embedding_size, ' ', progressbar.ETA(), ' ' ] bar = progressbar.ProgressBar(maxval=embedding_size, widgets=widgets) bar.start() for a, e in u.batch_iteration_indices(embedding_size, batch_size): batch, obj_bbs_batch = self._dataset.render_embedding_image_batch( a, e) # import cv2 # cv2.imshow('',u.tiles(batch,10,10)) # cv2.waitKey(0) embedding_z[a:e] = session.run( self._encoder.z, feed_dict={self._encoder.x: batch}) if self.embed_bb: obj_bbs[a:e] = obj_bbs_batch bar.update(e) bar.finish() # embedding_z = embedding_z.T normalized_embedding = embedding_z / np.linalg.norm( embedding_z, axis=1, keepdims=True) else: normalized_embedding = loaded_emb obj_bbs = loaded_obj_bbs session.run(self.embedding_assign_op[model_name], {self.embedding: normalized_embedding}) if self.embed_bb: session.run(self.embed_obj_bbs_assign_op[model_name], {self.embed_obj_bbs: obj_bbs})
def run_conv_mnist(use_tensorboard=False, use_trace=False, arithmetic_type='tf'): ''' Run MNIST using the Wave custom matmul operator. ''' waveflow.waveflow_arithmetic = arithmetic_type tf.reset_default_graph() # Import data mnist = input_data.read_data_sets('./mnist_data') # Create the model x = tf.placeholder(tf.float32, [None, 784]) # Define loss and optimizer y_ = tf.placeholder(tf.int64, [None]) # Build the graph for the deep net y_conv, keep_prob = deepnn(x) # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.losses.sparse_softmax_cross_entropy on the raw # outputs of 'y', and then average across the batch. with tf.name_scope('loss'): cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_, logits=y_conv) avg_entropy = tf.reduce_mean(cross_entropy) if use_tensorboard: tf.summary.scalar("avg_loss", avg_entropy, family='Accuracy') # train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy) with tf.name_scope('adam_optimizer'): optimizer = tf.train.AdamOptimizer(1e-4) train_step = optimizer.minimize(avg_entropy) # tf.summary.scalar("learning_rate", optimizer._lr_t) with tf.name_scope('accuracy'): correct_prediction = tf.equal(tf.argmax(y_conv, 1), y_) correct_prediction = tf.cast(correct_prediction, tf.float32) accuracy = tf.reduce_mean(correct_prediction) if use_tensorboard: tf.summary.scalar("accuracy", accuracy, family='Accuracy') ''' with tf.name_scope('metrics'): y_prob = tf.one_hot(y_, 10) _, recall = tf.metrics.recall(y_conv, y_prob) _, precision = tf.metrics.precision(y_conv, y_prob) tf.summary.scalar('recall', recall) tf.summary.scalar('precision', precision) ''' tb_log = waveflow.TF_TBLogger(log_dir='./tb_conv_mnist_log/', enable_tb=use_tensorboard, enable_trace=use_trace, unified_trace=use_trace, arith_type=arithmetic_type) # print("vars:") if use_tensorboard: for v in tf.trainable_variables(): m_v = optimizer.get_slot(v, 'm') # print("var: %s, data: %s" % (v.name, m_v)) v_mean = tf.reduce_mean(m_v) tf.summary.scalar("%s" % (v.name), v_mean, family='Momentum') # print('nodes with trainng') # print('op list: ', op_list(tf.get_default_graph())) batch_size = 128 total_batches = 100 report_interval = 1 widgets = ["Training: ", pb.Percentage(), ' ', pb.Bar(), ' ', pb.ETA()] pbar = pb.ProgressBar(widgets=widgets, maxval=total_batches) # pbar.start() config = tf.ConfigProto(device_count={"CPU": 16}, inter_op_parallelism_threads=2, intra_op_parallelism_threads=16) with tf.Session('', config=config) as sess: print("Running model ...") tb_log.init_session(sess) tf.global_variables_initializer().run() # Train for i in range(total_batches): # pbar.update(i) batch_xs, batch_ys = mnist.train.next_batch(batch_size) tb_log.run_session(iter=i, ops=train_step, feed={x: batch_xs, y_: batch_ys, keep_prob: 0.5}) if i % report_interval == 0: train_accuracy = accuracy.eval(feed_dict={x: batch_xs, y_: batch_ys, keep_prob: 1.0}) print('step %d, training accuracy %g' % (i, train_accuracy)) # pbar.finish() tb_log.close() # Compute training set accuracy print("Evaluating full training set ...") sys.stdout.flush() train_accuracy = accuracy.eval(feed_dict={x: mnist.train.images, y_: mnist.train.labels, keep_prob: 1.0}) print("Train set accuracy: %s" % (train_accuracy)) # Compute test set accuracy print("Evaluating full test set ...") sys.stdout.flush() test_accuracy = accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}) print("Test set accuracy: %s" % (test_accuracy)) return test_accuracy
def generator(self, color): n_samples = len(self.train_list) # 进度条 widgets = ["数据集创建中: ", progressbar.Percentage(), " ", progressbar.Bar(), " ", progressbar.ETA()] pbar = progressbar.ProgressBar(maxval=n_samples, widgets=widgets).start() for i in range(n_samples): # 绘制当前文本 self.paint_text(self.train_list[i], i, color) pbar.update(i) pbar.finish()
def handle(infiles, tables, user_input_path, **kwargs): logger = logging.getLogger() msg = '{}: Starting'.format(VAR_NAME) logger.info(msg) logdir = kwargs.get('logdir') serial = kwargs.get('serial') # check that we have some input files for every variable zerofiles = False for variable in RAW_VARIABLES: if len(infiles[variable]) == 0: msg = '{}: Unable to find input files for {}'.format( VAR_NAME, variable) print_message(msg) logging.error(msg) zerofiles = True if zerofiles: return None # Create the logging directory and setup cmor if logdir: logpath = logdir else: outpath, _ = os.path.split(logger.__dict__['handlers'][0].baseFilename) logpath = os.path.join(outpath, 'cmor_logs') os.makedirs(logpath, exist_ok=True) logfile = os.path.join(logpath, VAR_NAME + '.log') cmor.setup( inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(str(user_input_path)) cmor.load_table(str(TABLE)) msg = '{}: CMOR setup complete'.format(VAR_NAME) logging.info(msg) # extract data from the input file msg = 'areacella: loading area' logger.info(msg) filename = infiles['area'][0] if not os.path.exists(filename): raise IOError("File not found: {}".format(filename)) f = cdms2.open(filename) # load the data for each variable variable_data = f('area') if not variable_data.any(): raise IOError("Variable data not found: {}".format(variable)) # load the lon and lat info & bounds data = { 'lat': variable_data.getLatitude(), 'lon': variable_data.getLongitude(), 'lat_bnds': f('lat_bnds'), 'lon_bnds': f('lon_bnds'), 'area': f('area') } msg = '{name}: loading axes'.format(name=VAR_NAME) logger.info(msg) axes = [{ str('table_entry'): str('latitude'), str('units'): data['lat'].units, str('coord_vals'): data['lat'][:], str('cell_bounds'): data['lat_bnds'][:] }, { str('table_entry'): str('longitude'), str('units'): data['lon'].units, str('coord_vals'): data['lon'][:], str('cell_bounds'): data['lon_bnds'][:] }] msg = 'areacella: running CMOR' logging.info(msg) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids) if serial: myMessage = progressbar.DynamicMessage('running') myMessage.__call__ = my_dynamic_message widgets = [ progressbar.DynamicMessage('running'), ' [', progressbar.Timer(), '] ', progressbar.Bar(), ' (', progressbar.ETA(), ') ' ] progressbar.DynamicMessage.__call__ = my_dynamic_message pbar = progressbar.ProgressBar( maxval=1, widgets=widgets) pbar.start() r = 6.37122e6 outdata = data['area'] * pow(r, 2) cmor.write( varid, outdata) if serial: pbar.update(1, running=msg) pbar.finish() msg = '{}: write complete, closing'.format(VAR_NAME) logger.debug(msg) cmor.close() msg = '{}: file close complete'.format(VAR_NAME) logger.debug(msg) return 'areacella'
def __save_new_documents(path_out: pathlib.Path, sentences: list) -> None: grade_levels = list(set(sentence.grade_level for sentence in sentences)) widgets = ['Saving by grade level: ', pb.Percentage(), ' ', pb.Bar(marker = '.', left = '[', right = ']'), ' ', pb.ETA()] with pb.ProgressBar(widgets = widgets, max_value = len(grade_levels)) as bar: gl_i = 0 for grade_level in grade_levels: bar.update(gl_i) file_out = path_out.joinpath(f'grade_level.{grade_level}.txt') with file_out.open('a', encoding = 'utf-8') as file_out: for sentence in sentences: if sentence.grade_level == grade_level: file_out.write(f'{sentence.text}\n') gl_i = gl_i + 1
def create_address_db(dbfilename, blockdir, table_len, startBlockDate="2019-01-01", endBlockDate="3000-12-31", startBlockFile=0, addressDB_yolo=False, outputToText=False, update=False, progress_bar=True, addresslistfile=None, multiFile=False): """Creates an AddressSet database and saves it to a file :param dbfilename: the file name where the database is saved (overwriting it) :type dbfilename: str :param blockdir: the data directory where the Bitcoin block files reside :type blockdir: str :param update: if True, the existing database file is updated from new txs :type update: bool :param progress_bar: True to enable the progress bar :type progress_bar: bool """ if update: print("Loading address database ...") address_set = AddressSet.fromfile(open(dbfilename, "r+b"), mmap_access=mmap.ACCESS_WRITE) first_filenum = address_set.last_filenum print() else: first_filenum = startBlockFile if not addresslistfile: for filename in glob.iglob(path.join(blockdir, "blk*.dat")): if path.isfile(filename): break else: raise ValueError( "no block files exist in blocks directory '{}'".format( blockdir)) filename = "blk{:05}.dat".format(first_filenum) if not path.isfile(path.join(blockdir, filename)): raise ValueError( "first block file '{}' doesn't exist in blocks directory '{}'". format(filename, blockdir)) if not update: # Open the file early to make sure we can, but don't overwrite it yet # (see AddressSet.tofile() for why io.open() instead of open() is used) try: dbfile = io.open(dbfilename, "r+b") except IOError: dbfile = io.open(dbfilename, "wb") #Try to create the AddressDB. If the addresset is sufficiently large (eg: BTC) then this requires 64 bit python and will crash if attempted with 32 bit Python... try: # With the default bytes_per_addr and max_load, this allocates # about 8 GiB which is room for a little over 800 million addresses (Required as of 2019) address_set = AddressSet(1 << table_len) except OverflowError: print() exit( "AddressDB too large for use with 32 bit Python. You will need to install a 64 bit (x64) version of Python 3 from python.org and try again" ) if addresslistfile: import btcrecover.btcrseed print("Initial AddressDB Contains", len(address_set), "Addresses") for i in range(9999): if multiFile: addresslistfile = addresslistfile[:-4] + '{:04d}'.format(i) try: with open(addresslistfile) as addressList_file: print("Loading: ", addresslistfile) addresses_loaded = 0 for address in addressList_file: try: if (address[0:2] != '0x'): address_set.add( btcrecover.btcrseed.WalletBase. _addresses_to_hash160s([address.rstrip() ]).pop()) else: address_set.add( btcrecover.btcrseed.WalletEthereum. _addresses_to_hash160s([address.rstrip() ]).pop()) addresses_loaded += 1 if (addresses_loaded % 1000000 == 0): print("Checked:", addresses_loaded, "addresses in current file,", len(address_set), "in unique Hash160s in AddressDB") except bitcoinlib.encoding.EncodingError: print("Skipping Invalid Address:", address.rstrip()) print("Finished: ", addresslistfile) if not multiFile: break except FileNotFoundError: if multiFile: continue else: print("File:", addresslistfile, " not found") exit() print("Finished AddressDB Contains", len(address_set), "Addresses") else: if progress_bar: try: import progressbar except ImportError: progress_bar = False if progress_bar: print("Parsing block files ...") for filenum in itertools.count(first_filenum): filename = path.join(blockdir, "blk{:05}.dat".format(filenum)) if not path.isfile(filename): break progress_label = progressbar.FormatLabel( " {:11,} addrs. %(elapsed)s, ".format(len(address_set))) block_bar_widgets = [ progressbar.SimpleProgress(), " ", progressbar.Bar(left="[", fill="-", right="]"), progress_label, progressbar.ETA() ] progress_bar = progressbar.ProgressBar(maxval=filenum - first_filenum, widgets=block_bar_widgets) progress_bar.start() else: print("Block file Address count") print("------------ -------------") # e.g. blk00943.dat 255,212,706 for filenum in itertools.count(first_filenum): filename = path.join(blockdir, "blk{:05}.dat".format(filenum)) if not path.isfile(filename): break address_set.last_filenum = filenum with open(filename, "rb") as blockfile: if not progress_bar: print(path.basename(filename), end=" ") header = blockfile.read( 8 ) # read in the magic and remaining (after these 8 bytes) block length chain_magic = header[:4] #print("Found Magic:", chain_magic.encode("hex")) while len(header) == 8 and header[4:] != b"\0\0\0\0": if supportedChains( chain_magic ) != 1: # Check magic to see if it is a chain we support if not addressDB_yolo: #Ignore checks on the blockchain type #Throw an error message and exit if we encounter unsupported magic value if supportedChains(chain_magic) == -1: print( "Unrecognised Block Protocol (Unrecognised Magic), Found:", chain_magic, " You can force an AddressDB creation attempt by re-running this tool with the flag --dbyolo" ) if supportedChains(chain_magic) == 0: print( "Incompatible Block Protocol, You can force an AddressDB creation attempt by re-running this tool with the flag --dbyolo, but it probably won't work" ) exit() block = blockfile.read( struct.unpack_from( "<I", header, 4)[0]) # read in the rest of the block tx_count, offset = varint(block, 80) # skips 80 bytes of header #Extract Block Header info (Useful for debugging extra new chains) #print("Block Header: ", block[0:80].encode("hex")) #print() #Get Block Header Info (Useful for debugging and limiting date range) block_version = block[0:4] block_prevHash = block[4:36] block_merkleRoot = block[36:68] block_time = struct.unpack("<I", block[68:72])[0] block_bits = struct.unpack("<I", block[72:76])[0] block_nonce = struct.unpack("<I", block[76:80])[0] #print_debug = False #if block_prevHash.encode("hex") =='52aa3101be5119a77cce7a8f2e2a8fcdfcbcf6ca0f3e15000000000000000000': # print_debug = True #print("Block Version: ", block_version.hex()) #print("Block PrevHash: ", block_prevHash.hex()) #print("Block MerkleRoot: ", block_merkleRoot.hex()) #print("Block Bits: ", block_bits) #print("Block Nonce: ", block_nonce) #print("Block TIme: ", block_time, " " , datetime.fromtimestamp(float(block_time))) blockDate = datetime.fromtimestamp(float(block_time)) #Only add addresses which occur in blocks that are within the time window we are looking at if datetime.strptime(startBlockDate + " 00:00:00", '%Y-%m-%d %H:%M:%S' ) <= blockDate and datetime.strptime( endBlockDate + " 23:59:59", '%Y-%m-%d %H:%M:%S') >= blockDate: for tx_num in range(tx_count): offset += 4 # skips 4-byte tx version is_bip144 = block[offset] == 0 # bip-144 marker if is_bip144: offset += 2 # skips 1-byte marker & 1-byte flag txin_count, offset = varint(block, offset) for txin_num in range(txin_count): sigscript_len, offset = varint( block, offset + 36 ) # skips 32-byte tx id & 4-byte tx index offset += sigscript_len + 4 # skips sequence number & sigscript txout_count, offset = varint(block, offset) for txout_num in range(txout_count): pkscript_len, offset = varint( block, offset + 8) # skips 8-byte satoshi count #if print_debug: # print("Tx Data: ", block[offset:offset+100].encode("hex")) #Print all TX data (plus more for debugging) # If this is a P2PKH script (OP_DUP OP_HASH160 PUSH(20) <20 address bytes> OP_EQUALVERIFY OP_CHECKSIG) if pkscript_len == 25 and block[ offset:offset + 3] == b"\x76\xa9\x14" and block[ offset + 23:offset + 25] == b"\x88\xac": address_set.add( block[offset + 3:offset + 23], outputToText, 'P2PKH') elif block[ offset:offset + 2] == b"\xa9\x14": #Check for Segwit Address address_set.add( block[offset + 2:offset + 22], outputToText, 'P2SH') elif block[ offset:offset + 2] == b"\x00\x14": #Check for Native Segwit Address address_set.add( block[offset + 2:offset + 22], outputToText, 'Bech32') offset += pkscript_len # advances past the pubkey script if is_bip144: for txin_num in range(txin_count): stackitem_count, offset = varint( block, offset) for stackitem_num in range( stackitem_count): stackitem_len, offset = varint( block, offset) offset += stackitem_len # skips this stack item offset += 4 # skips the 4-byte locktime header = blockfile.read( 8) # read in the next magic and remaining block length if progress_bar: block_bar_widgets[3] = progressbar.FormatLabel( " {:11,} addrs. %(elapsed)s, ".format( len(address_set))) # updates address count nextval = progress_bar.currval + 1 if nextval > progress_bar.maxval: # can happen if the bitcoin client is left running progress_bar.maxval = nextval progress_bar.update(nextval) else: print("{:13,}".format(len(address_set))) if progress_bar: progress_bar.widgets.pop() # remove the ETA progress_bar.finish() if update: print("\nSaving changes to address database ...") address_set.close() else: print("\nSaving address database ...") dbfile.truncate(0) address_set.tofile(dbfile) dbfile.close() print("\nDone.")
def __document_to_sentences(document_name: pathlib.Path) -> list: lines = u.read_document(document_name) sentences = lines sentences = [sentence.strip() for sentence in sentences] sentences = [sentence for sentence in sentences if len(sentence) > 0] widgets = ['Calculating grade level: ', pb.Percentage(), ' ', pb.Bar(marker = '.', left = '[', right = ']'), ' ', pb.ETA()] with pb.ProgressBar(widgets = widgets, max_value = len(sentences)) as bar: for i in range(0, len(sentences)): bar.update(i) sentence = sentences[i] grade_level = __calculate_sentences_median_grade_level(sentence) sentences[i] = sentenceplus(sentence, grade_level) return sentences
def recon_mpi(sinograms, theta, recon_series, stepsize=10, center=None, recon=None, **kwds): """reconstruction using mpi. This method needs to be run on several mpi nodes to achieve parallalization. sth similar to $ mpirun -np NODES python "code to call this method" * theta: angles in radians * recon: reconstruction method """ import logging logger = logging.getLogger("mpi") import imars3d.io from mpi4py import MPI comm = MPI.COMM_WORLD size = comm.Get_size() rank = comm.Get_rank() totalN = len(sinograms) N = int(np.ceil(totalN * 1. / size)) start, stop = rank * N, min(totalN, (rank + 1) * N) # print("node %s of %s handles %s" % (rank, size, layers)) # print("N, start, stop=%s, %s, %s" % (N, start, stop)) if recon is None: from .use_tomopy import recon_batch_singlenode as recon # progress bar # for simplicity, we just report the progress at rank 0, which should be a # good indication of progress of all nodes any way if rank == 0: bar = progressbar.ProgressBar(widgets=[ "Reconstructing", progressbar.Percentage(), progressbar.Bar(), ' [', progressbar.ETA(), '] ', ], max_value=stop - start, **pb_config) start0 = start # shall be fine to define inside if-block. Just to make flake8/pylint feel betters # avoid infinite loop loop = -1 while start < stop or loop < MAX_LOOP: # update loop loop += 1 stop1 = min(start + stepsize, stop) logger.debug("node %s of %s working on %s:%s" % (rank, size, start, stop1)) sinograms1 = sinograms[start:stop1] if not len(sinograms): continue recon_series1 = recon_series[start:stop1] try: recon(sinograms1, theta, recon_series1, center=center, **kwds) except: logger.info("node %s of %s: recon %s:%s failed" % (rank, size, start, stop1)) # update range start = stop1 # update bar (rank 0) if rank == 0: bar.update(start - start0) continue comm.Barrier() if rank == 0: print('\n') return
def download_file(self, url=None, outputFileName=None, outputPath=None, bytes=False): def fmt_size(num, suffix="B"): for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: if abs(num) < 1024.0: return "%3.1f%s%s" % (num, unit, suffix) num /= 1024.0 return "%.1f%s%s" % (num, "Yi", suffix) #: if not url: raise Exception("No URL specified.") if outputPath is None: # Default to current dir. outputPath = os.getcwd() else: if not os.path.isdir(outputPath): raise Exception( 'Specified path "{0}" does not exist'.format(outputPath)) fileName = os.path.basename(url) # Get URL filename userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0" if 'sourceforge.net' in url.lower(): userAgent = 'wget/1.18' # sourceforce <3 wget if url.lower().startswith("ftp://"): self.log("Requesting : {0}".format(url)) if outputFileName != None: fileName = outputFileName fullOutputPath = os.path.join(outputPath, fileName) urllib.request.urlretrieve(url, fullOutputPath) return fullOutputPath req = requests.get(url, stream=True, headers={"User-Agent": userAgent}) if req.status_code != 200: req.raise_for_status() if "content-disposition" in req.headers: reSponse = re.findall("filename=(.+)", req.headers["content-disposition"]) if reSponse == None: fileName = os.path.basename(url) else: fileName = reSponse[0] size = None compressed = False if "Content-Length" in req.headers: size = int(req.headers["Content-Length"]) if "Content-Encoding" in req.headers: if req.headers["Content-Encoding"] == "gzip": compressed = True self.log("Requesting : {0} - {1}".format( url, fmt_size(size) if size != None else "?")) # terms = shutil.get_terminal_size((100,100)) # filler = 0 # if terms[0] > 100: # filler = int(terms[0]/4) widgetsNoSize = [ progressbar.FormatCustomText("Downloading: {:25.25}".format( os.path.basename(fileName))), " ", progressbar.AnimatedMarker(markers='|/-\\'), " ", progressbar.DataSize() # " "*filler ] widgets = [ progressbar.FormatCustomText("Downloading: {:25.25}".format( os.path.basename(fileName))), " ", progressbar.Percentage(), " ", progressbar.Bar(fill=chr(9617), marker=chr(9608), left="[", right="]"), " ", progressbar.DataSize(), "/", progressbar.DataSize(variable="max_value"), " |", progressbar.AdaptiveTransferSpeed(), " | ", progressbar.ETA(), # " "*filler ] pbar = None if size == None: pbar = progressbar.ProgressBar(widgets=widgetsNoSize, maxval=progressbar.UnknownLength) else: pbar = progressbar.ProgressBar(widgets=widgets, maxval=size) if outputFileName != None: fileName = outputFileName fullOutputPath = os.path.join(outputPath, fileName) updateSize = 0 if isinstance(pbar.max_value, int): updateSize = pbar.max_value if pbar.max_value < 1024 else 1024 if bytes == True: output = b"" bytesrecv = 0 pbar.start() for buffer in req.iter_content(chunk_size=1024): if buffer: output += buffer if compressed: pbar.update(updateSize) else: pbar.update(bytesrecv) bytesrecv += len(buffer) pbar.finish() return output else: with open(fullOutputPath, "wb") as file: bytesrecv = 0 pbar.start() for buffer in req.iter_content(chunk_size=1024): if buffer: file.write(buffer) file.flush() if compressed: pbar.update(updateSize) else: pbar.update(bytesrecv) bytesrecv += len(buffer) pbar.finish() return fullOutputPath
from __future__ import division, print_function import numpy as np import progressbar from utils import divide_on_feature, divide_on_feature2 bar_widgets = [ 'Training: ', progressbar.Percentage(), ' ', progressbar.Bar(marker="-", left="[", right="]"), ' ', progressbar.ETA(), '\n' ] class LeastSquaresLoss(): # loss类,包含一阶二阶梯度两个方法 """ 最小二乘损失 """ def gradient(self, actual, predicted): return actual - predicted # 对于平方损失而言,负梯度就是残差 def hess(self, actual, predicted): return np.ones_like(actual) # 对于平方损失而言,二阶导数为1 class DecisionNode(): """ 决策树的节点类 Parameters: ----------- feature_i: int 分裂的特征索引号 threshold: float 分裂特征的分裂阈值 value: float
def main(): seeding() # number of parallel agents parallel_envs = 8 # number of training episodes. # change this to higher number to experiment. say 30000. number_of_episodes = 1000 episode_length = 80 batchsize = 1000 # how many episodes to save policy and gif save_interval = 1000 t = 0 # amplitude of OU noise # this slowly decreases to 0 noise = 2 noise_reduction = 0.9999 # how many episodes before update episode_per_update = 2 * parallel_envs log_path = os.getcwd() + "/log" model_dir = os.getcwd() + "/model_dir" os.makedirs(model_dir, exist_ok=True) torch.set_num_threads(parallel_envs) env = envs.make_parallel_env(parallel_envs) # keep 5000 episodes worth of replay buffer = ReplayBuffer(int(5000 * episode_length)) # initialize policy and critic maddpg = MADDPG() logger = SummaryWriter(log_dir=log_path) agent0_reward = [] agent1_reward = [] agent2_reward = [] # training loop # show progressbar import progressbar as pb widget = [ 'episode: ', pb.Counter(), '/', str(number_of_episodes), ' ', pb.Percentage(), ' ', pb.ETA(), ' ', pb.Bar(marker=pb.RotatingMarker()), ' ' ] timer = pb.ProgressBar(widgets=widget, maxval=number_of_episodes).start() # use keep_awake to keep workspace from disconnecting for episode in keep_awake(range(0, number_of_episodes, parallel_envs)): timer.update(episode) reward_this_episode = np.zeros((parallel_envs, 3)) all_obs = env.reset() # obs, obs_full = transpose_list(all_obs) #for calculating rewards for this particular episode - addition of all time steps # save info or not save_info = ((episode) % save_interval < parallel_envs or episode == number_of_episodes - parallel_envs) frames = [] tmax = 0 if save_info: frames.append(env.render('rgb_array')) for episode_t in range(episode_length): t += parallel_envs # explore = only explore for a certain number of episodes # action input needs to be transposed actions = maddpg.act(transpose_to_tensor(obs), noise=noise) noise *= noise_reduction actions_array = torch.stack(actions).detach().numpy() # transpose the list of list # flip the first two indices # input to step requires the first index to correspond to number of parallel agents actions_for_env = np.rollaxis(actions_array, 1) # step forward one frame next_obs, next_obs_full, rewards, dones, info = env.step( actions_for_env) # add data to buffer transition = (obs, obs_full, actions_for_env, rewards, next_obs, next_obs_full, dones) buffer.push(transition) reward_this_episode += rewards obs, obs_full = next_obs, next_obs_full # save gif frame if save_info: frames.append(env.render('rgb_array')) tmax += 1 # update once after every episode_per_update if len(buffer ) > batchsize and episode % episode_per_update < parallel_envs: for a_i in range(3): samples = buffer.sample(batchsize) maddpg.update(samples, a_i, logger) maddpg.update_targets( ) #soft update the target network towards the actual networks for i in range(parallel_envs): agent0_reward.append(reward_this_episode[i, 0]) agent1_reward.append(reward_this_episode[i, 1]) agent2_reward.append(reward_this_episode[i, 2]) if episode % 100 == 0 or episode == number_of_episodes - 1: avg_rewards = [ np.mean(agent0_reward), np.mean(agent1_reward), np.mean(agent2_reward) ] agent0_reward = [] agent1_reward = [] agent2_reward = [] for a_i, avg_rew in enumerate(avg_rewards): logger.add_scalar('agent%i/mean_episode_rewards' % a_i, avg_rew, episode) #saving model save_dict_list = [] if save_info: for i in range(3): save_dict = { 'actor_params': maddpg.maddpg_agent[i].actor.state_dict(), 'actor_optim_params': maddpg.maddpg_agent[i].actor_optimizer.state_dict(), 'critic_params': maddpg.maddpg_agent[i].critic.state_dict(), 'critic_optim_params': maddpg.maddpg_agent[i].critic_optimizer.state_dict() } save_dict_list.append(save_dict) torch.save( save_dict_list, os.path.join(model_dir, 'episode-{}.pt'.format(episode))) # save gif files imageio.mimsave(os.path.join(model_dir, 'episode-{}.gif'.format(episode)), frames, duration=.04) env.close() logger.close() timer.finish()
def generate_fill(self, max_length=None, no_duration=False, verbose=1): """ :param max_length: :param no_duration: :param verbose: :return: """ # ----- Parameters ----- max_length = 300 / self.step_length if max_length is None else max_length # ----- Variables ----- if self.data_transformed_path is None: raise Exception('Some data need to be loaded before comparing the generation') sequence = Sequences.KerasSequence( path=self.data_transformed_path, nb_steps=self.nb_steps, batch_size=1, work_on=self.work_on ) # Return array instead of list (for instruments) max_length = int(min(max_length, len(sequence))) nb_instruments = sequence.nb_instruments # ----- Seeds ----- truth = sequence[0][0] filled_list = [np.copy(truth) for inst in range(nb_instruments)] mask = np.ones((nb_instruments, nb_instruments, self.nb_steps)) for inst in range(nb_instruments): filled_list[inst][inst] = 0 mask[inst, inst] = 0 # ----- Generation ----- cprint('Start generating (fill) ...', 'blue') bar = progressbar.ProgressBar(maxval=max_length, widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage(), ' ', progressbar.ETA()]) bar.start() # To see it working for l in range(max_length): s_input, s_output = sequence[l] to_fill_list = [np.copy(s_input) for inst in range(nb_instruments)] for inst in range(nb_instruments): to_fill_list[inst][inst] = 0 nn_input = np.concatenate( tuple(to_fill_list), axis=1 ) # (nb_instruments, batch=nb_instruments, nb_steps, step_size, input_size, channels) preds = self.keras_nn.generate(input=list(nn_input) + [mask]) preds = np.asarray(preds).astype( 'float64') # (nb_instruments, bath=nb_instruments, nb_steps=1, step_size, input_size, channels) if len(preds.shape) == 5: # Only one instrument : output of nn not a list preds = np.expand_dims(preds, axis=0) if len(s_output.shape) == 5: # Only one instrument : output of nn not a list s_output = np.expand_dims(s_output) preds = midi.create.normalize_activation(preds, mono=self.mono) truth = np.concatenate((truth, s_output), axis=2) for inst in range(nb_instruments): p = np.copy(s_output) p[inst] = np.take(preds, axis=1, indices=[inst])[inst] filled_list[inst] = np.concatenate( (filled_list[inst], p), axis=2) # (nb_instruments, batch=1, nb_steps, step_size, input_size, channels) bar.update(l + 1) bar.finish() # -------------------- Compute notes list -------------------- # ----- Reshape ----- truth = self.reshape_generated_array(truth) for inst in range(nb_instruments): filled_list[inst] = self.reshape_generated_array(filled_list[inst]) self.ensure_save_midis_path() self.save_midis_path.mkdir(parents=True, exist_ok=True) self.compute_generated_array( generated_array=truth, file_name=self.save_midis_path / 'generated_fill_truth', no_duration=no_duration, verbose=verbose, save_images=True ) for inst in range(nb_instruments): self.compute_generated_array( generated_array=filled_list[inst], file_name=self.save_midis_path / f'generated_fill_{inst}', no_duration=no_duration, array_truth=truth, verbose=verbose, save_truth=False, save_images=True ) cprint('Done generating (fill)', 'green')
def train(maddpg, env, n_episodes=1000, save_every=50): """Training loop helper for running the environment using the MADDPG algorithm. Params ====== maddpg (MADDPG): instance of MADDPG wrapper class env (UnityEnvironment): instance of Unity environment for training n_episodes (int): number of episodes to train for save_every (int): frequency to save model weights """ widget = [ "Episode: ", pb.Counter(), '/', str(n_episodes), ' ', pb.Percentage(), ' ', pb.ETA(), ' ', pb.Bar(marker=pb.RotatingMarker()), ' ', 'Rolling Average: ', pb.FormatLabel('') ] timer = pb.ProgressBar(widgets=widget, maxval=n_episodes).start() solved = False scores_total = [] scores_deque = deque(maxlen=100) rolling_score_averages = [] last_best_score = 0.0 # Environment information brain_name = env.brain_names[0] for i_episode in range(1, n_episodes + 1): current_average = 0.0 if i_episode == 1 else rolling_score_averages[-1] widget[12] = pb.FormatLabel(str(current_average)[:6]) timer.update(i_episode) env_info = env.reset(train_mode=True)[brain_name] states = env_info.vector_observations[:, -STATE_SIZE:] scores = np.zeros(NUM_AGENTS) maddpg.reset() while True: actions = maddpg.act(states) env_info = env.step(actions)[brain_name] next_states = env_info.vector_observations[:, -STATE_SIZE:] rewards = env_info.rewards dones = env_info.local_done maddpg.step(states, actions, rewards, next_states, dones) scores += rewards states = next_states if np.any(dones): break max_episode_score = np.max(scores) scores_deque.append(max_episode_score) scores_total.append(max_episode_score) average_score = np.mean(scores_deque) rolling_score_averages.append(average_score) if average_score >= 0.5 and not solved: print( '\nEnvironment solved in {:d} episodes!\tAverage Score: {:.2f}' .format(i_episode, average_score)) solved = True maddpg.save_model() last_best_score = average_score if i_episode % save_every == 0 and solved: # Only save these weights if they are better than the ones previously saved if average_score > last_best_score: last_best_score = average_score maddpg.save_model() return scores_total, rolling_score_averages
def compare_generation(self, max_length=None, no_duration=False, verbose=1): """ :return: """ # -------------------- Find informations -------------------- if self.data_transformed_path is None: raise Exception('Some data need to be loaded before comparing the generation') self.sequence = Sequences.AllInstSequence( path=str(self.data_transformed_path), nb_steps=self.nb_steps, batch_size=1, work_on=self.work_on, noise=0 ) max_length = len(self.sequence) if max_length is None else min(max_length, len(self.sequence)) # -------------------- Construct seeds -------------------- generated = np.array(self.sequence[0][0]) # (nb_instrument, 1, nb_steps, step_size, input_size, 2) (1=batch) generated_helped = np.copy(generated) # Each step will take the truth as an input generated_truth = np.copy(generated) # The truth mask = self.get_mask(self.sequence.nb_instruments) # -------------------- Generation -------------------- cprint('Start comparing generation ...', 'blue') bar = progressbar.ProgressBar(maxval=max_length, widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage(), ' ', progressbar.ETA()]) bar.start() # To see it working for l in range(max_length): ms_input, ms_output = self.sequence[l] sample = np.concatenate((generated[:, :, l:], np.array(ms_input)), axis=1) # (nb_instruments, 2, nb_steps, step_size, input_size, 2) # Generation preds = self.keras_nn.generate(input=list(sample) + mask) # Reshape preds = np.asarray(preds).astype('float64') # (nb_instruments, batch=2, nb_steps=1, length, 88, 2) preds_truth = np.array(ms_output) # (nb_instruments, 1, 1, step_size, input_size, 2) # if only one instrument if len(preds.shape) == 5: # Only one instrument : output of nn not a list preds = np.expand_dims(preds, axis=0) if len(preds_truth.shape) == 5: # Only one instrument : output of nn not a list preds_truth = np.expand_dims(preds_truth) preds = midi.create.normalize_activation(preds, mono=self.mono) # Normalize the activation part preds_helped = preds[:, [1]] # (nb_instruments, 1, 1, length, 88, 2) preds = preds[:, [0]] # Concatenation generated = np.concatenate((generated, preds), axis=2) # (nb_instruments, 1, nb_steps, length, 88, 2) generated_helped = np.concatenate((generated_helped, preds_helped), axis=2) # (nb_instruments, 1, nb_steps, length, 88, 2) generated_truth = np.concatenate((generated_truth, preds_truth), axis=2) bar.update(l + 1) bar.finish() # -------------------- Compute notes list -------------------- # Generated generated_midi_final = self.reshape_generated_array(generated) # Helped generated_midi_final_helped = self.reshape_generated_array(generated_helped) # Truth generated_midi_final_truth = self.reshape_generated_array(generated_truth) # ---------- find the name for the midi_file ---------- self.ensure_save_midis_path() self.save_midis_path.mkdir(parents=True, exist_ok=True) # Generated self.compute_generated_array( generated_array=generated_midi_final, file_name=self.save_midis_path / 'compare_generation_alone', no_duration=no_duration, array_truth=generated_midi_final_truth, verbose=verbose, save_truth=False, save_images=True ) # Helped self.compute_generated_array( generated_array=generated_midi_final_helped, file_name=self.save_midis_path / 'compare_generation_helped', no_duration=no_duration, array_truth=generated_midi_final_truth, verbose=verbose, save_truth=False, save_images=True ) # Truth self.compute_generated_array( generated_array=generated_midi_final_truth, file_name=self.save_midis_path / 'compare_generation_truth', no_duration=no_duration, array_truth=None, verbose=verbose, save_truth=False, save_images=True ) cprint('Done comparing generation', 'green')
def apply_batches(function, arguments, batch_size, description='', show_progressbar=False, show_error_output=True): """ Apply batches to a specified function. Parameters ---------- function : func Function that accepts one or more positional arguments. Each of them should be an array-like variable that have exactly the same number of rows. arguments : tuple, list The arguemnts that will be provided to the function specified in the ``function`` argument. batch_size : int Mini-batch size. description : str Short description that will be displayed near the progressbar in verbose mode. Defaults to ``''`` (empty string). show_progressbar : bool ``True`` means that function will show progressbar in the terminal. Defaults to ``False``. show_error_output : bool Assumes that outputs from the function errors. ``True`` will show information in the progressbar. Error will be related to the last epoch. Defaults to ``True``. Returns ------- list List of function outputs. """ if not arguments: raise ValueError("The argument parameter should be list or " "tuple with at least one element.") samples = arguments[0] n_samples = len(samples) batch_iterator = list(iter_batches(n_samples, batch_size)) if show_progressbar: widgets = [ progressbar.Timer(format='Time: %(elapsed)s'), ' |', progressbar.Percentage(), progressbar.Bar(), ' ', progressbar.ETA(), ] if show_error_output: widgets.extend([' | ', progressbar.DynamicMessage('error')]) bar = progressbar.ProgressBar( widgets=widgets, max_value=len(batch_iterator), poll_interval=0.1, ) bar.update(0) else: bar = progressbar.NullBar() outputs = [] for i, batch in enumerate(batch_iterator): sliced_arguments = [argument[batch] for argument in arguments] output = function(*sliced_arguments) outputs.append(output) if show_error_output: bar.update(i, error=np.atleast_1d(output).item(0)) else: bar.update(i) bar.finish('\r' + ' ' * bar.term_width + '\r') return outputs
def update_permissions(config, is_test, activ_path, conda_base, system_libs): directories = [] if not is_test: directories.append(conda_base) if system_libs is not None: # even if this is not a release, we need to update permissions on # shared system libraries directories.append(system_libs) group = config.get('e3sm_unified', 'group') new_uid = os.getuid() new_gid = grp.getgrnam(group).gr_gid print('changing permissions on activation scripts') read_perm = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH) exec_perm = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO if not is_test: activation_files = glob.glob('{}/*_compass*.sh'.format(activ_path)) for file_name in activation_files: os.chmod(file_name, read_perm) os.chown(file_name, new_uid, new_gid) print('changing permissions on environments') # first the base directories that don't seem to be included in # os.walk() for directory in directories: try: dir_stat = os.stat(directory) except OSError: continue perm = dir_stat.st_mode & mask if dir_stat.st_uid != new_uid: # current user doesn't own this dir so let's move on continue if perm == exec_perm and dir_stat.st_gid == new_gid: continue try: os.chown(directory, new_uid, new_gid) os.chmod(directory, exec_perm) except OSError: continue files_and_dirs = [] for base in directories: for root, dirs, files in os.walk(base): files_and_dirs.extend(dirs) files_and_dirs.extend(files) widgets = [ progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA() ] bar = progressbar.ProgressBar(widgets=widgets, maxval=len(files_and_dirs)).start() progress = 0 for base in directories: for root, dirs, files in os.walk(base): for directory in dirs: progress += 1 bar.update(progress) directory = os.path.join(root, directory) try: dir_stat = os.stat(directory) except OSError: continue if dir_stat.st_uid != new_uid: # current user doesn't own this dir so let's move on continue perm = dir_stat.st_mode & mask if perm == exec_perm and dir_stat.st_gid == new_gid: continue try: os.chown(directory, new_uid, new_gid) os.chmod(directory, exec_perm) except OSError: continue for file_name in files: progress += 1 bar.update(progress) file_name = os.path.join(root, file_name) try: file_stat = os.stat(file_name) except OSError: continue if file_stat.st_uid != new_uid: # current user doesn't own this file so let's move on continue perm = file_stat.st_mode & mask if perm & stat.S_IXUSR: # executable, so make sure others can execute it new_perm = exec_perm else: new_perm = read_perm if perm == new_perm and file_stat.st_gid == new_gid: continue try: os.chown(file_name, new_uid, new_gid) os.chmod(file_name, new_perm) except OSError: continue bar.finish() print(' done.')
def do_stuff(ctr): p = pb.ProgressBar(maxval=740, widgets = [pb.Percentage(),pb.Bar(),pb.ETA()]).start() pbctr = 0 for i,l in enumerate(lc): p.update(pbctr) pbctr += 1 restcut = (3000,7000) data = sncosmo.read_lc(l, format='salt2') try: z = data.meta['Redshift'] except: pass try: z = data.meta['Z_CMB'] except: pass try: survey = data.meta['SURVEY'] except: pass try: survey = data.meta['SET'] except: pass nickname = data.meta['SN'] try: nickname = str(int(nickname)) except: pass mwebv = data.meta['MWEBV'] dust = sncosmo.CCM89Dust() data = astropy.table.Table(data, masked=True) #rename columns so that my fitter can handle things data.rename_column('Filter', 'tmp') data.rename_column('Date', 'time') data.rename_column('Flux', 'flux') data.rename_column('Fluxerr', 'fluxerr') data.rename_column('MagSys', 'zpsys') data.rename_column('ZP', 'zp') if survey == 'SNLS': sn_nickname = l.split('/')[-1].split('.')[0].split('-')[-1] band = [] for j, bp in enumerate(data['tmp']): band.append( '%s-%s' %(sn_nickname, bp) ) band = astropy.table.Column(band, name='band') data.add_column(band) data.remove_column('tmp') else: data.rename_column('tmp', 'band') # deal with swope filters mask = (data['band'] == 'SWOPE2::V') nswopev = len(mask.nonzero()[0]) if nswopev > 0: band = [] for j, bp in enumerate(data['band']): if (bp == 'SWOPE2::V'): if (data['time'][j] < 53749): band.append('swope2::v_lc3014') elif (data['time'][j] < 53760): band.append('swope2::v_lc3009') else: band.append('swope2::v_lc9844') else: band.append(bp) data.remove_column('band') band = astropy.table.Column(band, name='band') data.add_column(band) ind = np.where( (data['band'] == 'SWOPE2::V') & (data['time']>53749.) & ((data['time']<=53760.)) ) data['band'][ind] = 'swope2::v_lc3009' ind = np.where( (data['band'] == 'SWOPE2::V') & (data['time']>53760.) ) data['band'][ind] = 'swope2::v_lc9844' # print ind #deal with filter coverage #also deal with STANDARD filter zeropoints unique_bands = np.unique(data['band']) fit_bands = [] nofit_bands = [] # print unique_bands tmperr = np.copy(data['fluxerr']) for ub in unique_bands: # print ub bp = sncosmo.get_bandpass(ub) rest = bp.wave_eff / (1.0+z) if (rest >= restcut[0]) & (rest <= restcut[1]): fit_bands.append(ub) else: nofit_bands.append(ub) if 'STANDARD' in ub: ind = np.where(data['band'] == ub) data['zp'][ind] = data['zp'][ind] - float(standard_zps[ub]) errcor = 10**(-0.4*standard_zps[ub]) data['fluxerr'][ind] *= errcor if '4SHOOTER2' in ub: ind = np.where(data['band'] == ub) data['zp'][ind] = data['zp'][ind] - float(FourShooter_zps[ub]) errcor = 10**(-0.4*FourShooter_zps[ub]) data['fluxerr'][ind] *= errcor if 'KEPLERCAM' in ub: ind = np.where(data['band'] == ub) data['zp'][ind] = data['zp'][ind] - float(keplercam_zps[ub]) errcor = 10**(-0.4*keplercam_zps[ub]) data['fluxerr'][ind] *= errcor # print ub # print data['zp'][ind] if 'swope' in ub.lower(): ind = np.where(data['band'] == ub) data['zp'][ind] = data['zp'][ind] - float(swope_zps[ub]) errcor = 10**(-0.4*swope_zps[ub]) data['fluxerr'][ind] *= errcor if 'sdss' in ub.lower(): ind = np.where(data['band'] == ub) data['zp'][ind] = data['zp'][ind] - float(sdss_zps[ub]) errcor = 10**(-0.4*sdss_zps[ub]) data['fluxerr'][ind] *= errcor for nfb in nofit_bands: mask = data['band'] == nfb for c in data.colnames: data[c].mask = (data[c].mask | mask) mwebv = data.meta['MWEBV'] mask = data['band'].mask.nonzero()[0] data.remove_rows(mask) ind = np.where(lcfits['SN'] == nickname) t0 = lcfits['DayMax'][ind][0] x1r, x1f, c = np.random.multivariate_normal(mean,cov,size=1)[0] mu = cosmo.distmod(z).value absmag = MB - ar*x1r - af*x1f + beta*c + np.random.normal(scale=sigint, size=1)[0] mB = mu + absmag source = Salt2XSource(version='2.4', modeldir=modeldir) model = sncosmo.Model(source=source, effects=[dust], effect_names=['mw'], effect_frames=['obs']) model.set(z=z, x1=x1f, s=x1r, c=c, t0=t0, mwebv=mwebv) model.set_source_peakabsmag(absmag, 'bessellb', 'ab') flux = model.bandflux(data['band'], data['time'], zp=data['zp'], zpsys=data['zpsys']) whocares, saltcov = model.bandfluxcov(data['band'], data['time'], data['zp'],data['zpsys']) # handle model cov blowups saltcov = np.copy(saltcov) diag = np.copy(saltcov.diagonal()) model_snr = whocares/np.sqrt(diag) ind = np.where((np.abs(model_snr) < 1) & ~np.isnan(model_snr)) diag[ind] = diag[ind] * np.abs(model_snr[ind])**2 np.fill_diagonal(saltcov, diag) diagerr = np.diag(data['fluxerr']**2) fullcov = saltcov + diagerr try: np.linalg.cholesky(fullcov) except: print 'Cholesky failed... exiting' sys.exit() noise = np.random.multivariate_normal(np.zeros(len(diagerr)), fullcov, size=1)[0] #lower zp, lower flux data['flux'] = flux + noise data.meta['x1r'] = x1r data.meta['x1f'] = x1f data.meta['c'] = c data.meta['alpha_r'] = ar data.meta['alpha_f'] = af data.meta['beta'] = beta data.meta['MB'] = MB data.meta['mB'] = mu+MB data.meta['DayMax'] = t0 data.meta['cosmology'] = 'Planck15' data.rename_column('band', 'Filter') data.rename_column('time', 'Date') data.rename_column('flux', 'Flux') data.rename_column('fluxerr', 'Fluxerr') data.rename_column('zpsys', 'MagSys') data.rename_column('zp', 'ZP') if survey == 'SNLS': for row in data: tmp = row['Filter'] ind = tmp.find('MEGACAM') row['Filter'] = row['Filter'][ind:] sncosmo.write_lc(data,'./cadence_sim/lc/%s_%s.list' %(nickname, ctr), format='salt2') p.finish()
def download(url, dest_path, config, exceptions=True): """ Download a file from a URL to the given path or path name Parameters ---------- url : str The URL (including file name) to download dest_path : str The path (including file name) where the downloaded file should be saved config : configparser.ConfigParser Configuration options used to find custom paths if ``dest_path`` is a config option exceptions : bool, optional Whether to raise exceptions when the download fails Returns ------- dest_path : str The resulting file name if the download was successful, or None if not """ in_file_name = os.path.basename(urlparse(url).path) dest_path = os.path.abspath(dest_path) out_file_name = os.path.basename(dest_path) do_download = config.getboolean('download', 'download') check_size = config.getboolean('download', 'check_size') verify = config.getboolean('download', 'verify') if not do_download: if not os.path.exists(dest_path): raise OSError(f'File not found and downloading is disabled: ' f'{dest_path}') return dest_path if not check_size and os.path.exists(dest_path): return dest_path session = requests.Session() if not verify: session.verify = False # dest_path contains full path, so we need to make the relevant # subdirectories if they do not exist already directory = os.path.dirname(dest_path) try: os.makedirs(directory) except OSError: pass try: response = session.get(url, stream=True) total_size = response.headers.get('content-length') except requests.exceptions.RequestException: if exceptions: raise else: print(f' {url} could not be reached!') return None try: response.raise_for_status() except requests.exceptions.HTTPError as e: if exceptions: raise else: print(f'ERROR while downloading {in_file_name}:') print(e) return None if total_size is None: # no content length header if not os.path.exists(dest_path): dest_dir = os.path.dirname(dest_path) with open(dest_path, 'wb') as f: print(f'Downloading {in_file_name}\n' f' to {dest_dir}...') try: f.write(response.content) except requests.exceptions.RequestException: if exceptions: raise else: print(f' {in_file_name} failed!') return None else: print(' {in_file_name} done.') else: # we can do the download in chunks and use a progress bar, yay! total_size = int(total_size) if os.path.exists(dest_path) and \ total_size == os.path.getsize(dest_path): # we already have the file, so just return return dest_path if out_file_name == in_file_name: file_names = in_file_name else: file_names = f'{in_file_name} as {out_file_name}' dest_dir = os.path.dirname(dest_path) print(f'Downloading {file_names} ({_sizeof_fmt(total_size)})\n' f' to {dest_dir}') widgets = [ progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA() ] bar = progressbar.ProgressBar(widgets=widgets, max_value=total_size).start() size = 0 with open(dest_path, 'wb') as f: try: for data in response.iter_content(chunk_size=4096): size += len(data) f.write(data) bar.update(size) bar.finish() except requests.exceptions.RequestException: if exceptions: raise else: print(f' {in_file_name} failed!') return None else: print(f' {in_file_name} done.') return dest_path
line = '' elif c == '\n': # line is done yield line line = '' elif c == '': break else: line += c #def main(): first_update = None widgets = [ progressbar.Percentage(), None, progressbar.Bar(), progressbar.ETA(), None ] pbar = None debug = False for line in read_stdin(): if debug: print('Line on next line:') print(line) parts = line.split() ''' if len(parts) > 3: print('parts:', parts) print('len(parts):', len(parts)) print('should end w/ %:', parts[1][-1]) print('should start w/ to-check:', parts[-1]) print()