def __init__(self, database_settings): self.settings = database_settings log_furl = get_logger('HcpReader').handlers[0].stream.name set_logger('HcpDownloader', database_settings['LOGGING']['downloader_level'], log_furl) self.logger = get_logger('HcpDownloader')
def __init__(self, database_settings): self.base_path = database_settings['SERVERS']['dti_server_url'] self.local_path = database_settings['DIRECTORIES'][ 'local_server_directory'] log_furl = get_logger('HcpReader').handlers[0].stream.name set_logger('DtiDownloader', database_settings['LOGGING']['downloader_level'], log_furl) self.logger = get_logger('DtiDownloader')
def test_logger_log(caplog): logger = get_logger(__name__) logger.error("This is an error") logger.warning("This is a warning") logger.info("This is an informational message") logger.debug("This is a debug message") assert "This is an error" in caplog.text assert "This is a warning" in caplog.text assert "This is an informational message" not in caplog.text assert "This is a debug message" not in caplog.text
def __init__(self, settings, params): self.logger = get_logger('HcpDataset') self.local_folder = settings['DIRECTORIES']['local_server_directory'] self.parc = params['PARCELLATION']['parcellation'] self.inflation = params['SURFACE']['inflation'] self.tr = float(params['FMRI']['tr']) self.physio_sampling_rate = int(params['FMRI']['physio_sampling_rate']) self.regress_physio = params['FMRI']['regress_physio'] list_file = 'subjects_inter.txt' list_url = os.path.join(get_root(), 'conf', list_file) subjects_strut = load_subjects(list_url) structural_file = 'struct_dti.mat' structural_url = os.path.join(get_root(), 'conf', 'hcpdata', structural_file) self.S = load_strucutural(subjects_strut, structural_url)
def __init__(self, params, device, regime, coarsen=None): database_settings = get_database_settings() log_furl = os.path.join(params['FILE']['experiment_path'], 'log', 'downloader.log') set_logger('HcpDataset', database_settings['LOGGING']['dataloader_level'], log_furl) self.logger = get_logger('HcpDataset') self.logger.info('*** starting new {:} dataset'.format(regime)) self.device = device self.session = params['SESSION'][regime] self.reader = HcpReader(database_settings, params) list_url = os.path.join(params['FILE']['experiment_path'], 'conf', regime, self.session, 'subjects.txt') self.subjects = self.reader.load_subject_list(list_url) if coarsen is None: coarsen = TrivialCoarsening() self.coarsen = coarsen self.transform = SlidingWindow(params['TIME_SERIES'], coarsen=coarsen)
def __init__(self, database_settings, params): log_furl = os.path.join(params['FILE']['experiment_path'], 'log', 'downloader.log') set_logger('HcpReader', database_settings['LOGGING']['dataloader_level'], log_furl) self.logger = get_logger('HcpReader') self.local_folder = database_settings['DIRECTORIES'][ 'local_server_directory'] self.delete_nii = strtobool( database_settings['DIRECTORIES']['delete_after_downloading']) self.hcp_downloader = HcpDownloader(database_settings) self.dti_downloader = DtiDownloader(database_settings) nib.imageglobals.logger = set_logger( 'Nibabel', database_settings['LOGGING']['nibabel_level'], log_furl) self.parcellation = params['PARCELLATION']['parcellation'] self.inflation = params['SURFACE']['inflation'] self.tr = float(params['FMRI']['tr']) self.fh = float(params['FMRI']['physio_sampling_rate']) self.physio_sampling_rate = int(params['FMRI']['physio_sampling_rate']) self.regress_physio = strtobool(params['FMRI']['regress_physio'])
import requests from django.conf import settings from requests import RequestException from custom_auth.oauth.exceptions import GithubError from custom_auth.oauth.helpers.github_helpers import convert_response_data_to_dictionary, get_first_and_last_name from util.logging import get_logger GITHUB_API_URL = "https://api.github.com" GITHUB_OAUTH_URL = "https://github.com/login/oauth" NON_200_ERROR_MESSAGE = "Github returned {} status code when getting an access token." logger = get_logger(__name__) class GithubUser: def __init__(self, code: str, client_id: str = None, client_secret: str = None): self.client_id = client_id or settings.GITHUB_CLIENT_ID self.client_secret = client_secret or settings.GITHUB_CLIENT_SECRET self.access_token = self._get_access_token(code) self.headers = {"Authorization": f"token {self.access_token}"} def _get_access_token(self, code) -> str: data = { "code": code, "client_id": self.client_id,
def test_get_logger(): logger = get_logger(__name__) assert logger.level == logging.WARNING
def run_training(): """Train mnist for a number of steps.""" logger = logging.get_logger('evaluationLogger', FLAGS.log_dir, 'evaluation', logging.INFO) # Get the sets of units and labels for training, validation, and # test on mnist. data_sets = input_data.read_data_sets(FLAGS.input_data_dir, FLAGS.fake_data) # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default(): # Generate placeholders for the units and labels. units_placeholder, labels_placeholder = placeholder_inputs( FLAGS.batch_size) # Build a Graph that computes predictions from the inference model. logits = lezhin_comics.inference(units_placeholder, FLAGS.hidden_units, FLAGS.dropout) # Add to the Graph the Ops for loss calculation. loss = lezhin_comics.loss(logits, labels_placeholder) # Add to the Graph the Ops that calculate and apply gradients. train_op = lezhin_comics.training(loss, FLAGS.learning_rate) # Add the Op to compare the logits to the labels during evaluation. eval_correct = lezhin_comics.evaluation(logits, labels_placeholder) # Build the summary Tensor based on the TF collection of Summaries. # summary = tf.summary.merge_all() # Add the variable initializer Op. init = tf.global_variables_initializer() # Create a saver for writing training checkpoints. saver = tf.train.Saver() # Create a session for running Ops on the Graph. sess = tf.Session() # Instantiate a SummaryWriter to output summaries and the Graph. # summary_writer = tf.summary.FileWriter(FLAGS.log_dir, sess.graph) # And then after everything is built: # Run the Op to initialize the variables. sess.run(init) logger.info(" ".join([ 'step', 'loss_value', 'training_num_examples', 'training_true_count', 'training_precision', 'validation_num_examples', 'validation_true_count', 'validation_precision', 'test_num_examples', 'test_true_count', 'test_precision' ])) # Start the training loop. pre_training_precision = 0 for step in range(FLAGS.max_steps): # start_time = time.time() # Fill a feed dictionary with the actual set of units and labels # for this particular training step. feed_dict = fill_feed_dict(data_sets.train, units_placeholder, labels_placeholder) # Run one step of the model. The return values are the activations # from the `train_op` (which is discarded) and the `loss` Op. To # inspect the values of your Ops or variables, you may include them # in the list passed to sess.run() and the value tensors will be # returned in the tuple from the call. _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) # duration = time.time() - start_time # Save a checkpoint and evaluate the model periodically. if step % 10000 == 0 or (step + 1) == FLAGS.max_steps: checkpoint_file = os.path.join(FLAGS.log_dir, 'model.ckpt') saver.save(sess, checkpoint_file, global_step=step) # Evaluate against the training set. # print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration)) # print('Training Data Eval:') training_num_examples, training_true_count, training_precision = do_eval( sess, eval_correct, units_placeholder, labels_placeholder, data_sets.train) # Evaluate against the validation set. # print('Validation Data Eval:') validation_num_examples, validation_true_count, validation_precision = do_eval( sess, eval_correct, units_placeholder, labels_placeholder, data_sets.validation) # Evaluate against the test set. # print('Test Data Eval:') test_num_examples, test_true_count, test_precision = do_eval( sess, eval_correct, units_placeholder, labels_placeholder, data_sets.test) logger.info( "{:d} {:f} {:d} {:d} {:f} {:d} {:d} {:f} {:d} {:d} {:f}". format(step, loss_value, training_num_examples, training_true_count, training_precision, validation_num_examples, validation_true_count, validation_precision, test_num_examples, test_true_count, test_precision)) if pre_training_precision > training_precision: logger.info("training_precision is getting low.") break pre_training_precision = training_precision
def run_training(flags, class_number=1, label_managing_function=default_label_managing_function, is_profit=False): """Train mnist_example for a number of steps.""" logger = logging.get_logger('evaluationLogger', flags.log_dir, 'evaluation', logging.INFO) # Get the sets of units and labels for training, validation, and # test on mnist_example. # data_sets = read_data(file_name=flags.file_name, company=flags.company, label_name=flags.label_name, # columns=flags.columns,class_number=class_number, label_profit=label_profit, # test_rate=flags.test_rate, validation_rate=flags.validation_rate, shuffle=False) data_sets = read_data(file_name=flags.file_name, company=flags.company, label_name=flags.label_name, columns=flags.columns, class_number=class_number, label_profit=label_managing_function, test_rate=flags.test_rate, shuffle=False) data_sets = to_recurrent_data(data_sets, flags.time_step) logger.info("train\ttest") logger.info("{:d}\t{:d}".format(len(data_sets.train.units), len(data_sets.test.units))) logger.info("") # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default(): # Generate placeholders for the units and labels. units_placeholder, labels_placeholder = placeholder_inputs( data_sets.batch_size, flags.time_step, data_sets.column_number) # Build a Graph that computes predictions from the inference model. logits = inference(units_placeholder, flags.hidden_units, data_sets.column_number, data_sets.class_number, data_sets.batch_size, flags.dropout) # Add to the Graph the Ops for loss calculation. loss = do_loss(logits, labels_placeholder) # Add to the Graph the Ops that calculate and apply gradients. train_op = training(loss, flags.learning_rate) # Add the Op to compare the logits to the labels during evaluation. eval_correct = evaluation(logits, labels_placeholder) # Add the variable initializer Op. init = tf.global_variables_initializer() # Create a saver for writing training checkpoints. saver = tf.train.Saver() # Create a session for running Ops on the Graph. sess = tf.Session() # Run the Op to initialize the variables. sess.run(init) logger.info("\t".join(['learning_rate', 'max_steps', 'hidden_units'])) logger.info("{:f}\t{:d}\t{}".format(flags.learning_rate, flags.max_steps, flags.hidden_units)) logger.info("") logger.info(" ".join( ['step', 'loss_value', 'training_precision', 'test_precision'])) # Start the training loop. for step in range(flags.max_steps + 1): # Fill a feed dictionary with the actual set of units and labels # for this particular training step. feed_dict, labels_feed, dates_feed = fill_feed_dict( data_sets.train, units_placeholder, labels_placeholder, flags) # Run one step of the model. The return values are the activations # from the `train_op` (which is discarded) and the `loss` Op. To # inspect the values of your Ops or variables, you may include them # in the list passed to sess.run() and the value tensors will be # returned in the tuple from the call. _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) # Save a checkpoint and evaluate the model periodically. if step % (flags.max_steps / 2) == 0: checkpoint_file = os.path.join(flags.log_dir, 'model.ckpt') saver.save(sess, checkpoint_file, global_step=step) # Evaluate against the training set. training_average_squared_error = do_eval( sess, eval_correct, units_placeholder, labels_placeholder, data_sets.train, flags) # Evaluate against the test set. test_average_squared_error = do_eval(sess, eval_correct, units_placeholder, labels_placeholder, data_sets.test, flags) logger.info("{:d}\t{:f}\t{:f}\t{:f}".format( step, loss_value, training_average_squared_error, test_average_squared_error)) # Compare labels(targets) with predictions. feed_dict, labels_feed, dates_feed = fill_feed_dict( data_sets.test, units_placeholder, labels_placeholder, flags) test_predictions = sess.run(logits, feed_dict) average_squared_error = sess.run( eval_correct, feed_dict=feed_dict) / data_sets.batch_size y_label = 'Result' if is_profit: labels_feed = np.array(labels_feed) + 1 labels_feed = labels_feed.cumprod() labels_feed = labels_feed - 1 test_predictions = np.array(test_predictions) + 1 test_predictions = test_predictions.cumprod() test_predictions = test_predictions - 1 y_label = 'Profit' # draw a test graph matplotlib.rc('font', family='NanumBarunGothicOTF') fig, ax = plt.subplots() ax.plot(dates_feed, labels_feed, 'r', label='target') ax.plot(dates_feed, test_predictions, 'b', label='prediction, {:4.3f}'.format(average_squared_error)) ax.legend() title = flags.file_name if flags.company is not None: title = title + ", " + flags.company plt.title(title) plt.xlabel('Time Period') plt.ylabel(y_label) plt.show()
def run_training(flags, class_number, label_profit): """Train mnist_example for a number of steps.""" logger = logging.get_logger('evaluationLogger', flags.log_dir, 'evaluation', logging.INFO) # Get the data_sets of units and labels for training, and test. data_sets = read_data(file_name=flags.file_name, company=flags.company, label_name=flags.label_name, columns=flags.columns, class_number=class_number, label_profit=label_profit, test_rate=flags.test_rate, shuffle=False) data_sets = to_recurrent_data(data_sets, flags.time_step) # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default(): # Generate placeholders for the units and labels. units_placeholder, labels_placeholder = placeholder_inputs( data_sets.batch_size, flags.time_step, data_sets.column_number) # Build a Graph that computes predictions from the inference model. logits = inference(units_placeholder, flags.hidden_units, data_sets.column_number, data_sets.class_number, data_sets.batch_size, flags.dropout) # Add to the Graph the Ops for loss calculation. loss = do_loss(logits, labels_placeholder) # Add to the Graph the Ops that calculate and apply gradients. train_op = training(loss, flags.learning_rate) # Add the Op to compare the logits to the labels during evaluation. eval_correct = evaluation(logits, labels_placeholder) # Add the variable initializer Op. init = tf.global_variables_initializer() # Create a saver for writing training checkpoints. saver = tf.train.Saver() # Create a session for running Ops on the Graph. sess = tf.Session() # Run the Op to initialize the variables. sess.run(init) logger.info("\t".join(['learning_rate', 'max_steps', 'hidden_units'])) logger.info("{:f}\t{:d}\t{}".format(flags.learning_rate, flags.max_steps, flags.hidden_units)) logger.info("") logger.info(" ".join( ['step', 'loss_value', 'training_SSE', 'test_SSE'])) # Start the training loop. for step in range(flags.max_steps + 1): # Fill a feed dictionary with the actual set of units and labels # for this particular training step. feed_dict = fill_feed_dict(data_sets.train, units_placeholder, labels_placeholder, flags) # Run one step of the model. The return values are the activations # from the `train_op` (which is discarded) and the `loss` Op. To # inspect the values of your Ops or variables, you may include them # in the list passed to sess.run() and the value tensors will be # returned in the tuple from the call. _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) # Save a checkpoint and evaluate the model periodically. if step % (flags.max_steps / 100) == 0: checkpoint_file = os.path.join(flags.log_dir, 'model.ckpt') saver.save(sess, checkpoint_file, global_step=step) # Evaluate against the training set. training_num_examples, training_true_count, training_precision = do_eval( sess, eval_correct, units_placeholder, labels_placeholder, data_sets.train, flags) # Evaluate against the test set. test_num_examples, test_true_count, test_precision = do_eval( sess, eval_correct, units_placeholder, labels_placeholder, data_sets.test, flags) logger.info("{:d}\t{:f}\t{:f}\t{:f}".format( step, loss_value, training_precision, test_precision))