def get_user_permissions(profile_id: str) -> List[Dict]: """Get user assigned roles from DV360.""" service = _get_service() page_token = None user_profiles = [] # We need to use page token because max is 1000 results only. while True: response = service.accountUserProfiles().list( profileId=profile_id, pageToken=page_token, ).execute() account_user_profiles = response['accountUserProfiles'] for i, profile in enumerate(account_user_profiles): # If debug enabled, show the first 3 entries. if i < 3: if logging.get_verbosity() == 1: # Debug. logging.debug('Debugging enabled, showing first 3 entries.') pp = pprint.PrettyPrinter(indent=2) logging.debug(pp.pformat(profile)) user_profiles.append({ 'account_id': profile.get('accountId', 'NA'), 'subaccount_id': profile.get('subaccountId', 'NA'), 'email': util.hash_single(profile['email']), 'advertisers': ','.join(profile['advertiserFilter'].get('objectIds', [])), 'advertiser_status': profile['advertiserFilter']['status'], 'campaigns': ','.join(profile['campaignFilter'].get('objectIds', [])), 'campaign_status': profile['campaignFilter']['status'], }) page_token = response.get('nextPageToken') if not page_token: break return user_profiles
def test_grad_of_jit_compilation_caching(self): if not hasattr(self, "assertLogs"): raise unittest.SkipTest("test requires assertLogs (python 3)") lax.add(1, 2) # make sure some initial warnings are already printed sin = api.jit(np.sin) prev_level = logging.get_verbosity() try: logging.set_verbosity('DEBUG') with self.assertLogs(level=logging.DEBUG) as l: ans1 = api.grad(sin)(2.) ans2 = api.grad(sin)(3.) finally: logging.set_verbosity(prev_level) self.assertLen(l.output, 2) self.assertAllClose(ans1, onp.cos(2.), check_dtypes=False) self.assertAllClose(ans2, onp.cos(3.), check_dtypes=False)
def _list_functions_for_serialization(self, unused_serialization_cache): """Return a dict of `Function`s of a trackable.""" functions = {} for attribute_name in dir(self): # We get the attributes, suppressing warnings and exceptions. logging_verbosity = logging.get_verbosity() try: logging.set_verbosity(logging.FATAL) attribute_value = getattr(self, attribute_name, None) except Exception: # pylint: disable=broad-except # We really don't want to throw an exception just because some object's # attribute accessor is broken. attribute_value = None finally: # We reset the verbosity setting in a `finally` block, to make # sure it always happens, even if we make the exception catching above # be less broad. logging.set_verbosity(logging_verbosity) if isinstance(attribute_value, (def_function.Function, defun.ConcreteFunction)): functions[attribute_name] = attribute_value return functions
def StartTeeLogsToFile(program_name: str = None, log_dir: str = None, file_log_level: int = logging.DEBUG) -> None: """Log messages to file as well as stderr. Args: program_name: The name of the program. log_dir: The directory to log to. file_log_level: The minimum verbosity level to log to file to. Raises: FileNotFoundError: If the requested log_dir does not exist. """ if not pathlib.Path(log_dir).is_dir(): raise FileNotFoundError(f"Log directory not found: '{log_dir}'") old_verbosity = logging.get_verbosity() logging.set_verbosity(file_log_level) logging.set_stderrthreshold(old_verbosity) logging.get_absl_handler().start_logging_to_file(program_name, log_dir) # The Absl logging handler function start_logging_to_file() sets logtostderr # to False. Re-enable whatever value it was before the call. FLAGS.logtostderr = False
def update_report(profile_id: str, report: dict): """Update existing report. Updates an existing report. Args: profile_id: The profile id of the user owning the report. report: The report object. Returns: The report object of the updated report. """ service = _get_service() response = service.reports().update( profileId=profile_id, reportId=report['id'], body=report, ).execute() if logging.get_verbosity() == 1: # Debug. pp = pprint.PrettyPrinter(indent=2) logging.debug(pp.pformat(response)) logging.info('Updated report with id: %s for report name: %s', response['id'], report['name']) return response
def test_set_verbosity_strings(self): old_level = logging.get_verbosity() # Lowercase names. logging.set_verbosity('debug') self.assertEquals(logging.get_verbosity(), logging.DEBUG) logging.set_verbosity('info') self.assertEquals(logging.get_verbosity(), logging.INFO) logging.set_verbosity('warning') self.assertEquals(logging.get_verbosity(), logging.WARNING) logging.set_verbosity('warn') self.assertEquals(logging.get_verbosity(), logging.WARNING) logging.set_verbosity('error') self.assertEquals(logging.get_verbosity(), logging.ERROR) logging.set_verbosity('fatal') # Uppercase names. self.assertEquals(logging.get_verbosity(), logging.FATAL) logging.set_verbosity('DEBUG') self.assertEquals(logging.get_verbosity(), logging.DEBUG) logging.set_verbosity('INFO') self.assertEquals(logging.get_verbosity(), logging.INFO) logging.set_verbosity('WARNING') self.assertEquals(logging.get_verbosity(), logging.WARNING) logging.set_verbosity('WARN') self.assertEquals(logging.get_verbosity(), logging.WARNING) logging.set_verbosity('ERROR') self.assertEquals(logging.get_verbosity(), logging.ERROR) logging.set_verbosity('FATAL') self.assertEquals(logging.get_verbosity(), logging.FATAL) # Integers as strings. logging.set_verbosity(str(logging.DEBUG)) self.assertEquals(logging.get_verbosity(), logging.DEBUG) logging.set_verbosity(str(logging.INFO)) self.assertEquals(logging.get_verbosity(), logging.INFO) logging.set_verbosity(str(logging.WARNING)) self.assertEquals(logging.get_verbosity(), logging.WARNING) logging.set_verbosity(str(logging.ERROR)) self.assertEquals(logging.get_verbosity(), logging.ERROR) logging.set_verbosity(str(logging.FATAL)) self.assertEquals(logging.get_verbosity(), logging.FATAL) logging.set_verbosity(old_level)
def _discretization(self, displacement: tf.Tensor, current_tokens: tf.Tensor) -> tf.Tensor: r"""Converts a displacement vector to the ids of the best tokens. Discretizes the displacement vector into a score over tokens by using the method described in Hotflip. (Ebrahimi et al. 2018) https://arxiv.org/abs/1712.06751 \frac{\partial L}{\partial indices} &= \frac{\partial L}{\partial embeddings} x \frac{\partial embeddings}{\partial indices} &= \frac{\partial L}{\partial embeddings} x embeddings \frac{\partial L}{\partial embeddings} is our averaged displacement vectors, so we just need to multiply the displacement vectors with our embedding matrix. Args: displacement: The expected scaled displacement over sampled replacements. <float32>[batch_size, embedding_dimension] current_tokens: The indices of the tokens we are currently at. <int32>[batch_size, 1] Returns: A <int32>[batch_size] tensor containing the index of the best replacement token for each sentence in the batch. """ # Get dot-product similarity between displacement vectors and embeddings. # displacement is a matrix [batch_size, emb_dim] and embeddings is # [vocab_size, emb_dim]. embeddings needs to be transposed so that we have # [batch_size, emb_dim] @ [emb_dim, vocab_size] = [batch_size, vocab_size]. displacement_token_similarities = tf.matmul(displacement, self._embeddings, transpose_b=True) if self._discretize_by_cosine: displacement_token_similarities = self._normalize_by_magnitudes( displacement, displacement_token_similarities) # displacement_to_original_similarity is [batch_size, 1]. displacement_to_original_similarity = tf.gather( displacement_token_similarities, current_tokens, batch_dims=1, axis=-1) # score_diff is [batch_size, vocab_size] and lowers displacement # similarities by the similarity of the current tokens. score_diff = (displacement_token_similarities - displacement_to_original_similarity) if self._special_token_mask is not None: score_diff = tf.where(self._special_token_mask, float('-inf'), score_diff) # This sets scores for current tokens to -inf so they can't be predicted. score_diff = self.scatter_helper( score_diff, current_tokens, tf.fill((score_diff.shape[0], ), float('-inf'))) # Log the k top scoring replacement tokens. if (self._vocab and logging.get_verbosity() == logging.DEBUG and tf.executing_eagerly()): top_k_indices = tf.nn.top_k(score_diff, k=5)[1].numpy().tolist() top_k_tokens = [] for example in top_k_indices: top_k_tokens.append([self._vocab[index] for index in example]) logging.debug('Top K tokens in discretization:\n %s', top_k_tokens) # new_candidates are [batch_size,] and are vocab items most similar to # displacement vectors after subtracting the scores of the current tokens. new_candidates = tf.argmax(score_diff, axis=-1, output_type=tf.int32) # We need new candidates to be [batch_size, 1] for future scatter updates. return tf.expand_dims(new_candidates, -1)
def main(argv): import tensorflow as tf # need to be here to have the env variables defined tf.get_logger().propagate = False # masking error related to cache logger.getLogger('googleapiclient.discovery_cache').setLevel(logger.ERROR) # set level of verbosity if FLAGS.verbosity_level == 'DEBUG': logging.set_verbosity(logging.DEBUG) print('logging.DEBUG') elif FLAGS.verbosity_level == 'INFO': logging.set_verbosity(logging.INFO) elif FLAGS.verbosity_level == 'WARNING': logging.set_verbosity(logging.WARNING) elif FLAGS.verbosity_level == 'ERROR': logging.set_verbosity(logging.ERROR) elif FLAGS.verbosity_level == 'FATAL': logging.set_verbosity(logging.FATAL) else: logging.set_verbosity(logging.INFO) # set level of verbosity for Tensorflow if FLAGS.verbosity_level == 'VERBOSE': tf.debugging.set_log_device_placement(True) tf.autograph.set_verbosity(10, alsologtostdout=False) # logger.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR) # fmt = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s" fmt = "[%(levelname)s] %(message)s" formatter = logger.Formatter(fmt) logging.get_absl_handler().setFormatter(formatter) logging.get_absl_handler().python_handler.stream = sys.stdout logging.set_stderrthreshold(logging.WARNING) # level_log = 'INFO' # # Instantiates a client # client = google.cloud.logging.Client() # # # Connects the logger to the root logging handler; by default this captures # # all logs at INFO level and higher # client.setup_logging(log_level=FLAGS.verbosity) # # print('loggerDict:', logger.root.manager.loggerDict.keys()) # # for i in logger.root.manager.loggerDict.keys(): # if i=='tensorflow': # #print('-> propagate False') # logger.getLogger(i).propagate = False # needed # elif i=='google.auth': # logger.getLogger(i).propagate = False # needed # elif i=='google_auth_httplib2': # logger.getLogger(i).propagate = False # needed # elif i=='pyasn1': # logger.getLogger(i).propagate = False # needed # elif i=='sklearn': # logger.getLogger(i).propagate = False # needed # elif i=='google.cloud': # logger.getLogger(i).propagate = False # needed # else: # logger.getLogger(i).propagate = True # needed # handler = logger.getLogger(i).handlers # if handler != []: # #print("logger's name=", i,handler) # for h in handler: # #print(' -> ', h) # if h.__class__ == logger.StreamHandler: # #print(' -> name=', h.__class__) # h.setStream(sys.stdout) # h.setLevel(level_log) # #print(' --> handlers =', h) # root_logger = logger.getLogger() # root_logger.handlers=[handler for handler in root_logger.handlers if isinstance(handler, (CloudLoggingHandler, ContainerEngineHandler, logging.ABSLHandler))] # for handler in root_logger.handlers: print("----- handler ", handler) print("---------class ", handler.__class__) # if handler.__class__ == CloudLoggingHandler: # handler.setStream(sys.stdout) # handler.setLevel(level_log) # if handler.__class__ == logging.ABSLHandler: # handler.python_handler.stream = sys.stdout # handler.setLevel(level_log) # # handler.handler.setStream(sys.stdout) # # for handler in root_logger.handlers: # print("----- handler ", handler) # # # Instantiates a client # #client = google.cloud.logging.Client() # # # Connects the logger to the root logging handler; by default this captures # # all logs at INFO level and higher # #client.setup_logging() # # # redirect abseil logging messages to the stdout stream # #logging.get_absl_handler().python_handler.stream = sys.stdout # # # some test # #tf.get_logger().addHandler(logger.StreamHandler(sys.stdout)) # #tf.get_logger().disabled = True # #tf.autograph.set_verbosity(5 ,alsologtostdout=True) # # ## DEBUG # #fmt = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s" # #formatter = logger.Formatter(fmt) # #logging.get_absl_handler().setFormatter(formatter) # # # set level of verbosity # #logging.set_verbosity(logging.DEBUG) # # print(' 0 print --- ') # logging.info(' 1 logging:') # logging.info(' 2 logging:') # # print(' 3 print --- ') # logging.debug(' 4 logging-test-debug') # logging.info(' 5 logging-test-info') # logging.warning(' 6 logging-test-warning') # logging.error(' 7 logging test-error') # print(' 8 print --- ') # #_=BertTokenizer.from_pretrained('bert-base-uncased') # print(' 9 print --- ') # _= tf.distribute.MirroredStrategy() # print('10 print --- ') # ## DEBUG print('logging.get_verbosity()', logging.get_verbosity()) # print flags abseil_flags = [ 'logtostderr', 'alsologtostderr', 'log_dir', 'v', 'verbosity', 'stderrthreshold', 'showprefixforinfo', 'run_with_pdb', 'pdb_post_mortem', 'run_with_profiling', 'profile_file', 'use_cprofile_for_profiling', 'only_check_args', 'flagfile', 'undefok' ] logging.info('-- Custom flags:') for name in list(FLAGS): if name not in abseil_flags: logging.info('custom flags: {:40} with value: {:50}'.format( name, str(FLAGS[name].value))) logging.info('\n-- Abseil flags:') for name in list(FLAGS): if name in abseil_flags: logging.info('abseil flags: {:40} with value: {:50}'.format( name, str(FLAGS[name].value))) if os.environ.get('LOG_FILE_TO_WRITE') is not None: logging.info('os.environ[LOG_FILE_TO_WRITE]: {}'.format( os.environ['LOG_FILE_TO_WRITE'])) # split_path = os.environ['LOG_FILE_TO_WRITE'].split('/') # logging.get_absl_handler().use_absl_log_file(split_path[-1], '/'.join(split_path[:-1])) # fmt = "[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s" # formatter = logger.Formatter(fmt) # logging.get_absl_handler().setFormatter(formatter) # set level of verbosity # logging.set_verbosity(FLAGS.verbosity) # logging.set_stderrthreshold(FLAGS.verbosity) logging.info(tf.__version__) logging.info(tf.keras.__version__) logging.info(list(FLAGS)) logging.debug('flags: \n {}'.format(FLAGS)) logging.debug('env variables: \n{}'.format(os.environ)) logging.debug('current dir: {}'.format(os.getcwd())) logging.debug('__package__: {}'.format(__package__)) logging.debug('__name__: {}'.format(__name__)) logging.debug('__file__: {}'.format(__file__)) # only for HP tuning! if os.environ.get('CLOUD_ML_HP_METRIC_TAG') is not None: logging.info('this is a hyper parameters job !') # setup the hp flag FLAGS.is_hyperparameter_tuning = True logging.info('FLAGS.is_hyperparameter_tuning: {}'.format( FLAGS.is_hyperparameter_tuning)) logging.info('os.environ[CLOUD_ML_HP_METRIC_TAG]: {}'.format( os.environ['CLOUD_ML_HP_METRIC_TAG'])) logging.info('os.environ[CLOUD_ML_HP_METRIC_FILE]: {}'.format( os.environ['CLOUD_ML_HP_METRIC_FILE'])) logging.info('os.environ[CLOUD_ML_TRIAL_ID]: {}'.format( os.environ['CLOUD_ML_TRIAL_ID'])) # variable name for hyper parameter tuning metric_accuracy = os.environ['CLOUD_ML_HP_METRIC_TAG'] logging.info('metric accuracy name: {}'.format(metric_accuracy)) else: metric_accuracy = 'NotDefined' if os.environ.get('TF_CONFIG') is not None: logging.info('os.environ[TF_CONFIG]: {}'.format( os.environ['TF_CONFIG'])) else: logging.error('os.environ[TF_CONFIG] doesn\'t exist !') if FLAGS.use_tpu: # Check or update the TensorFlow on the TPU cluster to match the one of the VM logging.info( 'setting up TPU: check that TensorFlow version is the same on the VM and on the TPU cluster' ) client_tpu = Client() # define TPU strategy before any ops client_tpu.configure_tpu_version(tf.__version__, restart_type='ifNeeded') logging.info('setting up TPU: cluster resolver') tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( ) logging.info('setting up TPU: \n {}'.format(tpu_cluster_resolver)) logging.info('running on TPU: \n {}'.format( tpu_cluster_resolver.cluster_spec().as_dict()['worker'])) tf.config.experimental_connect_to_cluster(tpu_cluster_resolver) tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver) strategy = tf.distribute.experimental.TPUStrategy(tpu_cluster_resolver) else: strategy = tf.distribute.MirroredStrategy() print('do nothing') logging.info('Number of devices: {}'.format(strategy.num_replicas_in_sync)) # choose language's model and tokenizer MODELS = [(TFBertModel, BertTokenizer, 'bert-base-multilingual-uncased')] model_index = 0 # BERT # model_class = MODELS[model_index][0] # i.e TFBertModel # tokenizer_class = MODELS[model_index][1] # i.e BertTokenizer pretrained_weights = MODELS[model_index][ 2] # 'i.e bert-base-multilingual-uncased' # download pre trained model: if FLAGS.pretrained_model_dir: # download pre trained model from a bucket logging.info('downloading pretrained model!') search = re.search('gs://(.*?)/(.*)', FLAGS.pretrained_model_dir) if search is not None: bucket_name = search.group(1) blob_name = search.group(2) local_path = '.' mu.download_blob(bucket_name, blob_name, local_path) pretrained_model_dir = local_path + '/' + blob_name else: pretrained_model_dir = FLAGS.pretrained_model_dir else: # download pre trained model from internet pretrained_model_dir = '.' # some check logging.info('Batch size: {:6}/{:6}'.format( FLAGS.batch_size_train, FLAGS.batch_size_eval)) logging.info('Step per epoch: {:6}/{:6}'.format( FLAGS.steps_per_epoch_train, FLAGS.steps_per_epoch_eval)) logging.info('Total number of batch: {:6}/{:6}'.format( FLAGS.steps_per_epoch_train * (FLAGS.epochs + 1), FLAGS.steps_per_epoch_eval * 1)) # with tf.summary.create_file_writer(FLAGS.output_dir, # filename_suffix='.oup', # name='test').as_default(): # tf.summary.scalar('metric_accuracy', 1.0, step=1) # print('-- 00001') # read TFRecords files, shuffle, map and batch size train_dataset = tf_bert.build_dataset(FLAGS.input_train_tfrecords, FLAGS.batch_size_train, 2048) valid_dataset = tf_bert.build_dataset(FLAGS.input_eval_tfrecords, FLAGS.batch_size_eval, 2048) # set repeat train_dataset = train_dataset.repeat(FLAGS.epochs + 1) valid_dataset = valid_dataset.repeat(2) # reset all variables used by Keras tf.keras.backend.clear_session() # create and compile the Keras model in the context of strategy.scope with strategy.scope(): logging.debug('pretrained_model_dir={}'.format(pretrained_model_dir)) model = tf_bert.create_model(pretrained_weights, pretrained_model_dir=pretrained_model_dir, num_labels=FLAGS.num_classes, learning_rate=FLAGS.learning_rate, epsilon=FLAGS.epsilon) # train the model tf_bert.train_and_evaluate(model, num_epochs=FLAGS.epochs, steps_per_epoch=FLAGS.steps_per_epoch_train, train_data=train_dataset, validation_steps=FLAGS.steps_per_epoch_eval, eval_data=valid_dataset, output_dir=FLAGS.output_dir, n_steps_history=FLAGS.n_steps_history, FLAGS=FLAGS, decay_type=FLAGS.decay_type, learning_rate=FLAGS.learning_rate, s=FLAGS.decay_learning_rate, n_batch_decay=FLAGS.n_batch_decay, metric_accuracy=metric_accuracy)
file_log_level: int = logging.DEBUG, >>>>>>> 4242aed2a... Automated code format. ) -> None: """Log messages to file as well as stderr. Args: program_name: The name of the program. log_dir: The directory to log to. file_log_level: The minimum verbosity level to log to file to. Raises: FileNotFoundError: If the requested log_dir does not exist. """ if not pathlib.Path(log_dir).is_dir(): raise FileNotFoundError(f"Log directory not found: '{log_dir}'") old_verbosity = logging.get_verbosity() logging.set_verbosity(file_log_level) logging.set_stderrthreshold(old_verbosity) logging.get_absl_handler().start_logging_to_file(program_name, log_dir) # The Absl logging handler function start_logging_to_file() sets logtostderr # to False. Re-enable whatever value it was before the call. FLAGS.logtostderr = False def StopTeeLogsToFile(): """Stop logging messages to file as well as stderr.""" logging.get_absl_handler().flush() logging.get_absl_handler().stream = sys.stderr FLAGS.logtostderr = True
from __future__ import absolute_import from __future__ import division from __future__ import print_function import io import re from absl import logging from absl.testing import absltest import mock # We do this before our library imports in order to catch any Python stderr # output they may generate. We don't want them to; capture and check. fake_stderr_type = io.BytesIO if bytes is str else io.StringIO logging.get_verbosity() # Access --verbosity before flag parsing. # Access --logtostderr before flag parsing. logging.get_absl_handler().use_absl_log_file() class Error(Exception): pass # Pre-initialization (aka "import" / __main__ time) test. Checked below. with mock.patch('sys.stderr', new=fake_stderr_type()) as pre_init_mock_stderr: # Trigger the notice to stderr once. infos and above go to stderr. logging.debug('Debug message at parse time.') logging.info('Info message at parse time.') logging.error('Error message at parse time.') logging.warning('Warning message at parse time.')
def run_report_and_wait(profile_id: str, report_id: str) -> List: """Run the report and wait for it to complete and return the data. Runs the report and waits for it to complete and returns the data. Args: profile_id: The profile id of the account being used. report_id: The id of the report to run. Returns: The report data. """ service = _get_service() report_file = service.reports().run( profileId=profile_id, reportId=report_id, ).execute() if logging.get_verbosity() == 1: # Debug. pp = pprint.PrettyPrinter(indent=2) logging.debug(pp.pformat(report_file)) # Wait for report file to finish processing. # An exponential backoff strategy is used to conserve request quota. sleep = 0 start_time = time.time() while True: report_file = service.files().get( reportId=report_id, fileId=report_file['id']).execute() status = report_file['status'] if status == 'REPORT_AVAILABLE': logging.info('File status is %s, ready to download.', status) break elif status != 'PROCESSING': logging.info('File status is %s, processing failed.', status) raise ReportRunError elif time.time() - start_time > MAX_RETRY_ELAPSED_TIME: logging.info('File processing deadline exceeded.') raise ReportRunDeadlineExceeded sleep = _next_sleep_interval(sleep) logging.info('File status is %s, sleeping for %d seconds.', status, sleep) time.sleep(sleep) bytesio = io.BytesIO() # Create a get request. request = service.files().get_media( reportId=report_id, fileId=report_file['id']) # Create a media downloader instance. # Optional: adjust the chunk size used when downloading the file. downloader = http.MediaIoBaseDownload(bytesio, request, chunksize=CHUNK_SIZE) # Execute the get request and download the file. download_finished = False while download_finished is False: _, download_finished = downloader.next_chunk() csvreader = csv.reader(bytesio.getvalue().decode('utf-8').splitlines()) report_data = list(csvreader) report_data_cleaned = _clean_up(report_data) heading = report_data_cleaned[0] report_data_objects = [] for row in report_data_cleaned[1:]: report_data_objects.append({h: r for h, r in zip(heading, row)}) return report_data_objects