Exemple #1
0
    def _PrintInfoAboutBucketListingRef(self, bucket_listing_ref):
        """Print listing info for given bucket_listing_ref.

    Args:
      bucket_listing_ref: BucketListing being listed.

    Returns:
      Tuple (number of objects, object size)

    Raises:
      Exception: if calling bug encountered.
    """
        obj = bucket_listing_ref.root_object
        url_str = bucket_listing_ref.url_string
        if (obj.metadata and S3_DELETE_MARKER_GUID
                in obj.metadata.additionalProperties):
            size_string = '0'
            num_bytes = 0
            num_objs = 0
            url_str += '<DeleteMarker>'
        else:
            size_string = (MakeHumanReadable(obj.size)
                           if self.human_readable else str(obj.size))
            num_bytes = obj.size
            num_objs = 1

        if not self.summary_only:
            url_detail = '{size:<11}  {url}{ending}'.format(
                size=size_string,
                url=six.ensure_text(url_str),
                ending=six.ensure_text(self.line_ending))
            print_to_fd(url_detail, file=sys.stdout, end='')

        return (num_objs, num_bytes)
Exemple #2
0
  def _EncryptionSetKey(self, bucket_metadata, bucket_url,
                        svc_acct_for_project_num):
    """Sets defaultKmsKeyName on a Cloud Storage bucket.

    Args:
      bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket.
      bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket.
      svc_acct_for_project_num: (Dict[int, str]) Mapping of project numbers to
          their corresponding service account.
    """
    bucket_project_number = bucket_metadata.projectNumber
    try:
      # newly_authorized will always be False if the project number is in our
      # cache dict, since we've already called _AuthorizeProject on it.
      service_account, newly_authorized = (
          svc_acct_for_project_num[bucket_project_number], False)
    except KeyError:
      service_account, newly_authorized = self._AuthorizeProject(
          bucket_project_number, self.kms_key)
      svc_acct_for_project_num[bucket_project_number] = service_account
    if newly_authorized:
      text_util.print_to_fd('Authorized service account %s to use key:\n%s' %
                            (service_account, self.kms_key))

    bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue(
        defaultKmsKeyName=self.kms_key)
    print('Setting default KMS key for bucket %s...' %
          str(bucket_url).rstrip('/'))
    self.gsutil_api.PatchBucket(bucket_url.bucket_name,
                                bucket_metadata,
                                fields=['encryption'],
                                provider=bucket_url.scheme)
Exemple #3
0
def PrintDir(bucket_listing_ref):
    """Default function for printing buckets or prefixes.

  Args:
    bucket_listing_ref: BucketListingRef of type BUCKET or PREFIX.
  """
    text_util.print_to_fd(bucket_listing_ref.url_string)
Exemple #4
0
  def _PrintLongListing(self, bucket_listing_ref):
    """Prints an object with ListingStyle.LONG."""
    obj = bucket_listing_ref.root_object
    url_str = bucket_listing_ref.url_string
    if (obj.metadata and
        S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties):
      size_string = '0'
      num_bytes = 0
      num_objs = 0
      url_str += '<DeleteMarker>'
    else:
      size_string = (MakeHumanReadable(obj.size)
                     if self.human_readable else str(obj.size))
      num_bytes = obj.size
      num_objs = 1

    timestamp = JSON_TIMESTAMP_RE.sub(r'\1T\2Z', str(obj.timeCreated))
    printstr = '%(size)10s  %(timestamp)s  %(url)s'
    encoded_etag = None
    encoded_metagen = None
    if self.all_versions:
      printstr += '  metageneration=%(metageneration)s'
      encoded_metagen = str(obj.metageneration)
    if self.include_etag:
      printstr += '  etag=%(etag)s'
      encoded_etag = obj.etag
    format_args = {
        'size': size_string,
        'timestamp': timestamp,
        'url': url_str,
        'metageneration': encoded_metagen,
        'etag': encoded_etag
    }
    text_util.print_to_fd(printstr % format_args)
    return (num_objs, num_bytes)
Exemple #5
0
  def _OutputHelp(self, help_str):
    """Outputs simply formatted string.

    This function paginates if the string is too long, PAGER is defined, and
    the output is a tty.

    Args:
      help_str: String to format.
    """
    # Remove <B> and </B> tags and replace them with ANSI control codes if
    # writing to a compatible tty.
    if IS_WINDOWS or not IsRunningInteractively():
      help_str = re.sub('<B>', '', help_str)
      help_str = re.sub('</B>', '', help_str)
      text_util.print_to_fd(help_str)
      return
    help_str = re.sub('<B>', '\033[1m', help_str)
    help_str = re.sub('</B>', '\033[0;0m', help_str)
    num_lines = len(help_str.split('\n'))
    if 'PAGER' in os.environ and num_lines >= GetTermLines():
      # Use -r option for less to make bolding work right.
      pager = os.environ['PAGER'].split(' ')
      if pager[0].endswith('less'):
        pager.append('-r')
      try:
        Popen(pager, stdin=PIPE,
              universal_newlines=True).communicate(input=help_str)
      except OSError as e:
        raise CommandException('Unable to open pager (%s): %s' %
                               (' '.join(pager), e))
    else:
      text_util.print_to_fd(help_str)
Exemple #6
0
    def _PrintLongListing(self, bucket_listing_ref):
        """Prints an object with ListingStyle.LONG."""
        obj = bucket_listing_ref.root_object
        url_str = bucket_listing_ref.url_string
        if (obj.metadata and S3_DELETE_MARKER_GUID
                in obj.metadata.additionalProperties):
            size_string = '0'
            num_bytes = 0
            num_objs = 0
            url_str += '<DeleteMarker>'
        else:
            size_string = (MakeHumanReadable(obj.size)
                           if self.human_readable else str(obj.size))
            num_bytes = obj.size
            num_objs = 1

        timestamp = JSON_TIMESTAMP_RE.sub(r'\1T\2Z', str(obj.timeCreated))
        printstr = '%(size)10s  %(timestamp)s  %(url)s'
        encoded_etag = None
        encoded_metagen = None
        if self.all_versions:
            printstr += '  metageneration=%(metageneration)s'
            encoded_metagen = str(obj.metageneration)
        if self.include_etag:
            printstr += '  etag=%(etag)s'
            encoded_etag = obj.etag
        format_args = {
            'size': size_string,
            'timestamp': timestamp,
            'url': url_str,
            'metageneration': encoded_metagen,
            'etag': encoded_etag
        }
        text_util.print_to_fd(printstr % format_args)
        return (num_objs, num_bytes)
Exemple #7
0
  def _PrintInfoAboutBucketListingRef(self, bucket_listing_ref):
    """Print listing info for given bucket_listing_ref.

    Args:
      bucket_listing_ref: BucketListing being listed.

    Returns:
      Tuple (number of objects, object size)

    Raises:
      Exception: if calling bug encountered.
    """
    obj = bucket_listing_ref.root_object
    url_str = bucket_listing_ref.url_string
    if (obj.metadata and
        S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties):
      size_string = '0'
      num_bytes = 0
      num_objs = 0
      url_str += '<DeleteMarker>'
    else:
      size_string = (MakeHumanReadable(obj.size)
                     if self.human_readable else str(obj.size))
      num_bytes = obj.size
      num_objs = 1

    if not self.summary_only:
      url_detail = '{size:<11}  {url}{ending}'.format(
          size=size_string,
          url=six.ensure_text(url_str),
          ending=six.ensure_text(self.line_ending))
      print_to_fd(url_detail, file=sys.stdout, end='')

    return (num_objs, num_bytes)
def PrintDir(bucket_listing_ref):
  """Default function for printing buckets or prefixes.

  Args:
    bucket_listing_ref: BucketListingRef of type BUCKET or PREFIX.
  """
  text_util.print_to_fd(bucket_listing_ref.url_string)
def PrintDirHeader(bucket_listing_ref):
  """Default function for printing headers for prefixes.

  Header is printed prior to listing the contents of the prefix.

  Args:
    bucket_listing_ref: BucketListingRef of type PREFIX.
  """
  text_util.print_to_fd('{}:'.format(bucket_listing_ref.url_string))
Exemple #10
0
def PrintDirHeader(bucket_listing_ref):
    """Default function for printing headers for prefixes.

  Header is printed prior to listing the contents of the prefix.

  Args:
    bucket_listing_ref: BucketListingRef of type PREFIX.
  """
    text_util.print_to_fd('{}:'.format(bucket_listing_ref.url_string))
Exemple #11
0
    def _AuthorizeProject(self, project_id, kms_key):
        """Authorizes a project's service account to be used with a KMS key.

    Authorizes the Cloud Storage-owned service account for project_id to be used
    with kms_key.

    Args:
      project_id: (str) Project id string (not number).
      kms_key: (str) Fully qualified resource name for the KMS key.

    Returns:
      (str, bool) A 2-tuple consisting of:
      1) The email address for the service account associated with the project,
         which is authorized to encrypt/decrypt with the specified key.
      2) A bool value - True if we had to grant the service account permission
         to encrypt/decrypt with the given key; False if the required permission
         was already present.
    """
        # Request the Cloud Storage-owned service account for project_id, creating
        # it if it does not exist.
        service_account = self.gsutil_api.GetProjectServiceAccount(
            project_id, provider='gs').email_address

        kms_api = KmsApi(logger=self.logger)
        self.logger.debug('Getting IAM policy for %s', kms_key)
        try:
            policy = kms_api.GetKeyIamPolicy(kms_key)
            self.logger.debug('Current policy is %s', policy)

            # Check if the required binding is already present; if not, add it and
            # update the key's IAM policy.
            added_new_binding = False
            binding = Binding(
                role='roles/cloudkms.cryptoKeyEncrypterDecrypter',
                members=['serviceAccount:%s' % service_account])
            if binding not in policy.bindings:
                policy.bindings.append(binding)
                kms_api.SetKeyIamPolicy(kms_key, policy)
                added_new_binding = True
            return (service_account, added_new_binding)
        except AccessDeniedException:
            if self.warn_on_key_authorize_failure:
                text_util.print_to_fd('\n'.join(
                    textwrap.wrap(
                        'Warning: Unable to check the IAM policy for the specified '
                        'encryption key. Check that your Cloud Platform project\'s '
                        'service account has the '
                        '"cloudkms.cryptoKeyEncrypterDecrypter" role for the '
                        'specified key. Without this role, you may not be able to '
                        'encrypt or decrypt objects using the key which will '
                        'prevent you from uploading or downloading objects.')))
                return (service_account, False)
            else:
                raise
Exemple #12
0
def PrintObject(bucket_listing_ref):
    """Default printing function for objects.

  Args:
    bucket_listing_ref: BucketListingRef of type OBJECT.

  Returns:
    (num_objects, num_bytes).
  """
    try:
        text_util.print_to_fd(bucket_listing_ref.url_string)
    except IOError as e:
        # Windows throws an IOError 0 here for object names containing Unicode
        # chars. Ignore it.
        if not (IS_WINDOWS and e.errno == 0):
            raise
    return (1, 0)
def PrintObject(bucket_listing_ref):
  """Default printing function for objects.

  Args:
    bucket_listing_ref: BucketListingRef of type OBJECT.

  Returns:
    (num_objects, num_bytes).
  """
  try:
    text_util.print_to_fd(bucket_listing_ref.url_string)
  except IOError as e:
    # Windows throws an IOError 0 here for object names containing Unicode
    # chars. Ignore it.
    if not (IS_WINDOWS and e.errno == 0):
      raise
  return (1, 0)
Exemple #14
0
  def MaybePromptForPythonUpdate(self, command_name):
    """Alert the user that they should install Python 3.

    Args:
      command_name: The name of the command being run.

    Returns:
      True if a prompt was output.
    """
    if (sys.version_info.major != 2 or self.SkipUpdateCheck() or
        command_name not in ('update', 'ver', 'version') or
        boto.config.getbool('GSUtil', 'skip_python_update_prompt', False)):
      return False

    # Notify the user about Python 2 deprecation.
    print_to_fd(
        'Gsutil 5 drops Python 2 support. Please install Python 3 to update '
        'to the latest version of gsutil. https://goo.gle/py3\n')
    return True
def _OutputAndExit(message, exception=None):
  """Outputs message to stderr and exits gsutil with code 1.

  This function should only be called in single-process, single-threaded mode.

  Args:
    message: Message to print to stderr.
    exception: The exception that caused gsutil to fail.
  """
  if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS or test_exception_traces:
    stack_trace = traceback.format_exc()
    err = ('DEBUG: Exception stack trace:\n    %s\n%s\n' %
           (re.sub('\\n', '\n    ', stack_trace), message))
  else:
    err = '%s\n' % message
  try:
    text_util.print_to_fd(err, end='', file=sys.stderr)
  except UnicodeDecodeError:
    # Can happen when outputting invalid Unicode filenames.
    sys.stderr.write(err)
  if exception:
    metrics.LogFatalError(exception)
  sys.exit(1)
Exemple #16
0
def _OutputAndExit(message, exception=None):
  """Outputs message to stderr and exits gsutil with code 1.

  This function should only be called in single-process, single-threaded mode.

  Args:
    message: Message to print to stderr.
    exception: The exception that caused gsutil to fail.
  """
  if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS or test_exception_traces:
    stack_trace = traceback.format_exc()
    err = ('DEBUG: Exception stack trace:\n    %s\n%s\n' %
           (re.sub('\\n', '\n    ', stack_trace), message))
  else:
    err = '%s\n' % message
  try:
    text_util.print_to_fd(err, end='', file=sys.stderr)
  except UnicodeDecodeError:
    # Can happen when outputting invalid Unicode filenames.
    sys.stderr.write(err)
  if exception:
    metrics.LogFatalError(exception)
  sys.exit(1)
  def _Get(self):
    """Gets logging configuration for a bucket."""
    bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
        self.args[0], bucket_fields=['logging'])

    if bucket_url.scheme == 's3':
      text_util.print_to_fd(self.gsutil_api.XmlPassThroughGetLogging(
          bucket_url, provider=bucket_url.scheme),
                            end='')
    else:
      if (bucket_metadata.logging and bucket_metadata.logging.logBucket and
          bucket_metadata.logging.logObjectPrefix):
        text_util.print_to_fd(
            str(encoding.MessageToJson(bucket_metadata.logging)))
      else:
        text_util.print_to_fd('%s has no logging configuration.' % bucket_url)
    return 0
Exemple #18
0
 def _PrintSummaryLine(self, num_bytes, name):
   size_string = (MakeHumanReadable(num_bytes)
                  if self.human_readable else six.text_type(num_bytes))
   text_util.print_to_fd('{size:<11}  {name}'.format(
       size=size_string, name=six.ensure_text(name)),
                         end=self.line_ending)
Exemple #19
0
    def _PrintBucketInfo(self, bucket_blr, listing_style):
        """Print listing info for given bucket.

    Args:
      bucket_blr: BucketListingReference for the bucket being listed
      listing_style: ListingStyle enum describing type of output desired.

    Returns:
      Tuple (total objects, total bytes) in the bucket.
    """
        if (listing_style == ListingStyle.SHORT
                or listing_style == ListingStyle.LONG):
            text_util.print_to_fd(bucket_blr)
            return
        # listing_style == ListingStyle.LONG_LONG:
        # We're guaranteed by the caller that the root object is populated.
        bucket = bucket_blr.root_object
        location_constraint = bucket.location
        storage_class = bucket.storageClass
        fields = {
            'bucket': bucket_blr.url_string,
            'storage_class': storage_class,
            'location_constraint': location_constraint,
            'acl': AclTranslation.JsonFromMessage(bucket.acl),
            'default_acl':
            AclTranslation.JsonFromMessage(bucket.defaultObjectAcl),
            'versioning': bucket.versioning and bucket.versioning.enabled,
            'website_config': 'Present' if bucket.website else 'None',
            'logging_config': 'Present' if bucket.logging else 'None',
            'cors_config': 'Present' if bucket.cors else 'None',
            'lifecycle_config': 'Present' if bucket.lifecycle else 'None',
            'requester_pays': bucket.billing and bucket.billing.requesterPays
        }
        if bucket.retentionPolicy:
            fields['retention_policy'] = 'Present'
        if bucket.labels:
            fields['labels'] = LabelTranslation.JsonFromMessage(
                bucket.labels, pretty_print=True)
        else:
            fields['labels'] = 'None'
        if bucket.encryption and bucket.encryption.defaultKmsKeyName:
            fields['default_kms_key'] = bucket.encryption.defaultKmsKeyName
        else:
            fields['default_kms_key'] = 'None'
        fields[
            'encryption_config'] = 'Present' if bucket.encryption else 'None'
        # Fields not available in all APIs (e.g. the XML API)
        if bucket.locationType:
            fields['location_type'] = bucket.locationType
        if bucket.metageneration:
            fields['metageneration'] = bucket.metageneration
        if bucket.timeCreated:
            fields['time_created'] = bucket.timeCreated.strftime(
                '%a, %d %b %Y %H:%M:%S GMT')
        if bucket.updated:
            fields['updated'] = bucket.updated.strftime(
                '%a, %d %b %Y %H:%M:%S GMT')
        if bucket.defaultEventBasedHold:
            fields['default_eventbased_hold'] = bucket.defaultEventBasedHold
        if bucket.iamConfiguration and bucket.iamConfiguration.bucketPolicyOnly:
            enabled = bucket.iamConfiguration.bucketPolicyOnly.enabled
            fields['bucket_policy_only_enabled'] = enabled

        # For field values that are multiline, add indenting to make it look
        # prettier.
        for key in fields:
            previous_value = fields[key]
            if (not isinstance(previous_value, six.string_types)
                    or '\n' not in previous_value):
                continue
            new_value = previous_value.replace('\n', '\n\t  ')
            # Start multiline values on a new line if they aren't already.
            if not new_value.startswith('\n'):
                new_value = '\n\t  ' + new_value
            fields[key] = new_value

        # Only display certain properties if the given API returned them (JSON API
        # returns many fields that the XML API does not).
        location_type_line = ''
        metageneration_line = ''
        time_created_line = ''
        time_updated_line = ''
        default_eventbased_hold_line = ''
        retention_policy_line = ''
        bucket_policy_only_enabled_line = ''
        if 'location_type' in fields:
            location_type_line = '\tLocation type:\t\t\t{location_type}\n'
        if 'metageneration' in fields:
            metageneration_line = '\tMetageneration:\t\t\t{metageneration}\n'
        if 'time_created' in fields:
            time_created_line = '\tTime created:\t\t\t{time_created}\n'
        if 'updated' in fields:
            time_updated_line = '\tTime updated:\t\t\t{updated}\n'
        if 'default_eventbased_hold' in fields:
            default_eventbased_hold_line = (
                '\tDefault Event-Based Hold:\t{default_eventbased_hold}\n')
        if 'retention_policy' in fields:
            retention_policy_line = '\tRetention Policy:\t\t{retention_policy}\n'
        if 'bucket_policy_only_enabled' in fields:
            bucket_policy_only_enabled_line = (
                '\tBucket Policy Only enabled:\t'
                '{bucket_policy_only_enabled}\n')

        text_util.print_to_fd(
            ('{bucket} :\n'
             '\tStorage class:\t\t\t{storage_class}\n' + location_type_line +
             '\tLocation constraint:\t\t{location_constraint}\n'
             '\tVersioning enabled:\t\t{versioning}\n'
             '\tLogging configuration:\t\t{logging_config}\n'
             '\tWebsite configuration:\t\t{website_config}\n'
             '\tCORS configuration: \t\t{cors_config}\n'
             '\tLifecycle configuration:\t{lifecycle_config}\n'
             '\tRequester Pays enabled:\t\t{requester_pays}\n' +
             retention_policy_line + default_eventbased_hold_line +
             '\tLabels:\t\t\t\t{labels}\n' +
             '\tDefault KMS key:\t\t{default_kms_key}\n' + time_created_line +
             time_updated_line + metageneration_line +
             bucket_policy_only_enabled_line + '\tACL:\t\t\t\t{acl}\n'
             '\tDefault ACL:\t\t\t{default_acl}').format(**fields))
        if bucket_blr.storage_url.scheme == 's3':
            text_util.print_to_fd(
                'Note: this is an S3 bucket so configuration values may be '
                'blank. To retrieve bucket configuration values, use '
                'individual configuration commands such as gsutil acl get '
                '<bucket>.')
Exemple #20
0
  def RunCommand(self):
    """Command entry point for the hash command."""
    (calc_crc32c, calc_md5, format_func, cloud_format_func,
     output_format) = (self._ParseOpts(self.sub_opts, self.logger))

    matched_one = False
    for url_str in self.args:
      for file_ref in self.WildcardIterator(url_str).IterObjects(
          bucket_listing_fields=[
              'crc32c',
              'customerEncryption',
              'md5Hash',
              'size',
          ]):
        matched_one = True
        url = StorageUrlFromString(url_str)
        file_name = file_ref.storage_url.object_name
        if StorageUrlFromString(url_str).IsFileUrl():
          file_size = os.path.getsize(file_name)
          self.gsutil_api.status_queue.put(
              FileMessage(url,
                          None,
                          time.time(),
                          size=file_size,
                          finished=False,
                          message_type=FileMessage.FILE_HASH))
          callback_processor = ProgressCallbackWithTimeout(
              file_size,
              FileProgressCallbackHandler(self.gsutil_api.status_queue,
                                          src_url=StorageUrlFromString(url_str),
                                          operation_name='Hashing').call)
          hash_dict = self._GetHashClassesFromArgs(calc_crc32c, calc_md5)
          with open(file_name, 'rb') as fp:
            hashing_helper.CalculateHashesFromContents(
                fp, hash_dict, callback_processor=callback_processor)
          self.gsutil_api.status_queue.put(
              FileMessage(url,
                          None,
                          time.time(),
                          size=file_size,
                          finished=True,
                          message_type=FileMessage.FILE_HASH))
        else:
          hash_dict = {}
          obj_metadata = file_ref.root_object
          file_size = obj_metadata.size
          md5_present = obj_metadata.md5Hash is not None
          crc32c_present = obj_metadata.crc32c is not None
          if not md5_present and not crc32c_present:
            logging.getLogger().warn('No hashes present for %s', url_str)
            continue
          if md5_present:
            hash_dict['md5'] = obj_metadata.md5Hash
          if crc32c_present:
            hash_dict['crc32c'] = obj_metadata.crc32c
        text_util.print_to_fd('Hashes [%s] for %s:' %
                              (output_format, file_name))
        for name, digest in six.iteritems(hash_dict):
          text_util.print_to_fd('\tHash (%s):\t\t%s' %
                                (name, (format_func(digest) if url.IsFileUrl()
                                        else cloud_format_func(digest))))

    if not matched_one:
      raise CommandException('No files matched')
    _PutToQueueWithTimeout(self.gsutil_api.status_queue,
                           FinalMessage(time.time()))
    return 0
Exemple #21
0
def PrintNewLine():
    """Default function for printing new lines between directories."""
    text_util.print_to_fd()
    def RunCommand(self,
                   command_name,
                   args=None,
                   headers=None,
                   debug=0,
                   return_stdout=False,
                   return_stderr=False,
                   return_log_handler=False,
                   cwd=None):
        """Method for calling gslib.command_runner.CommandRunner.

    Passes parallel_operations=False for all tests, optionally saving/returning
    stdout output. We run all tests multi-threaded, to exercise those more
    complicated code paths.
    TODO: Change to run with parallel_operations=True for all tests. At
    present when you do this it causes many test failures.

    Args:
      command_name: The name of the command being run.
      args: Command-line args (arg0 = actual arg, not command name ala bash).
      headers: Dictionary containing optional HTTP headers to pass to boto.
      debug: Debug level to pass in to boto connection (range 0..3).
      return_stdout: If True, will save and return stdout produced by command.
      return_stderr: If True, will save and return stderr produced by command.
      return_log_handler: If True, will return a MockLoggingHandler instance
           that was attached to the command's logger while running.
      cwd: The working directory that should be switched to before running the
           command. The working directory will be reset back to its original
           value after running the command. If not specified, the working
           directory is left unchanged.

    Returns:
      One or a tuple of requested return values, depending on whether
      return_stdout, return_stderr, and/or return_log_handler were specified.
      Return Types:
        stdout - str (binary in Py2, text in Py3)
        stderr - str (binary in Py2, text in Py3)
        log_handler - MockLoggingHandler
    """
        args = args or []

        command_line = six.ensure_text(' '.join([command_name] + args))
        if self.is_debugging:
            print_to_fd('\nRunCommand of {}\n'.format(command_line),
                        file=self.stderr_save)

        # Save and truncate stdout and stderr for the lifetime of RunCommand. This
        # way, we can return just the stdout and stderr that was output during the
        # RunNamedCommand call below.
        sys.stdout.seek(0)
        sys.stderr.seek(0)
        stdout = sys.stdout.read()
        stderr = sys.stderr.read()
        if stdout:
            self.accumulated_stdout.append(stdout)
        if stderr:
            self.accumulated_stderr.append(stderr)
        sys.stdout.seek(0)
        sys.stderr.seek(0)
        sys.stdout.truncate()
        sys.stderr.truncate()

        mock_log_handler = MockLoggingHandler()
        logging.getLogger(command_name).addHandler(mock_log_handler)
        if debug:
            logging.getLogger(command_name).setLevel(logging.DEBUG)

        try:
            with WorkingDirectory(cwd):
                self.command_runner.RunNamedCommand(command_name,
                                                    args=args,
                                                    headers=headers,
                                                    debug=debug,
                                                    parallel_operations=False,
                                                    do_shutdown=False)
        finally:
            sys.stdout.seek(0)
            sys.stderr.seek(0)
            if six.PY2:
                stdout = sys.stdout.read()
                stderr = sys.stderr.read()
            else:
                try:
                    stdout = sys.stdout.read()
                    stderr = sys.stderr.read()
                except UnicodeDecodeError:
                    sys.stdout.seek(0)
                    sys.stderr.seek(0)
                    stdout = sys.stdout.buffer.read()
                    stderr = sys.stderr.buffer.read()
            logging.getLogger(command_name).removeHandler(mock_log_handler)
            mock_log_handler.close()

            log_output = '\n'.join(
                '%s:\n  ' % level + '\n  '.join(records)
                for level, records in six.iteritems(mock_log_handler.messages)
                if records)

            _id = six.ensure_text(self.id())
            if self.is_debugging and log_output:
                print_to_fd('==== logging RunCommand {} {} ====\n'.format(
                    _id, command_line),
                            file=self.stderr_save)
                print_to_fd(log_output, file=self.stderr_save)
                print_to_fd('\n==== end logging ====\n', file=self.stderr_save)
            if self.is_debugging and stdout:
                print_to_fd('==== stdout RunCommand {} {} ====\n'.format(
                    _id, command_line),
                            file=self.stderr_save)
                print_to_fd(stdout, file=self.stderr_save)
                print_to_fd('==== end stdout ====\n', file=self.stderr_save)
            if self.is_debugging and stderr:
                print_to_fd('==== stderr RunCommand {} {} ====\n'.format(
                    _id, command_line),
                            file=self.stderr_save)
                print_to_fd(stderr, file=self.stderr_save)
                print_to_fd('==== end stderr ====\n', file=self.stderr_save)

            # Reset stdout and stderr files, so that we won't print them out again
            # in tearDown if debugging is enabled.
            sys.stdout.seek(0)
            sys.stderr.seek(0)
            sys.stdout.truncate()
            sys.stderr.truncate()

        to_return = []
        if return_stdout:
            to_return.append(stdout)
        if return_stderr:
            to_return.append(stderr)
        if return_log_handler:
            to_return.append(mock_log_handler)
        if len(to_return) == 1:
            return to_return[0]
        return tuple(to_return)
Exemple #23
0
 def _PrintPrefixLong(blr):
     text_util.print_to_fd(
         '%-33s%s' % ('', six.ensure_text(blr.url_string)))
Exemple #24
0
 def _PrintPrefixLong(blr):
     text_util.print_to_fd('%-33s%s' %
                           ('', blr.url_string.decode(UTF8)))
def PrintFullInfoAboutObject(bucket_listing_ref, incl_acl=True):
  """Print full info for given object (like what displays for gsutil ls -L).

  Args:
    bucket_listing_ref: BucketListingRef being listed.
                        Must have ref_type OBJECT and a populated root_object
                        with the desired fields.
    incl_acl: True if ACL info should be output.

  Returns:
    Tuple (number of objects, object_length)

  Raises:
    Exception: if calling bug encountered.
  """
  url_str = bucket_listing_ref.url_string
  storage_url = StorageUrlFromString(url_str)
  obj = bucket_listing_ref.root_object

  if (obj.metadata and
      S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties):
    num_bytes = 0
    num_objs = 0
    url_str += '<DeleteMarker>'
  else:
    num_bytes = obj.size
    num_objs = 1

  text_util.print_to_fd('{}:'.format(url_str))
  if obj.timeCreated:
    text_util.print_to_fd(
        MakeMetadataLine('Creation time',
                         obj.timeCreated.strftime('%a, %d %b %Y %H:%M:%S GMT')))
  if obj.updated:
    text_util.print_to_fd(
        MakeMetadataLine('Update time',
                         obj.updated.strftime('%a, %d %b %Y %H:%M:%S GMT')))
  if (obj.timeStorageClassUpdated and
      obj.timeStorageClassUpdated != obj.timeCreated):
    text_util.print_to_fd(
        MakeMetadataLine(
            'Storage class update time',
            obj.timeStorageClassUpdated.strftime('%a, %d %b %Y %H:%M:%S GMT')))
  if obj.storageClass:
    text_util.print_to_fd(MakeMetadataLine('Storage class', obj.storageClass))
  if obj.temporaryHold:
    text_util.print_to_fd(MakeMetadataLine('Temporary Hold', 'Enabled'))
  if obj.eventBasedHold:
    text_util.print_to_fd(MakeMetadataLine('Event-Based Hold', 'Enabled'))
  if obj.retentionExpirationTime:
    text_util.print_to_fd(
        MakeMetadataLine(
            'Retention Expiration',
            obj.retentionExpirationTime.strftime('%a, %d %b %Y %H:%M:%S GMT')))
  if obj.kmsKeyName:
    text_util.print_to_fd(MakeMetadataLine('KMS key', obj.kmsKeyName))
  if obj.cacheControl:
    text_util.print_to_fd(MakeMetadataLine('Cache-Control', obj.cacheControl))
  if obj.contentDisposition:
    text_util.print_to_fd(
        MakeMetadataLine('Content-Disposition', obj.contentDisposition))
  if obj.contentEncoding:
    text_util.print_to_fd(
        MakeMetadataLine('Content-Encoding', obj.contentEncoding))
  if obj.contentLanguage:
    text_util.print_to_fd(
        MakeMetadataLine('Content-Language', obj.contentLanguage))
  text_util.print_to_fd(MakeMetadataLine('Content-Length', obj.size))
  text_util.print_to_fd(MakeMetadataLine('Content-Type', obj.contentType))
  if obj.componentCount:
    text_util.print_to_fd(
        MakeMetadataLine('Component-Count', obj.componentCount))
  if obj.timeDeleted:
    text_util.print_to_fd(
        MakeMetadataLine('Archived time',
                         obj.timeDeleted.strftime('%a, %d %b %Y %H:%M:%S GMT')))
  marker_props = {}
  if obj.metadata and obj.metadata.additionalProperties:
    non_marker_props = []
    for add_prop in obj.metadata.additionalProperties:
      if add_prop.key not in S3_MARKER_GUIDS:
        non_marker_props.append(add_prop)
      else:
        marker_props[add_prop.key] = add_prop.value
    if non_marker_props:
      text_util.print_to_fd(MakeMetadataLine('Metadata', ''))
      for ap in non_marker_props:
        ap_key = '{}'.format(ap.key)
        ap_value = '{}'.format(ap.value)
        meta_data_line = MakeMetadataLine(ap_key, ap_value, indent=2)
        text_util.print_to_fd(meta_data_line)
  if obj.customerEncryption:
    if not obj.crc32c:
      text_util.print_to_fd(MakeMetadataLine('Hash (crc32c)', 'encrypted'))
    if not obj.md5Hash:
      text_util.print_to_fd(MakeMetadataLine('Hash (md5)', 'encrypted'))
    text_util.print_to_fd(
        MakeMetadataLine('Encryption algorithm',
                         obj.customerEncryption.encryptionAlgorithm))
    text_util.print_to_fd(
        MakeMetadataLine('Encryption key SHA256',
                         obj.customerEncryption.keySha256))
  if obj.crc32c:
    text_util.print_to_fd(MakeMetadataLine('Hash (crc32c)', obj.crc32c))
  if obj.md5Hash:
    text_util.print_to_fd(MakeMetadataLine('Hash (md5)', obj.md5Hash))
  text_util.print_to_fd(MakeMetadataLine('ETag', obj.etag.strip('"\'')))
  if obj.generation:
    generation_str = GenerationFromUrlAndString(storage_url, obj.generation)
    text_util.print_to_fd(MakeMetadataLine('Generation', generation_str))
  if obj.metageneration:
    text_util.print_to_fd(MakeMetadataLine('Metageneration',
                                           obj.metageneration))
  if incl_acl:
    # JSON API won't return acls as part of the response unless we have
    # full control scope
    if obj.acl:
      text_util.print_to_fd(
          MakeMetadataLine('ACL', AclTranslation.JsonFromMessage(obj.acl)))
    elif S3_ACL_MARKER_GUID in marker_props:
      text_util.print_to_fd(
          MakeMetadataLine('ACL', marker_props[S3_ACL_MARKER_GUID]))
    else:
      # Empty ACLs are possible with Bucket Policy Only and no longer imply
      # ACCESS DENIED anymore.
      text_util.print_to_fd(MakeMetadataLine('ACL', '[]'))

  return (num_objs, num_bytes)
Exemple #26
0
  def _PrintBucketInfo(self, bucket_blr, listing_style):
    """Print listing info for given bucket.

    Args:
      bucket_blr: BucketListingReference for the bucket being listed
      listing_style: ListingStyle enum describing type of output desired.

    Returns:
      Tuple (total objects, total bytes) in the bucket.
    """
    if (listing_style == ListingStyle.SHORT or
        listing_style == ListingStyle.LONG):
      text_util.print_to_fd(bucket_blr)
      return
    # listing_style == ListingStyle.LONG_LONG:
    # We're guaranteed by the caller that the root object is populated.
    bucket = bucket_blr.root_object
    location_constraint = bucket.location
    storage_class = bucket.storageClass
    fields = {
        'bucket': bucket_blr.url_string,
        'storage_class': storage_class,
        'location_constraint': location_constraint,
        'acl': AclTranslation.JsonFromMessage(bucket.acl),
        'default_acl': AclTranslation.JsonFromMessage(bucket.defaultObjectAcl),
        'versioning': bucket.versioning and bucket.versioning.enabled,
        'website_config': 'Present' if bucket.website else 'None',
        'logging_config': 'Present' if bucket.logging else 'None',
        'cors_config': 'Present' if bucket.cors else 'None',
        'lifecycle_config': 'Present' if bucket.lifecycle else 'None',
        'requester_pays': bucket.billing and bucket.billing.requesterPays
    }
    if bucket.retentionPolicy:
      fields['retention_policy'] = 'Present'
    if bucket.labels:
      fields['labels'] = LabelTranslation.JsonFromMessage(bucket.labels,
                                                          pretty_print=True)
    else:
      fields['labels'] = 'None'
    if bucket.encryption and bucket.encryption.defaultKmsKeyName:
      fields['default_kms_key'] = bucket.encryption.defaultKmsKeyName
    else:
      fields['default_kms_key'] = 'None'
    fields['encryption_config'] = 'Present' if bucket.encryption else 'None'
    # Fields not available in all APIs (e.g. the XML API)
    if bucket.metageneration:
      fields['metageneration'] = bucket.metageneration
    if bucket.timeCreated:
      fields['time_created'] = bucket.timeCreated.strftime(
          '%a, %d %b %Y %H:%M:%S GMT')
    if bucket.updated:
      fields['updated'] = bucket.updated.strftime('%a, %d %b %Y %H:%M:%S GMT')
    if bucket.defaultEventBasedHold:
      fields['default_eventbased_hold'] = bucket.defaultEventBasedHold
    if bucket.iamConfiguration and bucket.iamConfiguration.bucketPolicyOnly:
      enabled = bucket.iamConfiguration.bucketPolicyOnly.enabled
      fields['bucket_policy_only_enabled'] = enabled

    # For field values that are multiline, add indenting to make it look
    # prettier.
    for key in fields:
      previous_value = fields[key]
      if (not isinstance(previous_value, six.string_types) or
          '\n' not in previous_value):
        continue
      new_value = previous_value.replace('\n', '\n\t  ')
      # Start multiline values on a new line if they aren't already.
      if not new_value.startswith('\n'):
        new_value = '\n\t  ' + new_value
      fields[key] = new_value

    # Only display certain properties if the given API returned them (JSON API
    # returns many fields that the XML API does not).
    metageneration_line = ''
    time_created_line = ''
    time_updated_line = ''
    default_eventbased_hold_line = ''
    retention_policy_line = ''
    bucket_policy_only_enabled_line = ''
    if 'metageneration' in fields:
      metageneration_line = '\tMetageneration:\t\t\t{metageneration}\n'
    if 'time_created' in fields:
      time_created_line = '\tTime created:\t\t\t{time_created}\n'
    if 'updated' in fields:
      time_updated_line = '\tTime updated:\t\t\t{updated}\n'
    if 'default_eventbased_hold' in fields:
      default_eventbased_hold_line = (
          '\tDefault Event-Based Hold:\t{default_eventbased_hold}\n')
    if 'retention_policy' in fields:
      retention_policy_line = '\tRetention Policy:\t\t{retention_policy}\n'
    if 'bucket_policy_only_enabled' in fields:
      bucket_policy_only_enabled_line = ('\tBucket Policy Only enabled:\t'
                                         '{bucket_policy_only_enabled}\n')

    text_util.print_to_fd(
        ('{bucket} :\n'
         '\tStorage class:\t\t\t{storage_class}\n'
         '\tLocation constraint:\t\t{location_constraint}\n'
         '\tVersioning enabled:\t\t{versioning}\n'
         '\tLogging configuration:\t\t{logging_config}\n'
         '\tWebsite configuration:\t\t{website_config}\n'
         '\tCORS configuration: \t\t{cors_config}\n'
         '\tLifecycle configuration:\t{lifecycle_config}\n'
         '\tRequester Pays enabled:\t\t{requester_pays}\n' +
         retention_policy_line + default_eventbased_hold_line +
         '\tLabels:\t\t\t\t{labels}\n' +
         '\tDefault KMS key:\t\t{default_kms_key}\n' + time_created_line +
         time_updated_line + metageneration_line +
         bucket_policy_only_enabled_line + '\tACL:\t\t\t\t{acl}\n'
         '\tDefault ACL:\t\t\t{default_acl}').format(**fields))
    if bucket_blr.storage_url.scheme == 's3':
      text_util.print_to_fd(
          'Note: this is an S3 bucket so configuration values may be '
          'blank. To retrieve bucket configuration values, use '
          'individual configuration commands such as gsutil acl get '
          '<bucket>.')
Exemple #27
0
 def _PrintSummaryLine(self, num_bytes, name):
     size_string = (MakeHumanReadable(num_bytes)
                    if self.human_readable else six.text_type(num_bytes))
     text_util.print_to_fd('{size:<11}  {name}'.format(
         size=size_string, name=six.ensure_text(name)),
                           end=self.line_ending)
Exemple #28
0
 def MaybePrintBucketHeader(blr):
   if len(self.args) > 1:
     text_util.print_to_fd('%s:' % blr.url_string.decode(UTF8))
Exemple #29
0
def PrintFullInfoAboutObject(bucket_listing_ref, incl_acl=True):
    """Print full info for given object (like what displays for gsutil ls -L).

  Args:
    bucket_listing_ref: BucketListingRef being listed.
                        Must have ref_type OBJECT and a populated root_object
                        with the desired fields.
    incl_acl: True if ACL info should be output.

  Returns:
    Tuple (number of objects, object_length)

  Raises:
    Exception: if calling bug encountered.
  """
    url_str = bucket_listing_ref.url_string
    storage_url = StorageUrlFromString(url_str)
    obj = bucket_listing_ref.root_object

    if (obj.metadata
            and S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties):
        num_bytes = 0
        num_objs = 0
        url_str += '<DeleteMarker>'
    else:
        num_bytes = obj.size
        num_objs = 1

    text_util.print_to_fd('{}:'.format(url_str))
    if obj.timeCreated:
        text_util.print_to_fd(
            MakeMetadataLine(
                'Creation time',
                obj.timeCreated.strftime('%a, %d %b %Y %H:%M:%S GMT')))
    if obj.updated:
        text_util.print_to_fd(
            MakeMetadataLine(
                'Update time',
                obj.updated.strftime('%a, %d %b %Y %H:%M:%S GMT')))
    if (obj.timeStorageClassUpdated
            and obj.timeStorageClassUpdated != obj.timeCreated):
        text_util.print_to_fd(
            MakeMetadataLine(
                'Storage class update time',
                obj.timeStorageClassUpdated.strftime(
                    '%a, %d %b %Y %H:%M:%S GMT')))
    if obj.storageClass:
        text_util.print_to_fd(
            MakeMetadataLine('Storage class', obj.storageClass))
    if obj.temporaryHold:
        text_util.print_to_fd(MakeMetadataLine('Temporary Hold', 'Enabled'))
    if obj.eventBasedHold:
        text_util.print_to_fd(MakeMetadataLine('Event-Based Hold', 'Enabled'))
    if obj.retentionExpirationTime:
        text_util.print_to_fd(
            MakeMetadataLine(
                'Retention Expiration',
                obj.retentionExpirationTime.strftime(
                    '%a, %d %b %Y %H:%M:%S GMT')))
    if obj.kmsKeyName:
        text_util.print_to_fd(MakeMetadataLine('KMS key', obj.kmsKeyName))
    if obj.cacheControl:
        text_util.print_to_fd(
            MakeMetadataLine('Cache-Control', obj.cacheControl))
    if obj.contentDisposition:
        text_util.print_to_fd(
            MakeMetadataLine('Content-Disposition', obj.contentDisposition))
    if obj.contentEncoding:
        text_util.print_to_fd(
            MakeMetadataLine('Content-Encoding', obj.contentEncoding))
    if obj.contentLanguage:
        text_util.print_to_fd(
            MakeMetadataLine('Content-Language', obj.contentLanguage))
    text_util.print_to_fd(MakeMetadataLine('Content-Length', obj.size))
    text_util.print_to_fd(MakeMetadataLine('Content-Type', obj.contentType))
    if obj.componentCount:
        text_util.print_to_fd(
            MakeMetadataLine('Component-Count', obj.componentCount))
    if obj.timeDeleted:
        text_util.print_to_fd(
            MakeMetadataLine(
                'Archived time',
                obj.timeDeleted.strftime('%a, %d %b %Y %H:%M:%S GMT')))
    marker_props = {}
    if obj.metadata and obj.metadata.additionalProperties:
        non_marker_props = []
        for add_prop in obj.metadata.additionalProperties:
            if add_prop.key not in S3_MARKER_GUIDS:
                non_marker_props.append(add_prop)
            else:
                marker_props[add_prop.key] = add_prop.value
        if non_marker_props:
            text_util.print_to_fd(MakeMetadataLine('Metadata', ''))
            for ap in non_marker_props:
                ap_key = '{}'.format(ap.key)
                ap_value = '{}'.format(ap.value)
                meta_data_line = MakeMetadataLine(ap_key, ap_value, indent=2)
                text_util.print_to_fd(meta_data_line)
    if obj.customerEncryption:
        if not obj.crc32c:
            text_util.print_to_fd(
                MakeMetadataLine('Hash (crc32c)', 'encrypted'))
        if not obj.md5Hash:
            text_util.print_to_fd(MakeMetadataLine('Hash (md5)', 'encrypted'))
        text_util.print_to_fd(
            MakeMetadataLine('Encryption algorithm',
                             obj.customerEncryption.encryptionAlgorithm))
        text_util.print_to_fd(
            MakeMetadataLine('Encryption key SHA256',
                             obj.customerEncryption.keySha256))
    if obj.crc32c:
        text_util.print_to_fd(MakeMetadataLine('Hash (crc32c)', obj.crc32c))
    if obj.md5Hash:
        text_util.print_to_fd(MakeMetadataLine('Hash (md5)', obj.md5Hash))
    text_util.print_to_fd(MakeMetadataLine('ETag', obj.etag.strip('"\'')))
    if obj.generation:
        generation_str = GenerationFromUrlAndString(storage_url,
                                                    obj.generation)
        text_util.print_to_fd(MakeMetadataLine('Generation', generation_str))
    if obj.metageneration:
        text_util.print_to_fd(
            MakeMetadataLine('Metageneration', obj.metageneration))
    if incl_acl:
        # JSON API won't return acls as part of the response unless we have
        # full control scope
        if obj.acl:
            text_util.print_to_fd(
                MakeMetadataLine('ACL',
                                 AclTranslation.JsonFromMessage(obj.acl)))
        elif S3_ACL_MARKER_GUID in marker_props:
            text_util.print_to_fd(
                MakeMetadataLine('ACL', marker_props[S3_ACL_MARKER_GUID]))
        else:
            # Empty ACLs are possible with Bucket Policy Only and no longer imply
            # ACCESS DENIED anymore.
            text_util.print_to_fd(MakeMetadataLine('ACL', '[]'))

    return (num_objs, num_bytes)
Exemple #30
0
    def RunCommand(self):
        """Command entry point for the ls command."""
        got_nomatch_errors = False
        got_bucket_nomatch_errors = False
        listing_style = ListingStyle.SHORT
        get_bucket_info = False
        self.recursion_requested = False
        self.all_versions = False
        self.include_etag = False
        self.human_readable = False
        self.list_subdir_contents = True
        if self.sub_opts:
            for o, a in self.sub_opts:
                if o == '-a':
                    self.all_versions = True
                elif o == '-e':
                    self.include_etag = True
                elif o == '-b':
                    get_bucket_info = True
                elif o == '-h':
                    self.human_readable = True
                elif o == '-l':
                    listing_style = ListingStyle.LONG
                elif o == '-L':
                    listing_style = ListingStyle.LONG_LONG
                elif o == '-p':
                    # Project IDs are sent as header values when using gs and s3 XML APIs.
                    InsistAscii(
                        a, 'Invalid non-ASCII character found in project ID')
                    self.project_id = a
                elif o == '-r' or o == '-R':
                    self.recursion_requested = True
                elif o == '-d':
                    self.list_subdir_contents = False

        if not self.args:
            # default to listing all gs buckets
            self.args = ['gs://']

        total_objs = 0
        total_bytes = 0

        def MaybePrintBucketHeader(blr):
            if len(self.args) > 1:
                text_util.print_to_fd('%s:' % six.ensure_text(blr.url_string))

        print_bucket_header = MaybePrintBucketHeader

        for url_str in self.args:
            storage_url = StorageUrlFromString(url_str)
            if storage_url.IsFileUrl():
                raise CommandException('Only cloud URLs are supported for %s' %
                                       self.command_name)
            bucket_fields = None
            if (listing_style == ListingStyle.SHORT
                    or listing_style == ListingStyle.LONG):
                bucket_fields = ['id']
            elif listing_style == ListingStyle.LONG_LONG:
                bucket_fields = [
                    'acl',
                    'billing',
                    'cors',
                    'defaultObjectAcl',
                    'encryption',
                    'iamConfiguration',
                    'labels',
                    'location',
                    'locationType',
                    'logging',
                    'lifecycle',
                    'metageneration',
                    'retentionPolicy',
                    'defaultEventBasedHold',
                    'storageClass',
                    'timeCreated',
                    'updated',
                    'versioning',
                    'website',
                ]
            if storage_url.IsProvider():
                # Provider URL: use bucket wildcard to list buckets.
                for blr in self.WildcardIterator(
                        '%s://*' % storage_url.scheme).IterBuckets(
                            bucket_fields=bucket_fields):
                    self._PrintBucketInfo(blr, listing_style)
            elif storage_url.IsBucket() and get_bucket_info:
                # ls -b bucket listing request: List info about bucket(s).
                total_buckets = 0
                for blr in self.WildcardIterator(url_str).IterBuckets(
                        bucket_fields=bucket_fields):
                    if not ContainsWildcard(url_str) and not blr.root_object:
                        # Iterator does not make an HTTP call for non-wildcarded
                        # listings with fields=='id'. Ensure the bucket exists by calling
                        # GetBucket.
                        self.gsutil_api.GetBucket(blr.storage_url.bucket_name,
                                                  fields=['id'],
                                                  provider=storage_url.scheme)
                    self._PrintBucketInfo(blr, listing_style)
                    total_buckets += 1
                if not ContainsWildcard(url_str) and not total_buckets:
                    got_bucket_nomatch_errors = True
            else:
                # URL names a bucket, object, or object subdir ->
                # list matching object(s) / subdirs.
                def _PrintPrefixLong(blr):
                    text_util.print_to_fd(
                        '%-33s%s' % ('', six.ensure_text(blr.url_string)))

                if listing_style == ListingStyle.SHORT:
                    # ls helper by default readies us for a short listing.
                    listing_helper = LsHelper(
                        self.WildcardIterator,
                        self.logger,
                        all_versions=self.all_versions,
                        print_bucket_header_func=print_bucket_header,
                        should_recurse=self.recursion_requested,
                        list_subdir_contents=self.list_subdir_contents)
                elif listing_style == ListingStyle.LONG:
                    bucket_listing_fields = [
                        'name',
                        'size',
                        'timeCreated',
                        'updated',
                    ]
                    if self.all_versions:
                        bucket_listing_fields.extend([
                            'generation',
                            'metageneration',
                        ])
                    if self.include_etag:
                        bucket_listing_fields.append('etag')

                    listing_helper = LsHelper(
                        self.WildcardIterator,
                        self.logger,
                        print_object_func=self._PrintLongListing,
                        print_dir_func=_PrintPrefixLong,
                        print_bucket_header_func=print_bucket_header,
                        all_versions=self.all_versions,
                        should_recurse=self.recursion_requested,
                        fields=bucket_listing_fields,
                        list_subdir_contents=self.list_subdir_contents)

                elif listing_style == ListingStyle.LONG_LONG:
                    # List all fields
                    bucket_listing_fields = (UNENCRYPTED_FULL_LISTING_FIELDS +
                                             ENCRYPTED_FIELDS)
                    listing_helper = LsHelper(
                        self.WildcardIterator,
                        self.logger,
                        print_object_func=PrintFullInfoAboutObject,
                        print_dir_func=_PrintPrefixLong,
                        print_bucket_header_func=print_bucket_header,
                        all_versions=self.all_versions,
                        should_recurse=self.recursion_requested,
                        fields=bucket_listing_fields,
                        list_subdir_contents=self.list_subdir_contents)
                else:
                    raise CommandException('Unknown listing style: %s' %
                                           listing_style)

                exp_dirs, exp_objs, exp_bytes = (
                    listing_helper.ExpandUrlAndPrint(storage_url))
                if storage_url.IsObject() and exp_objs == 0 and exp_dirs == 0:
                    got_nomatch_errors = True
                total_bytes += exp_bytes
                total_objs += exp_objs

        if total_objs and listing_style != ListingStyle.SHORT:
            text_util.print_to_fd('TOTAL: %d objects, %d bytes (%s)' %
                                  (total_objs, total_bytes,
                                   MakeHumanReadable(float(total_bytes))))
        if got_nomatch_errors:
            raise CommandException('One or more URLs matched no objects.')
        if got_bucket_nomatch_errors:
            raise NotFoundException(
                'One or more bucket URLs matched no buckets.')

        return 0
Exemple #31
0
 def MaybePrintBucketHeader(blr):
     if len(self.args) > 1:
         text_util.print_to_fd('%s:' % six.ensure_text(blr.url_string))
  def MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug):
    """Checks the last time we checked for an update and offers one if needed.

    Offer is made if the time since the last update check is longer
    than the configured threshold offers the user to update gsutil.

    Args:
      command_name: The name of the command being run.
      debug: Debug level to pass in to boto connection (range 0..3).

    Returns:
      True if the user decides to update.
    """
    # Don't try to interact with user if:
    # - gsutil is not connected to a tty (e.g., if being run from cron);
    # - user is running gsutil -q
    # - user is running the config command (which could otherwise attempt to
    #   check for an update for a user running behind a proxy, who has not yet
    #   configured gsutil to go through the proxy; for such users we need the
    #   first connection attempt to be made by the gsutil config command).
    # - user is running the version command (which gets run when using
    #   gsutil -D, which would prevent users with proxy config problems from
    #   sending us gsutil -D output).
    # - user is running the update command (which could otherwise cause an
    #   additional note that an update is available when user is already trying
    #   to perform an update);
    # - user specified gs_host (which could be a non-production different
    #   service instance, in which case credentials won't work for checking
    #   gsutil tarball).
    # - user is using a Cloud SDK install (which should only be updated via
    #   gcloud components update)
    logger = logging.getLogger()
    if (not system_util.IsRunningInteractively() or
        command_name in ('config', 'update', 'ver', 'version') or
        not logger.isEnabledFor(logging.INFO) or HAS_NON_DEFAULT_GS_HOST or
        system_util.InvokedViaCloudSdk()):
      return False

    software_update_check_period = boto.config.getint(
        'GSUtil', 'software_update_check_period', 30)
    # Setting software_update_check_period to 0 means periodic software
    # update checking is disabled.
    if software_update_check_period == 0:
      return False

    last_checked_for_gsutil_update_timestamp_file = (
        boto_util.GetLastCheckedForGsutilUpdateTimestampFile())

    cur_ts = int(time.time())
    if not os.path.isfile(last_checked_for_gsutil_update_timestamp_file):
      # Set last_checked_ts from date of VERSION file, so if the user installed
      # an old copy of gsutil it will get noticed (and an update offered) the
      # first time they try to run it.
      last_checked_ts = gslib.GetGsutilVersionModifiedTime()
      with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f:
        f.write(str(last_checked_ts))
    else:
      try:
        with open(last_checked_for_gsutil_update_timestamp_file, 'r') as f:
          last_checked_ts = int(f.readline())
      except (TypeError, ValueError):
        return False

    if (cur_ts - last_checked_ts >
        software_update_check_period * SECONDS_PER_DAY):
      # Create a credential-less gsutil API to check for the public
      # update tarball.
      gsutil_api = GcsJsonApi(self.bucket_storage_uri_class,
                              logger,
                              DiscardMessagesQueue(),
                              credentials=NoOpCredentials(),
                              debug=debug)

      cur_ver = LookUpGsutilVersion(gsutil_api, GSUTIL_PUB_TARBALL)
      with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f:
        f.write(str(cur_ts))
      (g, m) = CompareVersions(cur_ver, gslib.VERSION)
      if m:
        print_to_fd('\n'.join(
            textwrap.wrap(
                'A newer version of gsutil (%s) is available than the version you '
                'are running (%s). NOTE: This is a major new version, so it is '
                'strongly recommended that you review the release note details at '
                '%s before updating to this version, especially if you use gsutil '
                'in scripts.' % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))))
        if gslib.IS_PACKAGE_INSTALL:
          return False
        print_to_fd('\n')
        answer = input('Would you like to update [y/N]? ')
        return answer and answer.lower()[0] == 'y'
      elif g:
        print_to_fd('\n'.join(
            textwrap.wrap(
                'A newer version of gsutil (%s) is available than the version you '
                'are running (%s). A detailed log of gsutil release changes is '
                'available at %s if you would like to read them before updating.'
                % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))))
        if gslib.IS_PACKAGE_INSTALL:
          return False
        print_to_fd('\n')
        answer = input('Would you like to update [Y/n]? ')
        return not answer or answer.lower()[0] != 'n'
    return False
    def tearDown(self):
        super(GsUtilUnitTestCase, self).tearDown()

        self.root_logger.handlers = self.log_handlers_save
        self.temp_log_handler.flush()
        self.temp_log_handler.close()
        self.log_handler_stream.seek(0)
        log_output = self.log_handler_stream.read()
        self.log_handler_stream.close()
        os.unlink(self.log_handler_file)

        sys.stdout.seek(0)
        sys.stderr.seek(0)
        if six.PY2:
            stdout = sys.stdout.read()
            stderr = sys.stderr.read()
        else:
            try:
                stdout = sys.stdout.read()
                stderr = sys.stderr.read()
            except UnicodeDecodeError:
                sys.stdout.seek(0)
                sys.stderr.seek(0)
                stdout = sys.stdout.buffer.read()
                stderr = sys.stderr.buffer.read()
        [six.ensure_text(string) for string in self.accumulated_stderr]
        [six.ensure_text(string) for string in self.accumulated_stdout]
        stdout = six.ensure_text(get_utf8able_str(stdout))
        stderr = six.ensure_text(get_utf8able_str(stderr))
        stdout += ''.join(self.accumulated_stdout)
        stderr += ''.join(self.accumulated_stderr)
        _AttemptToCloseSysFd(sys.stdout)
        _AttemptToCloseSysFd(sys.stderr)
        sys.stdout = self.stdout_save
        sys.stderr = self.stderr_save
        os.unlink(self.stdout_file)
        os.unlink(self.stderr_file)

        _id = six.ensure_text(self.id())
        if self.is_debugging and stdout:
            print_to_fd('==== stdout {} ====\n'.format(_id), file=sys.stderr)
            print_to_fd(stdout, file=sys.stderr)
            print_to_fd('==== end stdout ====\n', file=sys.stderr)
        if self.is_debugging and stderr:
            print_to_fd('==== stderr {} ====\n'.format(_id), file=sys.stderr)
            print_to_fd(stderr, file=sys.stderr)
            print_to_fd('==== end stderr ====\n', file=sys.stderr)
        if self.is_debugging and log_output:
            print_to_fd('==== log output {} ====\n'.format(_id),
                        file=sys.stderr)
            print_to_fd(log_output, file=sys.stderr)
            print_to_fd('==== end log output ====\n', file=sys.stderr)
Exemple #34
0
    def MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug):
        """Checks the last time we checked for an update and offers one if needed.

    Offer is made if the time since the last update check is longer
    than the configured threshold offers the user to update gsutil.

    Args:
      command_name: The name of the command being run.
      debug: Debug level to pass in to boto connection (range 0..3).

    Returns:
      True if the user decides to update.
    """
        # Don't try to interact with user if:
        # - gsutil is not connected to a tty (e.g., if being run from cron);
        # - user is running gsutil -q
        # - user is running the config command (which could otherwise attempt to
        #   check for an update for a user running behind a proxy, who has not yet
        #   configured gsutil to go through the proxy; for such users we need the
        #   first connection attempt to be made by the gsutil config command).
        # - user is running the version command (which gets run when using
        #   gsutil -D, which would prevent users with proxy config problems from
        #   sending us gsutil -D output).
        # - user is running the update command (which could otherwise cause an
        #   additional note that an update is available when user is already trying
        #   to perform an update);
        # - user specified gs_host (which could be a non-production different
        #   service instance, in which case credentials won't work for checking
        #   gsutil tarball).
        # - user is using a Cloud SDK install (which should only be updated via
        #   gcloud components update)
        logger = logging.getLogger()
        if (not system_util.IsRunningInteractively()
                or command_name in ('config', 'update', 'ver', 'version')
                or not logger.isEnabledFor(logging.INFO)
                or HAS_NON_DEFAULT_GS_HOST
                or system_util.InvokedViaCloudSdk()):
            return False

        software_update_check_period = boto.config.getint(
            'GSUtil', 'software_update_check_period', 30)
        # Setting software_update_check_period to 0 means periodic software
        # update checking is disabled.
        if software_update_check_period == 0:
            return False

        last_checked_for_gsutil_update_timestamp_file = (
            boto_util.GetLastCheckedForGsutilUpdateTimestampFile())

        cur_ts = int(time.time())
        if not os.path.isfile(last_checked_for_gsutil_update_timestamp_file):
            # Set last_checked_ts from date of VERSION file, so if the user installed
            # an old copy of gsutil it will get noticed (and an update offered) the
            # first time they try to run it.
            last_checked_ts = gslib.GetGsutilVersionModifiedTime()
            with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f:
                f.write(str(last_checked_ts))
        else:
            try:
                with open(last_checked_for_gsutil_update_timestamp_file,
                          'r') as f:
                    last_checked_ts = int(f.readline())
            except (TypeError, ValueError):
                return False

        if (cur_ts - last_checked_ts >
                software_update_check_period * SECONDS_PER_DAY):
            # Create a credential-less gsutil API to check for the public
            # update tarball.
            gsutil_api = GcsJsonApi(self.bucket_storage_uri_class,
                                    logger,
                                    DiscardMessagesQueue(),
                                    credentials=NoOpCredentials(),
                                    debug=debug)

            cur_ver = gslib.VERSION
            try:
                cur_ver = LookUpGsutilVersion(gsutil_api, GSUTIL_PUB_TARBALL)
            except Exception:
                return False

            with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f:
                f.write(str(cur_ts))
            (g, m) = CompareVersions(cur_ver, gslib.VERSION)
            if m:
                print_to_fd('\n'.join(
                    textwrap.wrap(
                        'A newer version of gsutil (%s) is available than the version you '
                        'are running (%s). NOTE: This is a major new version, so it is '
                        'strongly recommended that you review the release note details at '
                        '%s before updating to this version, especially if you use gsutil '
                        'in scripts.' %
                        (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))))
                if gslib.IS_PACKAGE_INSTALL:
                    return False
                print_to_fd('\n')
                answer = input('Would you like to update [y/N]? ')
                return answer and answer.lower()[0] == 'y'
            elif g:
                print_to_fd('\n'.join(
                    textwrap.wrap(
                        'A newer version of gsutil (%s) is available than the version you '
                        'are running (%s). A detailed log of gsutil release changes is '
                        'available at %s if you would like to read them before updating.'
                        % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))))
                if gslib.IS_PACKAGE_INSTALL:
                    return False
                print_to_fd('\n')
                answer = input('Would you like to update [Y/n]? ')
                return not answer or answer.lower()[0] != 'n'
        return False
Exemple #35
0
from gslib.utils import system_util, text_util

# pylint: disable=g-import-not-at-top
# This module also imports boto, and will override the UserAgent global variable
# if imported above.
from gslib import metrics

# We parse the options and arguments here so we can pass the results to the user
# agent helper.
try:
  opts, args = GetArgumentsAndOptions()
except CommandException as e:
  reason = e.reason if e.informational else 'CommandException: %s' % e.reason
  err = '%s\n' % reason
  try:
    text_util.print_to_fd(err, end='', file=sys.stderr)
  except UnicodeDecodeError:
    # Can happen when outputting invalid Unicode filenames.
    sys.stderr.write(err)
  if e:
    metrics.LogFatalError(e)
  sys.exit(1)

# This calculated user agent can be stored for use in StorageV1.
gslib.USER_AGENT = GetUserAgent(args, metrics.MetricsCollector.IsDisabled())
boto.UserAgent += gslib.USER_AGENT

# pylint: disable=g-bad-import-order
import httplib2
import oauth2client
from google_reauth import reauth_creds
Exemple #36
0
  def RunCommand(self):
    """Command entry point for the hash command."""
    (calc_crc32c, calc_md5, format_func, cloud_format_func,
     output_format) = (self._ParseOpts(self.sub_opts, self.logger))

    matched_one = False
    for url_str in self.args:
      for file_ref in self.WildcardIterator(url_str).IterObjects(
          bucket_listing_fields=[
              'crc32c',
              'customerEncryption',
              'md5Hash',
              'size',
          ]):
        matched_one = True
        url = StorageUrlFromString(url_str)
        file_name = file_ref.storage_url.object_name
        if StorageUrlFromString(url_str).IsFileUrl():
          file_size = os.path.getsize(file_name)
          self.gsutil_api.status_queue.put(
              FileMessage(url,
                          None,
                          time.time(),
                          size=file_size,
                          finished=False,
                          message_type=FileMessage.FILE_HASH))
          callback_processor = ProgressCallbackWithTimeout(
              file_size,
              FileProgressCallbackHandler(self.gsutil_api.status_queue,
                                          src_url=StorageUrlFromString(url_str),
                                          operation_name='Hashing').call)
          hash_dict = self._GetHashClassesFromArgs(calc_crc32c, calc_md5)
          with open(file_name, 'rb') as fp:
            hashing_helper.CalculateHashesFromContents(
                fp, hash_dict, callback_processor=callback_processor)
          self.gsutil_api.status_queue.put(
              FileMessage(url,
                          None,
                          time.time(),
                          size=file_size,
                          finished=True,
                          message_type=FileMessage.FILE_HASH))
        else:
          hash_dict = {}
          obj_metadata = file_ref.root_object
          file_size = obj_metadata.size
          md5_present = obj_metadata.md5Hash is not None
          crc32c_present = obj_metadata.crc32c is not None
          if not md5_present and not crc32c_present:
            logging.getLogger().warn('No hashes present for %s', url_str)
            continue
          if md5_present:
            hash_dict['md5'] = obj_metadata.md5Hash
          if crc32c_present:
            hash_dict['crc32c'] = obj_metadata.crc32c
        text_util.print_to_fd('Hashes [%s] for %s:' %
                              (output_format, file_name))
        for name, digest in six.iteritems(hash_dict):
          text_util.print_to_fd('\tHash (%s):\t\t%s' %
                                (name, (format_func(digest) if url.IsFileUrl()
                                        else cloud_format_func(digest))))

    if not matched_one:
      raise CommandException('No files matched')
    _PutToQueueWithTimeout(self.gsutil_api.status_queue,
                           FinalMessage(time.time()))
    return 0
Exemple #37
0
 def _PrintPrefixLong(blr):
   text_util.print_to_fd('%-33s%s' % ('', blr.url_string.decode(UTF8)))
Exemple #38
0
  def RunCommand(self):
    """Command entry point for the ls command."""
    got_nomatch_errors = False
    got_bucket_nomatch_errors = False
    listing_style = ListingStyle.SHORT
    get_bucket_info = False
    self.recursion_requested = False
    self.all_versions = False
    self.include_etag = False
    self.human_readable = False
    self.list_subdir_contents = True
    if self.sub_opts:
      for o, a in self.sub_opts:
        if o == '-a':
          self.all_versions = True
        elif o == '-e':
          self.include_etag = True
        elif o == '-b':
          get_bucket_info = True
        elif o == '-h':
          self.human_readable = True
        elif o == '-l':
          listing_style = ListingStyle.LONG
        elif o == '-L':
          listing_style = ListingStyle.LONG_LONG
        elif o == '-p':
          # Project IDs are sent as header values when using gs and s3 XML APIs.
          InsistAscii(a, 'Invalid non-ASCII character found in project ID')
          self.project_id = a
        elif o == '-r' or o == '-R':
          self.recursion_requested = True
        elif o == '-d':
          self.list_subdir_contents = False

    if not self.args:
      # default to listing all gs buckets
      self.args = ['gs://']

    total_objs = 0
    total_bytes = 0

    def MaybePrintBucketHeader(blr):
      if len(self.args) > 1:
        text_util.print_to_fd('%s:' % blr.url_string.decode(UTF8))

    print_bucket_header = MaybePrintBucketHeader

    for url_str in self.args:
      storage_url = StorageUrlFromString(url_str)
      if storage_url.IsFileUrl():
        raise CommandException('Only cloud URLs are supported for %s' %
                               self.command_name)
      bucket_fields = None
      if (listing_style == ListingStyle.SHORT or
          listing_style == ListingStyle.LONG):
        bucket_fields = ['id']
      elif listing_style == ListingStyle.LONG_LONG:
        bucket_fields = [
            'acl',
            'billing',
            'cors',
            'defaultObjectAcl',
            'encryption',
            'iamConfiguration',
            'labels',
            'location',
            'logging',
            'lifecycle',
            'metageneration',
            'retentionPolicy',
            'defaultEventBasedHold',
            'storageClass',
            'timeCreated',
            'updated',
            'versioning',
            'website',
        ]
      if storage_url.IsProvider():
        # Provider URL: use bucket wildcard to list buckets.
        for blr in self.WildcardIterator(
            '%s://*' %
            storage_url.scheme).IterBuckets(bucket_fields=bucket_fields):
          self._PrintBucketInfo(blr, listing_style)
      elif storage_url.IsBucket() and get_bucket_info:
        # ls -b bucket listing request: List info about bucket(s).
        total_buckets = 0
        for blr in self.WildcardIterator(url_str).IterBuckets(
            bucket_fields=bucket_fields):
          if not ContainsWildcard(url_str) and not blr.root_object:
            # Iterator does not make an HTTP call for non-wildcarded
            # listings with fields=='id'. Ensure the bucket exists by calling
            # GetBucket.
            self.gsutil_api.GetBucket(blr.storage_url.bucket_name,
                                      fields=['id'],
                                      provider=storage_url.scheme)
          self._PrintBucketInfo(blr, listing_style)
          total_buckets += 1
        if not ContainsWildcard(url_str) and not total_buckets:
          got_bucket_nomatch_errors = True
      else:
        # URL names a bucket, object, or object subdir ->
        # list matching object(s) / subdirs.
        def _PrintPrefixLong(blr):
          text_util.print_to_fd('%-33s%s' % ('', blr.url_string.decode(UTF8)))

        if listing_style == ListingStyle.SHORT:
          # ls helper by default readies us for a short listing.
          listing_helper = LsHelper(
              self.WildcardIterator,
              self.logger,
              all_versions=self.all_versions,
              print_bucket_header_func=print_bucket_header,
              should_recurse=self.recursion_requested,
              list_subdir_contents=self.list_subdir_contents)
        elif listing_style == ListingStyle.LONG:
          bucket_listing_fields = [
              'name',
              'size',
              'timeCreated',
              'updated',
          ]
          if self.all_versions:
            bucket_listing_fields.extend([
                'generation',
                'metageneration',
            ])
          if self.include_etag:
            bucket_listing_fields.append('etag')

          listing_helper = LsHelper(
              self.WildcardIterator,
              self.logger,
              print_object_func=self._PrintLongListing,
              print_dir_func=_PrintPrefixLong,
              print_bucket_header_func=print_bucket_header,
              all_versions=self.all_versions,
              should_recurse=self.recursion_requested,
              fields=bucket_listing_fields,
              list_subdir_contents=self.list_subdir_contents)

        elif listing_style == ListingStyle.LONG_LONG:
          # List all fields
          bucket_listing_fields = (UNENCRYPTED_FULL_LISTING_FIELDS +
                                   ENCRYPTED_FIELDS)
          listing_helper = LsHelper(
              self.WildcardIterator,
              self.logger,
              print_object_func=PrintFullInfoAboutObject,
              print_dir_func=_PrintPrefixLong,
              print_bucket_header_func=print_bucket_header,
              all_versions=self.all_versions,
              should_recurse=self.recursion_requested,
              fields=bucket_listing_fields,
              list_subdir_contents=self.list_subdir_contents)
        else:
          raise CommandException('Unknown listing style: %s' % listing_style)

        exp_dirs, exp_objs, exp_bytes = (
            listing_helper.ExpandUrlAndPrint(storage_url))
        if storage_url.IsObject() and exp_objs == 0 and exp_dirs == 0:
          got_nomatch_errors = True
        total_bytes += exp_bytes
        total_objs += exp_objs

    if total_objs and listing_style != ListingStyle.SHORT:
      text_util.print_to_fd(
          'TOTAL: %d objects, %d bytes (%s)' %
          (total_objs, total_bytes, MakeHumanReadable(float(total_bytes))))
    if got_nomatch_errors:
      raise CommandException('One or more URLs matched no objects.')
    if got_bucket_nomatch_errors:
      raise NotFoundException('One or more bucket URLs matched no buckets.')

    return 0
Exemple #39
0
 def MaybePrintBucketHeader(blr):
     if len(self.args) > 1:
         text_util.print_to_fd('%s:' % blr.url_string.decode(UTF8))
def PrintNewLine():
  """Default function for printing new lines between directories."""
  text_util.print_to_fd()