# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper functions for dealing with encryption keys used with cloud APIs."""
import base64
import binascii
from hashlib import sha256
import re
from gslib.exception import CommandException
from gslib.lazy_wrapper import LazyWrapper
MAX_DECRYPTION_KEYS = 100
VALID_CMEK_RE = LazyWrapper(
lambda: re.compile('projects/([^/]+)/'
'locations/([a-zA-Z0-9_-]{1,63})/'
'keyRings/([a-zA-Z0-9_-]{1,63})/'
'cryptoKeys/([a-zA-Z0-9_-]{1,63})$'))
class CryptoKeyType(object):
"""Enum of valid types of encryption keys used with cloud API requests."""
CSEK = 'CSEK'
CMEK = 'CMEK'
class CryptoKeyWrapper(object):
"""Class describing a crypto key used with cloud API requests.
This class should be instantiated via the `CryptoKeyWrapperFromKey` method.
"""
user_groups = GetUserGroups()
if len(user_groups) == 1:
return primary_gid
return [g for g in list(user_groups) if g != primary_gid][0]
def GetPrimaryGid():
return os.getgid()
def GetUserGroups():
return set(
[GetPrimaryGid()] +
[g.gr_gid for g in grp.getgrall() if USER_NAME() in g.gr_mem])
DEFAULT_MODE = int(GetDefaultMode(), 8)
USER_ID = os.getuid()
USER_NAME = LazyWrapper(lambda: pwd.getpwuid(USER_ID).pw_name)
# Take the current user's UID and increment it by one, this counts as an
# invalid UID, as the metric used is if the UID matches the current user's,
# exactly.
INVALID_UID = LazyWrapper(
lambda: sorted([user.pw_uid for user in pwd.getpwall()])[-1] + 1)
# Note that because the system's GID mapping can change mid-test, tests that
# check for specific errors should always re-fetch these GID-related values,
# rather than reusing these LazyWrapper values.
INVALID_GID = LazyWrapper(lambda: GetInvalidGid())
NON_PRIMARY_GID = LazyWrapper(lambda: GetNonPrimaryGid())
PRIMARY_GID = LazyWrapper(lambda: GetPrimaryGid())
# Get a list of all groups on the system where the current username is listed
# as a member of the group in the gr_mem group attribute. Make this a list of
# all group IDs and cast as a set for more efficient lookup times.
from decimal import Decimal
import re
from gslib.exception import CommandException
from gslib.lazy_wrapper import LazyWrapper
from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
SECONDS_IN_DAY = 24 * 60 * 60
SECONDS_IN_MONTH = 31 * SECONDS_IN_DAY
SECONDS_IN_YEAR = int(365.25 * SECONDS_IN_DAY)
_LOCK_PROMPT = (
'This will PERMANENTLY set the Retention Policy on gs://{} to:\n\n'
'{}\n\nThis setting cannot be reverted! Continue?')
# Regex to match retention period in years.
_RETENTION_IN_YEARS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)y$'))
# Regex to match retention period in months.
_RETENTION_IN_MONTHS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)m$'))
# Regex to match retention period in days.
_RETENTION_IN_DAYS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)d$'))
# Regex to match retention period in seconds.
_RETENTION_IN_SECONDS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)s$'))
def _ConfirmWithUserPrompt(question, default_response):
"""Prompts user to confirm an action with yes or no response.
Args:
question: Yes/No question to be used for the prompt.
default_response: Default response to the question: True, False
from gslib.utils.system_util import IS_CP1252
if six.PY3:
long = int
STORAGE_CLASS_SHORTHAND_TO_FULL_NAME = {
# Values should remain uppercase, as required by non-gs providers.
'CL': 'COLDLINE',
'DRA': 'DURABLE_REDUCED_AVAILABILITY',
'NL': 'NEARLINE',
'S': 'STANDARD',
'STD': 'STANDARD',
'A': 'ARCHIVE',
}
VERSION_MATCHER = LazyWrapper(
lambda: re.compile(r'^(?P<maj>\d+)(\.(?P<min>\d+)(?P<suffix>.*))?'))
def AddQueryParamToUrl(url_str, param_name, param_value):
"""Adds a query parameter to a URL string.
Appends a query parameter to the query string portion of a url. If a parameter
with the given name was already present, it is not removed; the new name/value
pair will be appended to the end of the query string. It is assumed that all
arguments will be of type `str` (either ASCII or UTF-8 encoded) or `unicode`.
Note that this method performs no URL-encoding. It is the caller's
responsibility to ensure proper URL encoding of the entire URL; i.e. if the
URL is already URL-encoded, you should pass in URL-encoded values for
param_name and param_value. If the URL is not URL-encoded, you should not pass
in URL-encoded parameters; instead, you could perform URL-encoding using the
from gslib.cloud_api import BadRequestException
from gslib.lazy_wrapper import LazyWrapper
from gslib.progress_callback import ProgressCallbackWithTimeout
from gslib.utils.constants import DEBUGLEVEL_DUMP_REQUESTS
from gslib.utils.constants import SSL_TIMEOUT_SEC
from gslib.utils.constants import TRANSFER_BUFFER_SIZE
from gslib.utils.constants import UTF8
from gslib.utils import text_util
import httplib2
from httplib2 import parse_uri
if six.PY3:
long = int
# A regex for matching any series of decimal digits.
DECIMAL_REGEX = LazyWrapper(lambda: (re.compile(r'\d+')))
class BytesTransferredContainer(object):
"""Container class for passing number of bytes transferred to lower layers.
For resumed transfers or connection rebuilds in the middle of a transfer, we
need to rebuild the connection class with how much we've transferred so far.
For uploads, we don't know the total number of bytes uploaded until we've
queried the server, but we need to create the connection class to pass to
httplib2 before we can query the server. This container object allows us to
pass a reference into Upload/DownloadCallbackConnection.
"""
def __init__(self):
self.__bytes_transferred = 0
