def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages template_ref = args.CONCEPTS.template.Parse() # TODO(b/109837200) make the dataproc discovery doc parameters consistent # Parent() fails for the collection because of projectId/projectsId and # regionId/regionsId inconsistencies. # parent = template_ref.Parent().RelativePath() parent = '/'.join(template_ref.RelativeName().split('/')[0:4]) if args.source: with files.FileReader(args.source) as stream: template = util.ReadYaml(message_type=msgs.WorkflowTemplate, stream=stream, schema_path=SCHEMA_PATH) else: template = util.ReadYaml(message_type=msgs.WorkflowTemplate, stream=sys.stdin, schema_path=SCHEMA_PATH) # Populate id field. template.id = template_ref.Name() try: old_template = dataproc.GetRegionsWorkflowTemplate(template_ref) except apitools_exceptions.HttpError as error: if error.status_code != 404: raise error # Template does not exist. Create a new one. request = msgs.DataprocProjectsRegionsWorkflowTemplatesCreateRequest( parent=parent, workflowTemplate=template) return dataproc.client.projects_regions_workflowTemplates.Create( request) # Update the existing template. console_io.PromptContinue( message=('Workflow template [{0}] will be overwritten.').format( template.id), cancel_on_no=True) # Populate version field and name field. template.version = old_template.version template.name = template_ref.RelativeName() return dataproc.client.projects_regions_workflowTemplates.Update( template)
def _GetTemplateFromFile(self, args, messages): if not os.path.exists(args.file_name): raise exceptions.BadFileException('No such file [{0}]'.format( args.file_name)) if os.path.isdir(args.file_name): raise exceptions.BadFileException('[{0}] is a directory'.format( args.file_name)) try: with files.FileReader(args.file_name) as import_file: if args.file_format == 'json': return security_policies_utils.SecurityPolicyFromFile( import_file, messages, 'json') return security_policies_utils.SecurityPolicyFromFile( import_file, messages, 'yaml') except Exception as exp: exp_msg = getattr(exp, 'message', six.text_type(exp)) msg = ('Unable to read security policy config from specified file ' '[{0}] because [{1}]'.format(args.file_name, exp_msg)) raise exceptions.BadFileException(msg)
def _FindMain(filename): """Check filename for 'package main' and 'func main'. Args: filename: (str) File name to check. Returns: (bool) True if main is found in filename. """ with files.FileReader(filename) as f: found_package = False found_func = False for line in f: if re.match('^package main', line): found_package = True elif re.match('^func main', line): found_func = True if found_package and found_func: return True return False
def LoadBuildArtifactFile(path): """Load images from a file containing JSON build data. Args: path: str, build artifacts file path. Returns: Docker image name and tag dictionary. """ with files.FileReader(path) as f: # Returns user-friendly error messages try: structured_data = yaml.load(f, file_hint=path) except yaml.Error as e: raise exceptions.ParserError(path, e.inner_error) images = {} for build in structured_data['builds']: # For b/191063894. Supporting both name for now. images[build.get('image', build.get('imageName'))] = build['tag'] return images
def get_disk_counters(): """Retrieves disk I/O statistics for all disks. Adapted from the psutil module's psutil._pslinux.disk_io_counters: http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py Originally distributed under under a BSD license. Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola. Returns: A dictionary containing disk names mapped to the disk counters from /disk/diskstats. """ # iostat documentation states that sectors are equivalent with blocks and # have a size of 512 bytes since 2.4 kernels. This value is needed to # calculate the amount of disk I/O in bytes. sector_size = 512 partitions = _get_partitions() retdict = {} try: with files.FileReader('/proc/diskstats') as f: lines = f.readlines() for line in lines: values = line.split()[:11] _, _, name, reads, _, rbytes, rtime, writes, _, wbytes, wtime = values if name in partitions: rbytes = int(rbytes) * sector_size wbytes = int(wbytes) * sector_size reads = int(reads) writes = int(writes) rtime = int(rtime) wtime = int(wtime) retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime) # This will catch access denied and file not found errors, which is expected # on non-Linux/limited access systems. All other errors will raise as normal. except files.Error: pass return retdict
def Run(self, args): dataproc = dp.Dataproc(self.ReleaseTrack()) msgs = dataproc.messages if args.source: with files.FileReader(args.source) as stream: cluster = util.ReadYaml(message_type=msgs.Cluster, stream=stream, schema_path=SCHEMA_PATH) else: cluster = util.ReadYaml(message_type=msgs.Cluster, stream=sys.stdin, schema_path=SCHEMA_PATH) cluster_ref = util.ParseCluster(args.name, dataproc) cluster.clusterName = cluster_ref.clusterName cluster.projectId = cluster_ref.projectId # Import only supports create, not update (for now). return clusters.CreateCluster(dataproc, cluster, args. async, args.timeout)
def load_path(path, round_trip=False): # type: (str, typing.Optional[bool]) -> typing.Any """Loads YAML from the given file path. Args: path: str, A file path to open and read from. round_trip: bool, True to use the RoundTripLoader which preserves ordering and line numbers. Raises: YAMLParseError: If the data could not be parsed. FileLoadError: If the file could not be opened or read. Returns: The parsed YAML data. """ try: with files.FileReader(path) as fp: return load(fp, file_hint=path, round_trip=round_trip) except files.Error as e: raise FileLoadError(e, f=path)
def load_all_path(path): """Loads multiple YAML documents from the given file path. Args: path: str, A file path to open and read from. Raises: YAMLParseError: If the data could not be parsed. FileLoadError: If the file could not be opened or read. Yields: The parsed YAML data. """ try: with files.FileReader(path) as fp: for x in load_all(fp, file_hint=path): yield x except files.Error as e: # EnvironmentError is parent of IOError, OSError and WindowsError. # Raised when file does not exist or can't be opened/read. raise FileLoadError(e, f=path)
def _get_partitions(): """Retrieves a list of disk partitions. Returns: An array of partition names as strings. """ partitions = [] try: with files.FileReader('/proc/partitions') as f: lines = f.readlines()[2:] for line in lines: _, _, _, name = line.split() if name[-1].isdigit(): partitions.append(name) # This will catch access denied and file not found errors, which is expected # on non-Linux/limited access systems. All other errors will raise as normal. except files.Error: pass return partitions
def ParseRequirementsFile(requirements_file_path): """Parses the given requirements file into a requirements dictionary. Expects the file to have one requirement specifier per line. Only performs lightweight parsing of the file in order to form an API call. The Composer frontend handles validation. Args: requirements_file_path: Filepath to the requirements file. Returns: {string: string}, dict mapping from PyPI package name to extras and version specifier, if provided. Raises: Error: if requirements file cannot be read. """ requirements = {} try: with files.FileReader(requirements_file_path) as requirements_file: for requirement_specifier in requirements_file: requirement_specifier = requirement_specifier.strip() if not requirement_specifier: continue package, version = SplitRequirementSpecifier( requirement_specifier) # Ensure package not already in entry list. if package in requirements: raise Error( 'Duplicate package in requirements file: {0}'.format( package)) requirements[package] = version return requirements except files.Error: # EnvironmentError is parent of IOError, OSError and WindowsError. # Raised when file does not exist or can't be opened/read. raise Error('Unable to read requirements file {0}'.format( requirements_file_path))
def _ReadTagMapping(file_name): """Imports legacy to secure tag mapping from a JSON file.""" try: with files.FileReader(file_name) as f: data = json.load(f) except FileNotFoundError: log.status.Print( 'File \'{file}\' was not found. Tag mapping was not imported.'. format(file=file_name)) return None except OSError: log.status.Print( 'OS error occurred when opening the file \'{file}\'. Tag mapping was not imported.' .format(file=file_name)) return None except Exception as e: # pylint: disable=broad-except log.status.Print( 'Unexpected error occurred when reading the JSON file \'{file}\'. Tag mapping was not imported.' .format(file=file_name)) log.status.Print(repr(e)) return None return data
def get_values_for_keys_from_file(file_path, keys): """Reads JSON or INI file and returns dict with values for requested keys. JSON file keys should be top level. INI file sections will be flattened. Args: file_path (str): Path of JSON or INI file to read. keys (list[str]): Search for these keys to return from file. Returns: Dict[cred_key: cred_value]. Raises: ValueError: The file was the incorrect format. KeyError: Duplicate key found. """ result = {} real_path = os.path.realpath(os.path.expanduser(file_path)) with files.FileReader(real_path) as file_reader: try: file_dict = json.loads(file_reader.read()) _extract_keys(keys, file_dict, result) except json.JSONDecodeError: # More file formats to try before raising error. config = configparser.ConfigParser() try: config.read(real_path) except configparser.ParsingError: raise ValueError( 'Source creds file must be JSON or INI format.') # Parse all sections of INI file into dict. for section in config: section_dict = dict(config[section]) _extract_keys(keys, section_dict, result) return result
def Run(self, args): if not os.path.exists(args.file_name): raise exceptions.BadFileException('No such file [{0}]'.format( args.file_name)) if os.path.isdir(args.file_name): raise exceptions.BadFileException('[{0}] is a directory'.format( args.file_name)) holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) ref = self.SECURITY_POLICY_ARG.ResolveAsResource( args, holder.resources) # Get the imported security policy config. try: with files.FileReader(args.file_name) as import_file: if args.file_format == 'json': imported = security_policies_utils.SecurityPolicyFromFile( import_file, holder.client.messages, 'json') else: imported = security_policies_utils.SecurityPolicyFromFile( import_file, holder.client.messages, 'yaml') except Exception as exp: exp_msg = getattr(exp, 'message', six.text_type(exp)) msg = ( 'Unable to read security policy config from specified file [{0}] ' 'because [{1}]'.format(args.file_name, exp_msg)) raise exceptions.BadFileException(msg) # Send the change to the service. security_policy = client.SecurityPolicy(ref, compute_client=holder.client) security_policy.Patch(security_policy=imported) msg = 'Updated [{0}] with config from [{1}].'.format( ref.Name(), args.file_name) log.status.Print(msg)
def LoadMessageFromPath(path, msg_type, msg_friendly_name, skip_camel_case=None): """Load a proto message from a file containing JSON or YAML text. Args: path: The path to a file containing the JSON or YAML data to be decoded. msg_type: The protobuf message type to create. msg_friendly_name: A readable name for the message type, for use in error messages. skip_camel_case: Contains proto field names or map keys whose values should not have camel case applied. Raises: files.MissingFileError: If the file does not exist. ParserError: If there was a problem parsing the file as a dict. ParseProtoException: If there was a problem interpreting the file as the given message type. Returns: Proto message, The message that got decoded. """ with files.FileReader(path) as f: # Returns user-friendly error messages return LoadMessageFromStream(f, msg_type, msg_friendly_name, skip_camel_case, path)
def FromJson(cls, path): with files.FileReader(path) as f: return cls(json.load(f, object_pairs_hook=OrderedDict))
def Run(self, args): message = ( 'A personal authentication session will propagate your personal ' 'credentials to the cluster, so make sure you trust the cluster ' 'and the user who created it.') console_io.PromptContinue( message=message, cancel_on_no=True, cancel_string='Enabling session aborted by user') dataproc = dp.Dataproc(self.ReleaseTrack()) cluster_ref = args.CONCEPTS.cluster.Parse() project = cluster_ref.projectId region = cluster_ref.region cluster_name = cluster_ref.clusterName get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest( projectId=project, region=region, clusterName=cluster_name) cluster = dataproc.client.projects_regions_clusters.Get(get_request) cluster_uuid = cluster.clusterUuid if args.access_boundary: with files.FileReader(args.access_boundary, mode='r') as abf: access_boundary_json = abf.read() else: access_boundary_json = flags.ProjectGcsObjectsAccessBoundary( project) openssl_executable = args.openssl_command if not openssl_executable: try: openssl_executable = files.FindExecutableOnPath('openssl') except ValueError: log.fatal( 'Could not find openssl on your system. The enable-session ' 'command requires openssl to be installed.') operation_poller = waiter.CloudOperationPollerNoResources( dataproc.client.projects_regions_operations, lambda operation: operation.name) try: cluster_key = clusters.ClusterKey(cluster) if not cluster_key: raise exceptions.PersonalAuthError( 'The cluster {} does not support personal auth.'.format( cluster_name)) with progress_tracker.ProgressTracker( 'Injecting initial credentials into the cluster {}'.format( cluster_name), autotick=True): self.inject_credentials(dataproc, project, region, cluster_name, cluster_uuid, cluster_key, access_boundary_json, openssl_executable, operation_poller) if not args.refresh_credentials: return update_message = ( 'Periodically refreshing credentials for cluster {}. This' ' will continue running until the command is interrupted' ).format(cluster_name) with progress_tracker.ProgressTracker(update_message, autotick=True): try: # Cluster keys are periodically regenerated, so fetch the latest # each time we inject credentials. cluster = dataproc.client.projects_regions_clusters.Get( get_request) cluster_key = clusters.ClusterKey(cluster) if not cluster_key: raise exceptions.PersonalAuthError( 'The cluster {} does not support personal auth.'. format(cluster_name)) failure_count = 0 while failure_count < 3: try: time.sleep(30) self.inject_credentials(dataproc, project, region, cluster_name, cluster_uuid, cluster_key, access_boundary_json, openssl_executable, operation_poller) failure_count = 0 except ValueError as err: log.error(err) failure_count += 1 raise exceptions.PersonalAuthError( 'Credential injection failed three times in a row, giving up...' ) except (console_io.OperationCancelledError, KeyboardInterrupt): return except exceptions.PersonalAuthError as err: log.error(err) return
def GetServices(): with files.FileReader(_SERVICE_CATALOG_PATH) as f: catalog = json.load(f) return catalog['services']
def FromJson(cls, path): with files.FileReader(path) as f: return cls(json.load(f))
def read_or_create_download_tracker_file(source_object_resource, destination_url, slice_start_byte=0, existing_file_size=0, component_number=None, create=True): """Checks for a download tracker file and creates one if it does not exist. For normal downloads, if the tracker file exists, the existing_file_size in bytes is presumed to downloaded from the server. Therefore, existing_file_size becomes the download start point. For sliced downloads, the number of bytes previously retrieved from the server cannot be determined from existing_file_size. Therefore, it is retrieved from the tracker file. Args: source_object_resource (resource_reference.ObjectResource): Needed for object etag and generation. destination_url (storage_url.StorageUrl): Destination URL for tracker file. slice_start_byte (int): Start byte to use if we cannot find a matching tracker file for a download slice. existing_file_size (int): Amount of file on disk that already exists. component_number (int?): The download component number to find the start point for. create (bool): Creates tracker file if one could not be found. Returns: tracker_file_path (str?): The path to the tracker file, if one was used. download_start_byte (int): The first byte that still needs to be downloaded. Raises: ValueCannotBeDeterminedError: Source object resource does not have necessary metadata to decide on download start byte. """ if not source_object_resource.etag: raise errors.ValueCannotBeDeterminedError( 'Source object resource is missing etag.') tracker_file_path = None if (not source_object_resource.size or (source_object_resource.size < properties.VALUES.storage.resumable_threshold.GetInt())): # There is no tracker file for small downloads, so start from scratch. return tracker_file_path, slice_start_byte if component_number is None: tracker_file_type = TrackerFileType.DOWNLOAD download_name_for_logger = destination_url.object_name else: tracker_file_type = TrackerFileType.DOWNLOAD_COMPONENT download_name_for_logger = '{} component {}'.format( destination_url.object_name, component_number) tracker_file_path = get_tracker_file_path( destination_url, tracker_file_type, component_number=component_number) tracker_file = None # Check to see if we already have a matching tracker file. try: tracker_file = files.FileReader(tracker_file_path) if tracker_file_type is TrackerFileType.DOWNLOAD: etag_value = tracker_file.readline().rstrip('\n') if etag_value == source_object_resource.etag: log.debug( 'Found tracker file starting at byte {} for {}.'.format( existing_file_size, download_name_for_logger)) return tracker_file_path, existing_file_size elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: component_data = json.loads(tracker_file.read()) if (component_data['etag'] == source_object_resource.etag and component_data['generation'] == source_object_resource.generation): start_byte = int(component_data['download_start_byte']) log.debug( 'Found tracker file starting at byte {} for {}.'.format( start_byte, download_name_for_logger)) return tracker_file_path, start_byte except files.MissingFileError: # Cannot read from file. pass finally: if tracker_file: tracker_file.close() if create: log.debug('No matching tracker file for {}.'.format( download_name_for_logger)) if tracker_file_type is TrackerFileType.DOWNLOAD: _write_tracker_file(tracker_file_path, source_object_resource.etag + '\n') elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: write_download_component_tracker_file(tracker_file_path, source_object_resource, slice_start_byte) # No matching tracker file, so starting point is slice_start_byte. return tracker_file_path, slice_start_byte
def _CreateGoogleAuthClientConfig(client_id_file=None): """Creates a client config from a client id file or gcloud's properties.""" if client_id_file: with files.FileReader(client_id_file) as f: return json.load(f) return _CreateGoogleAuthClientConfigFromProperties()
def Run(self, args): client = apis.GetClientInstance('storagetransfer', 'v1') messages = apis.GetMessagesModule('storagetransfer', 'v1') if args.creds_file: expanded_file_path = os.path.abspath(os.path.expanduser(args.creds_file)) with files.FileReader(expanded_file_path) as file_reader: try: parsed_creds_file = json.load(file_reader) account_email = parsed_creds_file['client_email'] is_service_account = parsed_creds_file['type'] == 'service_account' except (ValueError, KeyError) as e: log.error(e) raise ValueError('Invalid creds file format.' ' Run command with "--help" flag for more details.') prefixed_account_email = _get_iam_prefixed_email( account_email, is_service_account) else: account_email = properties.VALUES.core.account.Get() is_service_account = creds.IsServiceAccountCredentials(creds_store.Load()) prefixed_account_email = _get_iam_prefixed_email(account_email, is_service_account) project_id = properties.VALUES.core.project.Get() parsed_project_id = projects_util.ParseProject(project_id) project_iam_policy = projects_api.GetIamPolicy(parsed_project_id) existing_user_roles = _get_existing_transfer_roles_for_account( project_iam_policy, prefixed_account_email, EXPECTED_USER_ROLES) log.status.Print('User {} has roles:\n{}'.format(account_email, list(existing_user_roles))) missing_user_roles = EXPECTED_USER_ROLES - existing_user_roles log.status.Print('Missing roles:\n{}'.format(list(missing_user_roles))) all_missing_role_tuples = [ (prefixed_account_email, role) for role in missing_user_roles ] log.status.Print('***') transfer_p4sa_email = client.googleServiceAccounts.Get( messages.StoragetransferGoogleServiceAccountsGetRequest( projectId=project_id)).accountEmail prefixed_transfer_p4sa_email = _get_iam_prefixed_email( transfer_p4sa_email, is_service_account=True) existing_p4sa_roles = _get_existing_transfer_roles_for_account( project_iam_policy, prefixed_transfer_p4sa_email, EXPECTED_P4SA_ROLES) log.status.Print('Google-managed transfer account {} has roles:\n{}'.format( transfer_p4sa_email, list(existing_p4sa_roles))) missing_p4sa_roles = EXPECTED_P4SA_ROLES - existing_p4sa_roles log.status.Print('Missing roles:\n{}'.format(list(missing_p4sa_roles))) all_missing_role_tuples += [ (prefixed_transfer_p4sa_email, role) for role in missing_p4sa_roles ] if args.add_missing or all_missing_role_tuples: log.status.Print('***') if args.add_missing: if all_missing_role_tuples: log.status.Print('Adding roles:\n{}'.format(all_missing_role_tuples)) projects_api.AddIamPolicyBindings(parsed_project_id, all_missing_role_tuples) log.status.Print('***') # Source: # https://cloud.google.com/iam/docs/granting-changing-revoking-access log.status.Print( 'Done. Permissions typically take seconds to propagate, but,' ' in some cases, it can take up to seven minutes.') else: log.status.Print('No missing roles to add.') else: log.status.Print('Rerun with --add-missing to add missing roles.')
def Run(self, args): api_version = 'v1' # If in the future there are differences between API version, do NOT use # this patter of checking ReleaseTrack. Break this into multiple classes. if self.ReleaseTrack() == base.ReleaseTrack.BETA: api_version = 'v1beta2' if not os.path.exists(args.records_file): raise exceptions.ToolException('no such file [{0}]'.format( args.records_file)) if os.path.isdir(args.records_file): raise exceptions.ToolException('[{0}] is a directory'.format( args.records_file)) dns = apis.GetClientInstance('dns', api_version) # Get the managed-zone. zone_ref = util.GetRegistry(api_version).Parse( args.zone, params={ 'project': properties.VALUES.core.project.GetOrFail, }, collection='dns.managedZones') try: zone = dns.managedZones.Get( dns.MESSAGES_MODULE.DnsManagedZonesGetRequest( project=zone_ref.project, managedZone=zone_ref.managedZone)) except apitools_exceptions.HttpError as error: raise exceptions.HttpException(error) # Get the current record-sets. current = {} for record in list_pager.YieldFromList( dns.resourceRecordSets, dns.MESSAGES_MODULE.DnsResourceRecordSetsListRequest( project=zone_ref.project, managedZone=zone_ref.Name()), field='rrsets'): current[(record.name, record.type)] = record # Get the imported record-sets. try: with files.FileReader(args.records_file) as import_file: if args.zone_file_format: imported = import_util.RecordSetsFromZoneFile( import_file, zone.dnsName, api_version=api_version) else: imported = import_util.RecordSetsFromYamlFile( import_file, api_version=api_version) except Exception as exp: msg = ( 'unable to read record-sets from specified records-file [{0}] ' 'because [{1}]') msg = msg.format(args.records_file, exp.message) raise exceptions.ToolException(msg) # Get the change resulting from the imported record-sets. change = import_util.ComputeChange(current, imported, args.delete_all_existing, zone.dnsName, args.replace_origin_ns, api_version=api_version) if not change: msg = 'Nothing to do, all the records in [{0}] already exist.'.format( args.records_file) log.status.Print(msg) return None # Send the change to the service. result = dns.changes.Create( dns.MESSAGES_MODULE.DnsChangesCreateRequest( change=change, managedZone=zone.name, project=zone_ref.project)) change_ref = util.GetRegistry(api_version).Create( collection='dns.changes', project=zone_ref.project, managedZone=zone.name, changeId=result.id) msg = 'Imported record-sets from [{0}] into managed-zone [{1}].'.format( args.records_file, zone_ref.Name()) log.status.Print(msg) log.CreatedResource(change_ref) return result
def read_or_create_download_tracker_file(source_object_resource, destination_url, slice_start_byte=None, component_number=None, total_components=None): """Checks for a download tracker file and creates one if it does not exist. Args: source_object_resource (resource_reference.ObjectResource): Needed for object etag and generation. destination_url (storage_url.StorageUrl): Destination URL for tracker file. slice_start_byte (int|None): Start byte to use if we cannot find a matching tracker file for a download slice. component_number (int|None): The download component number to find the start point for. Indicates part of a multi-component download. total_components (int|None): The number of components in a sliced download. Indicates this is the parent tracker for a multi-component operation. Returns: tracker_file_path (str): The path to the tracker file (found or created). found_tracker_file (bool): False if tracker file had to be created. Raises: ValueCannotBeDeterminedError: Source object resource does not have necessary metadata to decide on download start byte. """ if not source_object_resource.etag: raise errors.ValueCannotBeDeterminedError( 'Source object resource is missing etag.') if total_components and (slice_start_byte is not None or component_number is not None): raise ValueError( 'total_components indicates this is the parent tracker file for a' ' multi-component operation. slice_start_byte and component_number' ' cannot be present since this is not for an individual component.' ) if component_number is not None: download_name_for_logger = '{} component {}'.format( destination_url.object_name, component_number) tracker_file_type = TrackerFileType.DOWNLOAD_COMPONENT else: download_name_for_logger = destination_url.object_name if total_components is not None: tracker_file_type = TrackerFileType.SLICED_DOWNLOAD else: tracker_file_type = TrackerFileType.DOWNLOAD tracker_file_path = get_tracker_file_path( destination_url, tracker_file_type, component_number=component_number) log.debug('Searching for tracker file at {}.'.format(tracker_file_path)) tracker_file = None does_tracker_file_match = False # Check to see if we already have a matching tracker file. try: tracker_file = files.FileReader(tracker_file_path) if tracker_file_type is TrackerFileType.DOWNLOAD: etag_value = tracker_file.readline().rstrip('\n') if etag_value == source_object_resource.etag: does_tracker_file_match = True else: component_data = json.loads(tracker_file.read()) if (component_data['etag'] == source_object_resource.etag and component_data['generation'] == source_object_resource.generation): if (tracker_file_type is TrackerFileType.SLICED_DOWNLOAD and component_data['total_components'] == total_components): does_tracker_file_match = True elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT and component_data[ 'slice_start_byte'] == slice_start_byte: does_tracker_file_match = True if does_tracker_file_match: log.debug( 'Found tracker file for {}.'.format(download_name_for_logger)) return tracker_file_path, True except files.MissingFileError: # Cannot read from file. pass finally: if tracker_file: tracker_file.close() if tracker_file: # The tracker file exists, but it's not valid. delete_download_tracker_files(destination_url) log.debug( 'No matching tracker file for {}.'.format(download_name_for_logger)) if tracker_file_type is TrackerFileType.DOWNLOAD: _write_tracker_file(tracker_file_path, source_object_resource.etag + '\n') elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: write_tracker_file_with_component_data( tracker_file_path, source_object_resource, slice_start_byte=slice_start_byte) elif tracker_file_type is TrackerFileType.SLICED_DOWNLOAD: write_tracker_file_with_component_data( tracker_file_path, source_object_resource, total_components=total_components) return tracker_file_path, False
def Run(self, args): api_version = util.GetApiFromTrackAndArgs(self.ReleaseTrack(), args) if not os.path.exists(args.records_file): raise import_util.RecordsFileNotFound( 'Specified record file [{0}] not found.'.format( args.records_file)) if os.path.isdir(args.records_file): raise import_util.RecordsFileIsADirectory( 'Specified record file [{0}] is a directory'.format( args.records_file)) dns = util.GetApiClient(api_version) # Get the managed-zone. zone_ref = util.GetRegistry(api_version).Parse( args.zone, params=util.GetParamsForRegistry(api_version, args), collection='dns.managedZones') try: get_request = dns.MESSAGES_MODULE.DnsManagedZonesGetRequest( project=zone_ref.project, managedZone=zone_ref.managedZone) if api_version == 'v2' and self._IsBetaOrAlpha(): get_request.location = args.location zone = dns.managedZones.Get(get_request) except apitools_exceptions.HttpError as error: raise calliope_exceptions.HttpException(error) # Get the current record-sets. current = {} list_request = dns.MESSAGES_MODULE.DnsResourceRecordSetsListRequest( project=zone_ref.project, managedZone=zone_ref.Name()) if api_version == 'v2': list_request.location = args.location for record in list_pager.YieldFromList(dns.resourceRecordSets, list_request, field='rrsets'): current[(record.name, record.type)] = record # Get the imported record-sets. try: with files.FileReader(args.records_file) as import_file: if args.zone_file_format: imported = import_util.RecordSetsFromZoneFile( import_file, zone.dnsName, api_version=api_version) else: imported = import_util.RecordSetsFromYamlFile( import_file, include_extended_records=self._IsAlpha(), api_version=api_version) except Exception as exp: msg = ( 'Unable to read record-sets from specified records-file [{0}] ' 'because [{1}]') msg = msg.format(args.records_file, exp.message) raise import_util.UnableToReadRecordsFile(msg) # Get the change resulting from the imported record-sets. change = import_util.ComputeChange(current, imported, args.delete_all_existing, zone.dnsName, args.replace_origin_ns, api_version=api_version) if not change: msg = 'Nothing to do, all the records in [{0}] already exist.'.format( args.records_file) log.status.Print(msg) return None # Send the change to the service. create_request = dns.MESSAGES_MODULE.DnsChangesCreateRequest( change=change, managedZone=zone.name, project=zone_ref.project) if api_version == 'v2' and self._IsBetaOrAlpha(): create_request.location = args.location result = dns.changes.Create(create_request) param = util.GetParamsForRegistry(api_version, args, parent='managedZones') param['changeId'] = result.id change_ref = util.GetRegistry(api_version).Parse( line=None, collection='dns.changes', params=param) msg = 'Imported record-sets from [{0}] into managed-zone [{1}].'.format( args.records_file, zone_ref.Name()) log.status.Print(msg) log.CreatedResource(change_ref) return result
def cached_read_json_file(file_path): """Convert JSON file to an in-memory dict.""" with files.FileReader(file_path) as file_reader: return json.load(file_reader)
def GetADCAsJson(): """Reads ADC from disk and converts it to a json object.""" if not os.path.isfile(config.ADCFilePath()): return None with files.FileReader(config.ADCFilePath()) as f: return json.load(f)
def read_or_create_download_tracker_file(source_object_resource, destination_url, existing_file_size=None, slice_start_byte=None, component_number=None, total_components=None, create=True): """Checks for a download tracker file and creates one if it does not exist. For normal downloads, if the tracker file exists, the existing_file_size in bytes is presumed to downloaded from the server. Therefore, existing_file_size becomes the download start point. For sliced downloads, the number of bytes previously retrieved from the server cannot be determined from existing_file_size. Therefore, it is retrieved from the tracker file. Args: source_object_resource (resource_reference.ObjectResource): Needed for object etag and generation. destination_url (storage_url.StorageUrl): Destination URL for tracker file. existing_file_size (int): Amount of file on disk that already exists. slice_start_byte (int|None): Start byte to use if we cannot find a matching tracker file for a download slice. component_number (int|None): The download component number to find the start point for. Indicates part of a multi-component download. total_components (int|None): The number of components in a sliced download. Indicates this is the master tracker for a multi-component operation. create (bool): Creates tracker file if one could not be found. Returns: tracker_file_path (str|None): The path to the tracker file, if one was used. download_start_byte (int|None): The first byte that still needs to be downloaded, if not a sliced download. Raises: ValueCannotBeDeterminedError: Source object resource does not have necessary metadata to decide on download start byte. """ if not source_object_resource.etag: raise errors.ValueCannotBeDeterminedError( 'Source object resource is missing etag.') if total_components and (slice_start_byte is not None or component_number is not None): raise ValueError( 'total_components indicates this is the master tracker file for a' ' multi-component operation. slice_start_byte and component_number' ' cannot be present since this is not for an individual component.' ) if component_number: download_name_for_logger = '{} component {}'.format( destination_url.object_name, component_number) tracker_file_type = TrackerFileType.DOWNLOAD_COMPONENT else: download_name_for_logger = destination_url.object_name if total_components: tracker_file_type = TrackerFileType.SLICED_DOWNLOAD else: tracker_file_type = TrackerFileType.DOWNLOAD tracker_file_path = get_tracker_file_path( destination_url, tracker_file_type, component_number=component_number) tracker_file = None # Check to see if we already have a matching tracker file. try: tracker_file = files.FileReader(tracker_file_path) if tracker_file_type is TrackerFileType.DOWNLOAD: etag_value = tracker_file.readline().rstrip('\n') if etag_value == source_object_resource.etag: log.debug( 'Found tracker file starting at byte {} for {}.'.format( existing_file_size, download_name_for_logger)) return tracker_file_path, existing_file_size else: component_data = json.loads(tracker_file.read()) if (component_data['etag'] == source_object_resource.etag and component_data['generation'] == source_object_resource.generation): if (tracker_file_type is TrackerFileType.SLICED_DOWNLOAD and component_data['total_components'] == total_components): log.debug( 'Found tracker file for sliced download {}.'.format( download_name_for_logger)) return tracker_file_path, None elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: # Normal resumable download. start_byte = int(component_data['download_start_byte']) log.debug('Found tracker file starting at byte {} for {}.'. format(start_byte, download_name_for_logger)) return tracker_file_path, start_byte except files.MissingFileError: # Cannot read from file. pass finally: if tracker_file: tracker_file.close() log.debug( 'No matching tracker file for {}.'.format(download_name_for_logger)) start_byte = 0 if create: if tracker_file_type is TrackerFileType.DOWNLOAD: _write_tracker_file(tracker_file_path, source_object_resource.etag + '\n') elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: write_tracker_file_with_component_data( tracker_file_path, source_object_resource, download_start_byte=slice_start_byte) start_byte = slice_start_byte elif tracker_file_type is TrackerFileType.SLICED_DOWNLOAD: # Delete component tracker files to reset full sliced download. delete_download_tracker_files(destination_url) write_tracker_file_with_component_data( tracker_file_path, source_object_resource, total_components=total_components) start_byte = None return tracker_file_path, start_byte