def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Raises: ToolException: when destination uri is not specified or invalid. """ apitools_client = self.context[commands.APITOOLS_CLIENT_KEY] bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY] resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY] project_id = properties.VALUES.core.project.Get(required=True) source_table_resource = resource_parser.Parse( args.source_table, collection='bigquery.tables') source_table_reference = message_conversions.TableResourceToReference( bigquery_messages, source_table_resource) if not args.destination_uri: raise exceptions.ToolException( 'At least one destination URI must be specified.') destination_uris = args.destination_uri for uri in destination_uris: if not uri.startswith('gs://'): raise exceptions.ToolException(( 'Illegal URI: {0}. Only Google Storage ("gs://") URIs are ' 'supported.').format(uri)) job = job_control.ExecuteJob( apitools_client, bigquery_messages, args, configuration=bigquery_messages.JobConfiguration( extract=bigquery_messages.JobConfigurationExtract( sourceTable=source_table_reference, destinationUris=destination_uris, destinationFormat=bigquery_client_helper. NormalizeTextualFormat(args.destination_format), fieldDelimiter=bigquery_client_helper. NormalizeFieldDelimiter(args.field_delimiter))), async=args. async, project_id=project_id, job_id=job_ids.JobIdProvider().GetJobId(args.job_id, args.fingerprint_job_id)) if args. async: job_resource = resource_parser.Create( 'bigquery.jobs', projectId=job.jobReference.projectId, jobId=job.jobReference.jobId) log.CreatedResource(job_resource)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Raises: ToolException: if no query was provided. Returns: If the --dry_run or --async flag was specified, None; otherwise, a SchemaAndRows object. """ apitools_client = self.context[commands.APITOOLS_CLIENT_KEY] bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY] resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY] project_id = properties.VALUES.core.project.Get(required=True) if not args.sql_query: raise exceptions.ToolException('No query string provided') destination_table = args.append_to or args.write_to if destination_table: output_resource = resource_parser.Parse( destination_table, collection='bigquery.tables') output_reference = message_conversions.TableResourceToReference( bigquery_messages, output_resource) else: output_reference = None query_configuration = bigquery_messages.JobConfigurationQuery( allowLargeResults=args.allow_large_results, createDisposition='CREATE_NEVER' if args.require_cache else None, # Set defaultDataset here if we choose to support a # --default-dataset-in-query flag. destinationTable=output_reference, flattenResults=not args.structured, preserveNulls=None, priority='BATCH' if args.batch else None, query=args.sql_query, useQueryCache=args.use_cache, writeDisposition=((args.append_to and 'WRITE_APPEND') or (args.write_to and 'WRITE_TRUNCATE'))) job = job_control.ExecuteJob( apitools_client, bigquery_messages, args, configuration=bigquery_messages.JobConfiguration( query=query_configuration, dryRun=args.dry_run), async=args. async, project_id=project_id, job_id=job_ids.JobIdProvider().GetJobId(args.job_id, args.fingerprint_job_id)) if args.dry_run: log.Print( 'Query successfully validated. Assuming the tables are not ' 'modified, running this query will process {0} bytes of data.'. format(job.statistics.query.totalBytesProcessed)) return None if args. async: job_resource = resource_parser.Create( 'bigquery.jobs', projectId=job.jobReference.projectId, jobId=job.jobReference.jobId) log.CreatedResource(job_resource) self.default_format = 'table(jobId, projectId)' return job_resource result = schema_and_rows.GetJobSchemaAndRows(apitools_client, bigquery_messages, job.jobReference, args.start_row, args.limit) if not result: return None self.default_format = result.GetDefaultFormat() return result.PrepareForDisplay()
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Returns: """ apitools_client = self.context[commands.APITOOLS_CLIENT_KEY] bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY] resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY] project_id = properties.VALUES.core.project.Get(required=True) table_resource = resource_parser.Parse(args.destination_table, collection='bigquery.tables') # TODO(user): Define constants for collection names in one place table_reference = message_conversions.TableResourceToReference( bigquery_messages, table_resource) sources = _ProcessSources(args.source) if args.schema: table_schema = bigquery_schemas.ReadSchema(args.schema, bigquery_messages) elif args.schema_file: table_schema = bigquery_schemas.ReadSchemaFile( args.schema_file, bigquery_messages) else: table_schema = None normalized_source_format = bigquery_client_helper.NormalizeTextualFormat( args.source_format) if (not normalized_source_format) or normalized_source_format == 'CSV': normalized_quote = ( args.quote and bigquery_client_helper.NormalizeFieldDelimiter(args.quote)) normalized_skip_leading_rows = args.skip_leading_rows else: # Server accepts non-None quote and skipLeadingRows only for CSV source # format: normalized_quote = None normalized_skip_leading_rows = None load_config = bigquery_messages.JobConfigurationLoad( allowJaggedRows=args.allow_jagged_rows, allowQuotedNewlines=args.allow_quoted_newlines, destinationTable=table_reference, encoding=args.encoding and args.encoding.upper(), fieldDelimiter=(args.field_delimiter and bigquery_client_helper.NormalizeFieldDelimiter( args.field_delimiter)), ignoreUnknownValues=args.ignore_unknown_values, maxBadRecords=args.max_bad_records, quote=normalized_quote, schema=table_schema, skipLeadingRows=normalized_skip_leading_rows, sourceFormat=normalized_source_format, sourceUris=sources if sources[0].startswith('gs://') else [], writeDisposition='WRITE_TRUNCATE' if args.replace else None, ) job = job_control.ExecuteJob( apitools_client, bigquery_messages, args, configuration=bigquery_messages.JobConfiguration(load=load_config), async=args. async, project_id=project_id, upload_file=None if sources[0].startswith('gs://') else sources[0], job_id=job_ids.JobIdProvider().GetJobId(args.job_id, args.fingerprint_job_id)) if args. async: job_resource = resource_parser.Create( 'bigquery.jobs', projectId=job.jobReference.projectId, jobId=job.jobReference.jobId) log.CreatedResource(job_resource)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Returns: None Raises: bigqueryError.BigqueryError: If the source and destination files are not both specified. calliope_exceptions.ToolException: If user cancels this operation. Exception: If an unexpected value for the --if-exists flag passed gcloud validation (which should never happen) """ apitools_client = self.context[commands.APITOOLS_CLIENT_KEY] bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY] resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY] project_id = properties.VALUES.core.project.Get(required=True) source_reference = resource_parser.Parse(args.source, collection='bigquery.tables') source_reference_message = message_conversions.TableResourceToReference( bigquery_messages, source_reference) destination_resource = resource_parser.Parse( args.destination, collection='bigquery.tables') destination_reference = message_conversions.TableResourceToReference( bigquery_messages, destination_resource) if args.if_exists == 'append': write_disposition = 'WRITE_APPEND' ignore_already_exists = True elif args.if_exists == 'fail': write_disposition = 'WRITE_EMPTY' ignore_already_exists = False elif args.if_exists == 'prompt': write_disposition = 'WRITE_TRUNCATE' ignore_already_exists = False if bigquery_client_helper.TableExists(apitools_client, bigquery_messages, destination_reference): if not console_io.PromptContinue(prompt_string='Replace {0}'. format(destination_resource)): raise calliope_exceptions.ToolException('canceled by user') elif args.if_exists == 'replace': write_disposition = 'WRITE_TRUNCATE' ignore_already_exists = False elif args.if_exists == 'skip': if bigquery_client_helper.TableExists(apitools_client, bigquery_messages, destination_reference): return else: # This should be unreachable. raise core_exceptions.InternalError( 'Unexpected value "{0}" for --if-exists flag.'.format( args.if_exists)) copy_config = bigquery_messages.JobConfigurationTableCopy( sourceTable=source_reference_message, destinationTable=destination_reference, writeDisposition=write_disposition) job_id = job_ids.JobIdProvider().GetJobId(args.job_id, args.fingerprint_job_id) try: job = job_control.ExecuteJob( apitools_client, bigquery_messages, args, configuration=bigquery_messages.JobConfiguration( copy=copy_config), project_id=project_id, job_id=job_id) except bigquery.DuplicateError as e: if ignore_already_exists: job = None else: raise e if job is None: log.status.Print('Table "{0}" already exists, skipping'.format( destination_resource)) elif args. async: registry = self.context[commands.BIGQUERY_REGISTRY_KEY] job_resource = registry.Create( 'bigquery.jobs', projectId=job.jobReference.projectId, jobId=job.jobReference.jobId) log.CreatedResource(job_resource) else: log.status.Print('Table [{0}] successfully copied to [{1}]'.format( source_reference, destination_resource))