def _DoAddRows(self, json_file, args): """Add rows from json_file to args.target_table.""" apitools_client = self.context[commands.APITOOLS_CLIENT_KEY] bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY] resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY] resource = resource_parser.Parse(args.target_table, collection='bigquery.tables') reference = message_conversions.TableResourceToReference( bigquery_messages, resource) message_builder = bigquery_json_object_messages.MessageBuilder( bigquery_messages) batch = [] lineno = 0 errors_found = False for line in json_file: lineno += 1 trimmed_line = line.strip() if trimmed_line: try: parsed_row = message_builder.Build(trimmed_line) except bigquery.ClientError as e: raise bigquery.Error( 'Line {num}: {msg}'.format(num=lineno, msg=str(e)), None, []) batch.append(parsed_row) if _MAX_ROWS_PER_REQUEST and (len(batch) == _MAX_ROWS_PER_REQUEST): result = TablesAddRows._InsertTableRows( reference, batch, apitools_client, bigquery_messages) del batch[:] if result.insertErrors: errors_found = True break if lineno == 0: log.status.Print('JSON file was empty.') return if batch and not errors_found: result = TablesAddRows._InsertTableRows(reference, batch, apitools_client, bigquery_messages) errors_found = bool(result.insertErrors) if errors_found: for entry in result.insertErrors: log.err.Print('Record {0} could not be inserted:'.format( entry.index)) for error in entry.errors: log.err.Print('\t{0}: {1}'.format(error.reason, error.message)) raise bigquery.Error('Rows not added') else: if not args.quiet: log.status.Print('Rows added successfully.')
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Raises: ToolException: when destination uri is not specified or invalid. """ apitools_client = self.context[commands.APITOOLS_CLIENT_KEY] bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY] resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY] project_id = properties.VALUES.core.project.Get(required=True) source_table_resource = resource_parser.Parse( args.source_table, collection='bigquery.tables') source_table_reference = message_conversions.TableResourceToReference( bigquery_messages, source_table_resource) if not args.destination_uri: raise exceptions.ToolException( 'At least one destination URI must be specified.') destination_uris = args.destination_uri for uri in destination_uris: if not uri.startswith('gs://'): raise exceptions.ToolException(( 'Illegal URI: {0}. Only Google Storage ("gs://") URIs are ' 'supported.').format(uri)) job = job_control.ExecuteJob( apitools_client, bigquery_messages, args, configuration=bigquery_messages.JobConfiguration( extract=bigquery_messages.JobConfigurationExtract( sourceTable=source_table_reference, destinationUris=destination_uris, destinationFormat=bigquery_client_helper. NormalizeTextualFormat(args.destination_format), fieldDelimiter=bigquery_client_helper. NormalizeFieldDelimiter(args.field_delimiter))), async=args. async, project_id=project_id, job_id=job_ids.JobIdProvider().GetJobId(args.job_id, args.fingerprint_job_id)) if args. async: job_resource = resource_parser.Create( 'bigquery.jobs', projectId=job.jobReference.projectId, jobId=job.jobReference.jobId) log.CreatedResource(job_resource)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespeace, All the arguments that were provided to this command invocation. Returns: Some value that we want to have printed later. """ apitools_client = self.context[commands.APITOOLS_CLIENT_KEY] bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY] resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY] resource = resource_parser.Parse(args.table_or_view, collection='bigquery.tables') reference = message_conversions.TableResourceToReference( bigquery_messages, resource) if args.expiration: expiration_instant_in_millis = int( (bigquery.CurrentTimeInSec() + args.expiration) * 1000) else: expiration_instant_in_millis = None if args.schema: new_schema = bigquery_schemas.ReadSchema(args.schema, bigquery_messages) elif args.schema_file: new_schema = bigquery_schemas.ReadSchemaFile( args.schema_file, bigquery_messages) else: new_schema = None request = bigquery_messages.BigqueryTablesPatchRequest( projectId=reference.projectId, datasetId=reference.datasetId, tableId=reference.tableId, table=bigquery_messages.Table( tableReference=reference, description=args.description, expirationTime=expiration_instant_in_millis, friendlyName=args.friendly_name, schema=new_schema)) try: apitools_client.tables.Patch(request) except apitools_base.HttpError as e: raise bigquery.Error.ForHttpError(e) log.UpdatedResource(reference)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Raises: bigquery.DuplicateError: if table already exists. Returns: None """ apitools_client = self.context[commands.APITOOLS_CLIENT_KEY] bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY] resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY] resource = resource_parser.Parse(args.table, collection='bigquery.tables') reference = message_conversions.TableResourceToReference( bigquery_messages, resource) table_or_view = 'View' if args.view else 'Table' if bigquery_client_helper.TableExists(apitools_client, bigquery_messages, reference): if args.if_exists == 'skip': log.status.Print( 'Skipping this operation because a table or view named ' '[{0}] already exists.'.format(reference)) return else: message = ( '{0} [{1}] could not be created; a table with this name ' 'already exists.'.format(table_or_view, reference)) raise bigquery.DuplicateError(message, None, []) if args.schema: schema = bigquery_schemas.ReadSchema(args.schema, bigquery_messages) elif args.schema_file: schema = bigquery_schemas.ReadSchemaFile(args.schema_file, bigquery_messages) else: schema = None if args.expiration: expiration_instant_seconds = time.time() + args.expiration expiration_instant_millis = int(1000 * expiration_instant_seconds) else: expiration_instant_millis = None if args.view: view_definition = bigquery_messages.ViewDefinition(query=args.view) else: view_definition = None request = bigquery_messages.BigqueryTablesInsertRequest( projectId=reference.projectId, datasetId=reference.datasetId, table=bigquery_messages.Table( tableReference=bigquery_messages.TableReference( projectId=reference.projectId, datasetId=reference.datasetId, tableId=reference.tableId), description=args.description, expirationTime=expiration_instant_millis, schema=schema, view=view_definition)) try: apitools_client.tables.Insert(request) except apitools_base.HttpError as server_error: raise bigquery.Error.ForHttpError(server_error) log.CreatedResource(resource)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Returns: None Raises: bigqueryError.BigqueryError: If the source and destination files are not both specified. calliope_exceptions.ToolException: If user cancels this operation. Exception: If an unexpected value for the --if-exists flag passed gcloud validation (which should never happen) """ apitools_client = self.context[commands.APITOOLS_CLIENT_KEY] bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY] resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY] project_id = properties.VALUES.core.project.Get(required=True) source_reference = resource_parser.Parse(args.source, collection='bigquery.tables') source_reference_message = message_conversions.TableResourceToReference( bigquery_messages, source_reference) destination_resource = resource_parser.Parse( args.destination, collection='bigquery.tables') destination_reference = message_conversions.TableResourceToReference( bigquery_messages, destination_resource) if args.if_exists == 'append': write_disposition = 'WRITE_APPEND' ignore_already_exists = True elif args.if_exists == 'fail': write_disposition = 'WRITE_EMPTY' ignore_already_exists = False elif args.if_exists == 'prompt': write_disposition = 'WRITE_TRUNCATE' ignore_already_exists = False if bigquery_client_helper.TableExists(apitools_client, bigquery_messages, destination_reference): if not console_io.PromptContinue(prompt_string='Replace {0}'. format(destination_resource)): raise calliope_exceptions.ToolException('canceled by user') elif args.if_exists == 'replace': write_disposition = 'WRITE_TRUNCATE' ignore_already_exists = False elif args.if_exists == 'skip': if bigquery_client_helper.TableExists(apitools_client, bigquery_messages, destination_reference): return else: # This should be unreachable. raise core_exceptions.InternalError( 'Unexpected value "{0}" for --if-exists flag.'.format( args.if_exists)) copy_config = bigquery_messages.JobConfigurationTableCopy( sourceTable=source_reference_message, destinationTable=destination_reference, writeDisposition=write_disposition) job_id = job_ids.JobIdProvider().GetJobId(args.job_id, args.fingerprint_job_id) try: job = job_control.ExecuteJob( apitools_client, bigquery_messages, args, configuration=bigquery_messages.JobConfiguration( copy=copy_config), project_id=project_id, job_id=job_id) except bigquery.DuplicateError as e: if ignore_already_exists: job = None else: raise e if job is None: log.status.Print('Table "{0}" already exists, skipping'.format( destination_resource)) elif args. async: registry = self.context[commands.BIGQUERY_REGISTRY_KEY] job_resource = registry.Create( 'bigquery.jobs', projectId=job.jobReference.projectId, jobId=job.jobReference.jobId) log.CreatedResource(job_resource) else: log.status.Print('Table [{0}] successfully copied to [{1}]'.format( source_reference, destination_resource))