def _DoAddRows(self, json_file, args):
"""Add rows from json_file to args.target_table."""
apitools_client = self.context[commands.APITOOLS_CLIENT_KEY]
bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY]
resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY]
resource = resource_parser.Parse(args.target_table,
collection='bigquery.tables')
reference = message_conversions.TableResourceToReference(
bigquery_messages, resource)
message_builder = bigquery_json_object_messages.MessageBuilder(
bigquery_messages)
batch = []
lineno = 0
errors_found = False
for line in json_file:
lineno += 1
trimmed_line = line.strip()
if trimmed_line:
try:
parsed_row = message_builder.Build(trimmed_line)
except bigquery.ClientError as e:
raise bigquery.Error(
'Line {num}: {msg}'.format(num=lineno, msg=str(e)),
None, [])
batch.append(parsed_row)
if _MAX_ROWS_PER_REQUEST and (len(batch)
== _MAX_ROWS_PER_REQUEST):
result = TablesAddRows._InsertTableRows(
reference, batch, apitools_client, bigquery_messages)
del batch[:]
if result.insertErrors:
errors_found = True
break
if lineno == 0:
log.status.Print('JSON file was empty.')
return
if batch and not errors_found:
result = TablesAddRows._InsertTableRows(reference, batch,
apitools_client,
bigquery_messages)
errors_found = bool(result.insertErrors)
if errors_found:
for entry in result.insertErrors:
log.err.Print('Record {0} could not be inserted:'.format(
entry.index))
for error in entry.errors:
log.err.Print('\t{0}: {1}'.format(error.reason,
error.message))
raise bigquery.Error('Rows not added')
else:
if not args.quiet:
log.status.Print('Rows added successfully.')
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
ToolException: when destination uri is not specified or invalid.
"""
apitools_client = self.context[commands.APITOOLS_CLIENT_KEY]
bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY]
resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY]
project_id = properties.VALUES.core.project.Get(required=True)
source_table_resource = resource_parser.Parse(
args.source_table, collection='bigquery.tables')
source_table_reference = message_conversions.TableResourceToReference(
bigquery_messages, source_table_resource)
if not args.destination_uri:
raise exceptions.ToolException(
'At least one destination URI must be specified.')
destination_uris = args.destination_uri
for uri in destination_uris:
if not uri.startswith('gs://'):
raise exceptions.ToolException((
'Illegal URI: {0}. Only Google Storage ("gs://") URIs are '
'supported.').format(uri))
job = job_control.ExecuteJob(
apitools_client,
bigquery_messages,
args,
configuration=bigquery_messages.JobConfiguration(
extract=bigquery_messages.JobConfigurationExtract(
sourceTable=source_table_reference,
destinationUris=destination_uris,
destinationFormat=bigquery_client_helper.
NormalizeTextualFormat(args.destination_format),
fieldDelimiter=bigquery_client_helper.
NormalizeFieldDelimiter(args.field_delimiter))),
async=args. async,
project_id=project_id,
job_id=job_ids.JobIdProvider().GetJobId(args.job_id,
args.fingerprint_job_id))
if args. async:
job_resource = resource_parser.Create(
'bigquery.jobs',
projectId=job.jobReference.projectId,
jobId=job.jobReference.jobId)
log.CreatedResource(job_resource)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespeace, All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
apitools_client = self.context[commands.APITOOLS_CLIENT_KEY]
bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY]
resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY]
resource = resource_parser.Parse(args.table_or_view,
collection='bigquery.tables')
reference = message_conversions.TableResourceToReference(
bigquery_messages, resource)
if args.expiration:
expiration_instant_in_millis = int(
(bigquery.CurrentTimeInSec() + args.expiration) * 1000)
else:
expiration_instant_in_millis = None
if args.schema:
new_schema = bigquery_schemas.ReadSchema(args.schema,
bigquery_messages)
elif args.schema_file:
new_schema = bigquery_schemas.ReadSchemaFile(
args.schema_file, bigquery_messages)
else:
new_schema = None
request = bigquery_messages.BigqueryTablesPatchRequest(
projectId=reference.projectId,
datasetId=reference.datasetId,
tableId=reference.tableId,
table=bigquery_messages.Table(
tableReference=reference,
description=args.description,
expirationTime=expiration_instant_in_millis,
friendlyName=args.friendly_name,
schema=new_schema))
try:
apitools_client.tables.Patch(request)
except apitools_base.HttpError as e:
raise bigquery.Error.ForHttpError(e)
log.UpdatedResource(reference)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
bigquery.DuplicateError: if table already exists.
Returns:
None
"""
apitools_client = self.context[commands.APITOOLS_CLIENT_KEY]
bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY]
resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY]
resource = resource_parser.Parse(args.table,
collection='bigquery.tables')
reference = message_conversions.TableResourceToReference(
bigquery_messages, resource)
table_or_view = 'View' if args.view else 'Table'
if bigquery_client_helper.TableExists(apitools_client,
bigquery_messages, reference):
if args.if_exists == 'skip':
log.status.Print(
'Skipping this operation because a table or view named '
'[{0}] already exists.'.format(reference))
return
else:
message = (
'{0} [{1}] could not be created; a table with this name '
'already exists.'.format(table_or_view, reference))
raise bigquery.DuplicateError(message, None, [])
if args.schema:
schema = bigquery_schemas.ReadSchema(args.schema,
bigquery_messages)
elif args.schema_file:
schema = bigquery_schemas.ReadSchemaFile(args.schema_file,
bigquery_messages)
else:
schema = None
if args.expiration:
expiration_instant_seconds = time.time() + args.expiration
expiration_instant_millis = int(1000 * expiration_instant_seconds)
else:
expiration_instant_millis = None
if args.view:
view_definition = bigquery_messages.ViewDefinition(query=args.view)
else:
view_definition = None
request = bigquery_messages.BigqueryTablesInsertRequest(
projectId=reference.projectId,
datasetId=reference.datasetId,
table=bigquery_messages.Table(
tableReference=bigquery_messages.TableReference(
projectId=reference.projectId,
datasetId=reference.datasetId,
tableId=reference.tableId),
description=args.description,
expirationTime=expiration_instant_millis,
schema=schema,
view=view_definition))
try:
apitools_client.tables.Insert(request)
except apitools_base.HttpError as server_error:
raise bigquery.Error.ForHttpError(server_error)
log.CreatedResource(resource)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Returns:
None
Raises:
bigqueryError.BigqueryError: If the source and destination files are not
both specified.
calliope_exceptions.ToolException: If user cancels this operation.
Exception: If an unexpected value for the --if-exists flag passed gcloud
validation (which should never happen)
"""
apitools_client = self.context[commands.APITOOLS_CLIENT_KEY]
bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY]
resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY]
project_id = properties.VALUES.core.project.Get(required=True)
source_reference = resource_parser.Parse(args.source,
collection='bigquery.tables')
source_reference_message = message_conversions.TableResourceToReference(
bigquery_messages, source_reference)
destination_resource = resource_parser.Parse(
args.destination, collection='bigquery.tables')
destination_reference = message_conversions.TableResourceToReference(
bigquery_messages, destination_resource)
if args.if_exists == 'append':
write_disposition = 'WRITE_APPEND'
ignore_already_exists = True
elif args.if_exists == 'fail':
write_disposition = 'WRITE_EMPTY'
ignore_already_exists = False
elif args.if_exists == 'prompt':
write_disposition = 'WRITE_TRUNCATE'
ignore_already_exists = False
if bigquery_client_helper.TableExists(apitools_client,
bigquery_messages,
destination_reference):
if not console_io.PromptContinue(prompt_string='Replace {0}'.
format(destination_resource)):
raise calliope_exceptions.ToolException('canceled by user')
elif args.if_exists == 'replace':
write_disposition = 'WRITE_TRUNCATE'
ignore_already_exists = False
elif args.if_exists == 'skip':
if bigquery_client_helper.TableExists(apitools_client,
bigquery_messages,
destination_reference):
return
else:
# This should be unreachable.
raise core_exceptions.InternalError(
'Unexpected value "{0}" for --if-exists flag.'.format(
args.if_exists))
copy_config = bigquery_messages.JobConfigurationTableCopy(
sourceTable=source_reference_message,
destinationTable=destination_reference,
writeDisposition=write_disposition)
job_id = job_ids.JobIdProvider().GetJobId(args.job_id,
args.fingerprint_job_id)
try:
job = job_control.ExecuteJob(
apitools_client,
bigquery_messages,
args,
configuration=bigquery_messages.JobConfiguration(
copy=copy_config),
project_id=project_id,
job_id=job_id)
except bigquery.DuplicateError as e:
if ignore_already_exists:
job = None
else:
raise e
if job is None:
log.status.Print('Table "{0}" already exists, skipping'.format(
destination_resource))
elif args. async:
registry = self.context[commands.BIGQUERY_REGISTRY_KEY]
job_resource = registry.Create(
'bigquery.jobs',
projectId=job.jobReference.projectId,
jobId=job.jobReference.jobId)
log.CreatedResource(job_resource)
else:
log.status.Print('Table [{0}] successfully copied to [{1}]'.format(
source_reference, destination_resource))
