def test__retry_read_rows_exception_miss_wrapped_in_grpc():
from google.api_core.exceptions import Conflict
from google.cloud.bigtable.row_data import _retry_read_rows_exception
wrapped = Conflict("testing")
exception = _make_grpc_call_error(wrapped)
assert not _retry_read_rows_exception(exception)
def create_column_family(self,
column_family_name,
table_name,
max_age=None,
nr_max_versions=None,
gc_rule_union=None):
"""Create a column family and add it to a table. Garbage collection rules
can be included to the column family.
Args:
column_family_name (str):
table_name (str):
max_age (int): the time to live in days
nr_max_versions (int): the number of versions that should be kept
gc_rule_union (bool or None): if both max_age and nr_max_versions are specified,
then this parameter should be a bool. If True, then the max age and the max
versions rules are unified, if False, then the intersection of the rules is
used.
Returns:
google.cloud.bigtable.column_family.ColumnFamily
"""
if max_age and nr_max_versions:
# Both rules are specified, this also means a merge method must be specified (union or intersection)
time_to_live = dt.timedelta(days=max_age)
max_age_rule = bt_column_family.MaxAgeGCRule(time_to_live)
max_versions_rule = bt_column_family.MaxVersionsGCRule(
nr_max_versions)
if gc_rule_union is None:
raise Conflict(
"If max_age and nr_max_versions are both specified, then gc_rule_union cannot be None."
)
elif gc_rule_union:
gc_rule = bt_column_family.GCRuleUnion(
rules=[max_age_rule, max_versions_rule])
else:
gc_rule = bt_column_family.GCRuleIntersection(
rules=[max_age_rule, max_versions_rule])
elif max_age:
# only max age is specified
time_to_live = dt.timedelta(days=max_age)
gc_rule = bt_column_family.MaxAgeGCRule(time_to_live)
elif nr_max_versions:
# only max number of versions is specified
gc_rule = bt_column_family.MaxVersionsGCRule(nr_max_versions)
else:
# no rule is specified
gc_rule = None
table = self.instance.table(table_name)
if not table.exists():
raise NotFound(
"Table name '{}' does not exist.".format(table_name))
logging.info("Creating column family '%s' in table '%s'.",
column_family_name, table_name)
column_family = bt_column_family.ColumnFamily(column_family_name,
table, gc_rule)
column_family.create()
def create(self, mode="all", if_exists="raise"):
"""Creates BigQuery datasets given `dataset_id`.
It can create two datasets:
- <dataset_id> (mode = 'prod')
- <dataset_id>_staging (mode = 'staging')
If mode is all, it creates both.
Parameters
----------
mode : str, optional
Which dataset to create [prod|staging|all], by default "all"
if_exists : str, optional
What to do if dataset exists, by default "raise"
- 'raise' : Raises Conflic exception
- 'replace' : Drop all tables and replace dataset
- 'update' : Update dataset description
- 'pass' : Do nothing
Raises
------
google.api_core.exceptions.Conflict
Dataset already exists and if_exists is set to 'raise'
"""
if if_exists == "replace":
self.delete(mode)
elif if_exists == "update":
self.update()
return
# Set dataset_id to the ID of the dataset to create.
for m in self._loop_modes(mode):
# Construct a full Dataset object to send to the API.
dataset_obj = self._setup_dataset_object(m["id"])
# Send the dataset to the API for creation, with an explicit timeout.
# Raises google.api_core.exceptions.Conflict if the Dataset already
# exists within the project.
try:
job = m["client"].create_dataset(
dataset_obj) # Make an API request.
except Conflict:
if if_exists == "pass":
return
else:
raise Conflict(f"Dataset {self.dataset_id} already exists")
# Make prod dataset public
self.publicize()
def create(self, mode="all", if_exists="raise"):
"""Creates BigQuery datasets given `dataset_id`.
It can create two datasets:
* `<dataset_id>` (mode = 'prod')
* `<dataset_id>_staging` (mode = 'staging')
If `mode` is all, it creates both.
Args:
mode (str): Optional. Which dataset to create [prod|staging|all].
if_exists (str): Optional. What to do if dataset exists
* raise : Raises Conflic exception
* replace : Drop all tables and replace dataset
* update : Update dataset description
* pass : Do nothing
Raises:
Warning: Dataset already exists and if_exists is set to `raise`
"""
if if_exists == "replace":
self.delete(mode)
elif if_exists == "update":
self.update()
return
# Set dataset_id to the ID of the dataset to create.
for m in self._loop_modes(mode):
# Construct a full Dataset object to send to the API.
dataset_obj = self._setup_dataset_object(m["id"])
# Send the dataset to the API for creation, with an explicit timeout.
# Raises google.api_core.exceptions.Conflict if the Dataset already
# exists within the project.
try:
job = m["client"].create_dataset(
dataset_obj) # Make an API request.
except Conflict:
if if_exists == "pass":
return
else:
raise Conflict(f"Dataset {self.dataset_id} already exists")
# Make prod dataset public
self.publicize()
def create(self):
if self.exists():
raise Conflict('POST https://www.googleapis.com/storage/v1'
'/b?project=%s: Sorry, that name is not available.'
' Please try a different one.')
if 'location' in self._changes and 'location' in self._properties:
location = self._properties['location'].upper()
else:
location = 'US'
self.client.mock_gcs_fs[self.name] = dict(blobs={},
lifecycle_rules=[],
location=location)
def create(self, location=None):
if self.exists():
raise Conflict(
'POST https://www.googleapis.com/storage/v1'
'/b?project=%s: Sorry, that name is not available.'
' Please try a different one.')
if location:
location = location.upper()
else:
location = 'US' # default bucket location
self.client.mock_gcs_fs[self.name] = dict(
blobs={}, lifecycle_rules=[], location=location)
def _check_restrictions(self, doc: BaseModel, is_update: bool = False):
# Check for any restrictions
for key in self.get_unique_keys():
value = getattr(doc, key)
try:
doc_db = self.get_by_attribute(key, value)
# If the clashing document is itself the allow clash
if is_update and doc.id == doc_db.id:
continue
raise Conflict(
f"{self.name} with {key} {value} already exists")
except NotFound:
# No document with given unique key found
pass
def load(self, location: str = "US"):
"""Load the Data to BigQuery table
Please use ``.bq.table()`` to set the destination table name, and the table
must not exists, otherwise the ``Conflict`` exception will be raised.
:param location: The BigQuery location, default is ``US``
:type: str
:raises google.api_core.exceptions.Conflict: table already
exists exception.
"""
if "_bq" not in self.__dict__ or "_worksheet" not in self.__dict__:
raise ValueError(
".worksheet() and .bq() must be called before run")
if "_table" not in self._bq.__dict__:
raise ValueError(
"bigquery table must be specify for the load"
)
self._load()
if self._bq.is_exist():
raise Conflict(f"{self._bq._table} exists")
if "_schema" not in self._bq.__dict__:
job_config = bigquery.LoadJobConfig(
autodetect=True,
source_format=self._bq._format,
)
else:
job_config = bigquery.LoadJobConfig(
schema=self._bq._schema,
source_format=self._bq._format
)
load_job = self._bq._client.load_table_from_json(
self._json_to_be_load, self._bq._table, location=location, job_config=job_config)
load_job.result()
logger.info(
f"{self._sheet_id} is loaded into {self._bq._project}.{self._bq._table}")
def _check_restrictions_attributes(self, doc_id: str,
attributes: Dict[str, Any]):
# Check for any restrictions
for key in self.get_unique_keys():
if key not in attributes:
continue
value = attributes.get(key)
try:
doc_db = self.get_by_attribute(key, value)
# If the clashing document is itself then allow clash
if doc_id == doc_db.id:
continue
raise Conflict(
f"{self.name} with {key} {value} already exists")
except NotFound:
# No document with given unique key found
pass
def test_w_miss_wrapped_in_grpc(self):
from google.api_core.exceptions import Conflict
wrapped = Conflict("testing")
exception = self._make_grpc_call_error(wrapped)
self.assertFalse(self._call_fut(exception))
def test_w_miss(self):
from google.api_core.exceptions import Conflict
exception = Conflict("testing")
self.assertFalse(self._call_fut(exception))
def test__retry_read_rows_exception_miss():
from google.api_core.exceptions import Conflict
from google.cloud.bigtable.row_data import _retry_read_rows_exception
exception = Conflict("testing")
assert not _retry_read_rows_exception(exception)
