def test__retry_read_rows_exception_miss_wrapped_in_grpc(): from google.api_core.exceptions import Conflict from google.cloud.bigtable.row_data import _retry_read_rows_exception wrapped = Conflict("testing") exception = _make_grpc_call_error(wrapped) assert not _retry_read_rows_exception(exception)
def create_column_family(self, column_family_name, table_name, max_age=None, nr_max_versions=None, gc_rule_union=None): """Create a column family and add it to a table. Garbage collection rules can be included to the column family. Args: column_family_name (str): table_name (str): max_age (int): the time to live in days nr_max_versions (int): the number of versions that should be kept gc_rule_union (bool or None): if both max_age and nr_max_versions are specified, then this parameter should be a bool. If True, then the max age and the max versions rules are unified, if False, then the intersection of the rules is used. Returns: google.cloud.bigtable.column_family.ColumnFamily """ if max_age and nr_max_versions: # Both rules are specified, this also means a merge method must be specified (union or intersection) time_to_live = dt.timedelta(days=max_age) max_age_rule = bt_column_family.MaxAgeGCRule(time_to_live) max_versions_rule = bt_column_family.MaxVersionsGCRule( nr_max_versions) if gc_rule_union is None: raise Conflict( "If max_age and nr_max_versions are both specified, then gc_rule_union cannot be None." ) elif gc_rule_union: gc_rule = bt_column_family.GCRuleUnion( rules=[max_age_rule, max_versions_rule]) else: gc_rule = bt_column_family.GCRuleIntersection( rules=[max_age_rule, max_versions_rule]) elif max_age: # only max age is specified time_to_live = dt.timedelta(days=max_age) gc_rule = bt_column_family.MaxAgeGCRule(time_to_live) elif nr_max_versions: # only max number of versions is specified gc_rule = bt_column_family.MaxVersionsGCRule(nr_max_versions) else: # no rule is specified gc_rule = None table = self.instance.table(table_name) if not table.exists(): raise NotFound( "Table name '{}' does not exist.".format(table_name)) logging.info("Creating column family '%s' in table '%s'.", column_family_name, table_name) column_family = bt_column_family.ColumnFamily(column_family_name, table, gc_rule) column_family.create()
def create(self, mode="all", if_exists="raise"): """Creates BigQuery datasets given `dataset_id`. It can create two datasets: - <dataset_id> (mode = 'prod') - <dataset_id>_staging (mode = 'staging') If mode is all, it creates both. Parameters ---------- mode : str, optional Which dataset to create [prod|staging|all], by default "all" if_exists : str, optional What to do if dataset exists, by default "raise" - 'raise' : Raises Conflic exception - 'replace' : Drop all tables and replace dataset - 'update' : Update dataset description - 'pass' : Do nothing Raises ------ google.api_core.exceptions.Conflict Dataset already exists and if_exists is set to 'raise' """ if if_exists == "replace": self.delete(mode) elif if_exists == "update": self.update() return # Set dataset_id to the ID of the dataset to create. for m in self._loop_modes(mode): # Construct a full Dataset object to send to the API. dataset_obj = self._setup_dataset_object(m["id"]) # Send the dataset to the API for creation, with an explicit timeout. # Raises google.api_core.exceptions.Conflict if the Dataset already # exists within the project. try: job = m["client"].create_dataset( dataset_obj) # Make an API request. except Conflict: if if_exists == "pass": return else: raise Conflict(f"Dataset {self.dataset_id} already exists") # Make prod dataset public self.publicize()
def create(self, mode="all", if_exists="raise"): """Creates BigQuery datasets given `dataset_id`. It can create two datasets: * `<dataset_id>` (mode = 'prod') * `<dataset_id>_staging` (mode = 'staging') If `mode` is all, it creates both. Args: mode (str): Optional. Which dataset to create [prod|staging|all]. if_exists (str): Optional. What to do if dataset exists * raise : Raises Conflic exception * replace : Drop all tables and replace dataset * update : Update dataset description * pass : Do nothing Raises: Warning: Dataset already exists and if_exists is set to `raise` """ if if_exists == "replace": self.delete(mode) elif if_exists == "update": self.update() return # Set dataset_id to the ID of the dataset to create. for m in self._loop_modes(mode): # Construct a full Dataset object to send to the API. dataset_obj = self._setup_dataset_object(m["id"]) # Send the dataset to the API for creation, with an explicit timeout. # Raises google.api_core.exceptions.Conflict if the Dataset already # exists within the project. try: job = m["client"].create_dataset( dataset_obj) # Make an API request. except Conflict: if if_exists == "pass": return else: raise Conflict(f"Dataset {self.dataset_id} already exists") # Make prod dataset public self.publicize()
def create(self): if self.exists(): raise Conflict('POST https://www.googleapis.com/storage/v1' '/b?project=%s: Sorry, that name is not available.' ' Please try a different one.') if 'location' in self._changes and 'location' in self._properties: location = self._properties['location'].upper() else: location = 'US' self.client.mock_gcs_fs[self.name] = dict(blobs={}, lifecycle_rules=[], location=location)
def create(self, location=None): if self.exists(): raise Conflict( 'POST https://www.googleapis.com/storage/v1' '/b?project=%s: Sorry, that name is not available.' ' Please try a different one.') if location: location = location.upper() else: location = 'US' # default bucket location self.client.mock_gcs_fs[self.name] = dict( blobs={}, lifecycle_rules=[], location=location)
def _check_restrictions(self, doc: BaseModel, is_update: bool = False): # Check for any restrictions for key in self.get_unique_keys(): value = getattr(doc, key) try: doc_db = self.get_by_attribute(key, value) # If the clashing document is itself the allow clash if is_update and doc.id == doc_db.id: continue raise Conflict( f"{self.name} with {key} {value} already exists") except NotFound: # No document with given unique key found pass
def load(self, location: str = "US"): """Load the Data to BigQuery table Please use ``.bq.table()`` to set the destination table name, and the table must not exists, otherwise the ``Conflict`` exception will be raised. :param location: The BigQuery location, default is ``US`` :type: str :raises google.api_core.exceptions.Conflict: table already exists exception. """ if "_bq" not in self.__dict__ or "_worksheet" not in self.__dict__: raise ValueError( ".worksheet() and .bq() must be called before run") if "_table" not in self._bq.__dict__: raise ValueError( "bigquery table must be specify for the load" ) self._load() if self._bq.is_exist(): raise Conflict(f"{self._bq._table} exists") if "_schema" not in self._bq.__dict__: job_config = bigquery.LoadJobConfig( autodetect=True, source_format=self._bq._format, ) else: job_config = bigquery.LoadJobConfig( schema=self._bq._schema, source_format=self._bq._format ) load_job = self._bq._client.load_table_from_json( self._json_to_be_load, self._bq._table, location=location, job_config=job_config) load_job.result() logger.info( f"{self._sheet_id} is loaded into {self._bq._project}.{self._bq._table}")
def _check_restrictions_attributes(self, doc_id: str, attributes: Dict[str, Any]): # Check for any restrictions for key in self.get_unique_keys(): if key not in attributes: continue value = attributes.get(key) try: doc_db = self.get_by_attribute(key, value) # If the clashing document is itself then allow clash if doc_id == doc_db.id: continue raise Conflict( f"{self.name} with {key} {value} already exists") except NotFound: # No document with given unique key found pass
def test_w_miss_wrapped_in_grpc(self): from google.api_core.exceptions import Conflict wrapped = Conflict("testing") exception = self._make_grpc_call_error(wrapped) self.assertFalse(self._call_fut(exception))
def test_w_miss(self): from google.api_core.exceptions import Conflict exception = Conflict("testing") self.assertFalse(self._call_fut(exception))
def test__retry_read_rows_exception_miss(): from google.api_core.exceptions import Conflict from google.cloud.bigtable.row_data import _retry_read_rows_exception exception = Conflict("testing") assert not _retry_read_rows_exception(exception)