def start_bundle(self): self._client = helper.get_client(self._project, namespace=None) self._init_batch() self._batch_sizer = util.DynamicBatchSizer() self._target_batch_size = self._batch_sizer.get_batch_size( time.time() * 1000)
def process(self, query, *unused_args, **unused_kwargs): if query.namespace is None: query.namespace = '' _client = helper.get_client(query.project, query.namespace) client_query = query._to_client_query(_client) # Create request count metric resource = resource_identifiers.DatastoreNamespace( query.project, query.namespace) labels = { monitoring_infos.SERVICE_LABEL: 'Datastore', monitoring_infos.METHOD_LABEL: 'BatchDatastoreRead', monitoring_infos.RESOURCE_LABEL: resource, monitoring_infos.DATASTORE_NAMESPACE_LABEL: query.namespace, monitoring_infos.DATASTORE_PROJECT_ID_LABEL: query.project, monitoring_infos.STATUS_LABEL: 'ok' } service_call_metric = ServiceCallMetric( request_count_urn=monitoring_infos.API_REQUEST_COUNT_URN, base_labels=labels) try: for client_entity in client_query.fetch(query.limit): yield types.Entity.from_client_entity(client_entity) service_call_metric.call('ok') except (ClientError, GoogleAPICallError) as e: # e.code.value contains the numeric http status code. service_call_metric.call(e.code.value) except HttpError as e: service_call_metric.call(e)
def process(self, query, *args, **kwargs): client = helper.get_client(query.project, query.namespace) try: # Short circuit estimating num_splits if split is not possible. query_splitter.validate_split(query) if self._num_splits == 0: estimated_num_splits = self.get_estimated_num_splits(client, query) else: estimated_num_splits = self._num_splits _LOGGER.info("Splitting the query into %d splits", estimated_num_splits) query_splits = query_splitter.get_splits( client, query, estimated_num_splits) except query_splitter.QuerySplitterError: _LOGGER.info("Unable to parallelize the given query: %s", query, exc_info=True) query_splits = [query] return query_splits
def process(self, query, *unused_args, **unused_kwargs): _client = helper.get_client(query.project, query.namespace) client_query = query._to_client_query(_client) for client_entity in client_query.fetch(query.limit): yield types.Entity.from_client_entity(client_entity)