def _init_subscriber_client(self, auth_client):
# Silly emulator constraints
creds = getattr(auth_client, 'creds', None)
client = pubsub.SubscriberClient(credentials=creds)
try:
client.create_subscription(self.config['subscription'],
self.config['topic'])
except google_exceptions.AlreadyExists:
# subscription already exists
pass
except google_exceptions.NotFound as e:
msg = f'Topic "{self.config["topic"]}" does not exist.'
logging.error(msg, exc_info=e)
raise exceptions.GCPGordonError(msg)
except Exception as e:
sub = self.config['subscription']
msg = f'Error trying to create subscription "{sub}": {e}'
logging.error(msg, exc_info=e)
raise exceptions.GCPGordonError(msg)
max_messages = self.config.get('max_messages', 25)
flow_control = types.FlowControl(max_messages=max_messages)
logging.info(f'Starting a "{self.config["subscription"]}" subscriber '
f'to "{self.config["topic"]}" topic.')
return client, flow_control
def _load_keyfile(self, keyfile):
if not keyfile:
return None
try:
with open(keyfile, 'r') as f:
return json.load(f)
except FileNotFoundError as e:
msg = f'Keyfile {keyfile} was not found.'
logging.error(msg, exc_info=e)
raise exceptions.GCPGordonError(msg)
except json.JSONDecodeError as e:
msg = f'Keyfile {keyfile} is not valid JSON.'
logging.error(msg, exc_info=e)
raise exceptions.GCPGordonError(msg)
async def _poll_for_instance_data(self, resource_name, msg_logger):
exception = None
backoff = 2
base_url = f'https://www.googleapis.com/compute/v1/{resource_name}'
# Poll until instance data contains all necessary information.
for attempt in range(1, self.config['retries'] + 1):
try:
msg_logger.debug(f'Attempt {attempt}: fetching {base_url}')
instance_data = await self._http_client.get_json(base_url)
self._check_instance_data(instance_data)
return instance_data
except exceptions.GCPHTTPError as e:
exception = e
break
except (KeyError, IndexError) as e:
exception = e
if attempt == self.config['retries']:
continue
await asyncio.sleep(backoff)
backoff = backoff**2
msg = (f'Could not get necessary information for {resource_name}: '
f'{exception.__class__.__name__}: {exception}')
raise exceptions.GCPGordonError(msg)
def _load_schemas(self):
schema_path = pathlib.Path(self.HERE, self.SCHEMA_DIR).absolute()
schema_path_contents = schema_path.glob('*.json')
schemas = {}
for schema_file in schema_path_contents:
schema_name = schema_file.name.split('.')[0]
try:
with open(schema_file, 'r') as f:
schemas[schema_name] = json.load(f)
logging.info(
f'Successfully loaded schema "{schema_name}".')
except (FileNotFoundError, json.JSONDecodeError) as e:
msg = f'Error loading schema "{schema_name}": {e}.'
logging.error(msg, exc_info=e)
raise exceptions.GCPGordonError(msg)
if not schemas:
msg = 'Unable to load any schemas.'
logging.error(msg)
raise exceptions.GCPGordonError(msg)
return schemas
def _manage_subs(self):
# NOTE: automatically extends deadline in the background;
# must `nack()` if can't finish. We don't proactively
# `nack` in this plugin since it'll just get redelivered.
# A dead process will also timeout the message, with which
# it will redeliver.
future = self._subscriber.subscribe(self._subscription,
self._schedule_pubsub_msg,
flow_control=self._flow_control)
try:
# we're running in a threadpool because this is blocking
future.result()
except Exception as e:
self._subscriber.close()
logging.error(f'Issue polling subscription: {e}', exc_info=e)
raise exceptions.GCPGordonError(e)