def retry_upon_exception(exc, delay, max_delay, max_attempts):
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_exception_type(exc),
wait=tenacity.wait_exponential(
multiplier=delay, max=max_delay),
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
def call_func(self, func, **kwargs):
"""General method for calling any Monasca API function."""
@tenacity.retry(
wait=tenacity.wait_fixed(self._retry_interval),
stop=tenacity.stop_after_attempt(self._max_retries),
retry=(tenacity.retry_if_exception_type(MonascaServiceException) |
tenacity.retry_if_exception_type(MonascaException)))
def _inner():
try:
return func(**kwargs)
except (exc.http.InternalServerError,
exc.http.ServiceUnavailable,
exc.http.BadGateway,
exc.connection.ConnectionError) as e:
LOG.exception(e)
msg = '%s: %s' % (e.__class__.__name__, e)
raise MonascaServiceException(msg)
except exc.http.HttpError as e:
LOG.exception(e)
msg = '%s: %s' % (e.__class__.__name__, e)
status_code = e.http_status
if not isinstance(status_code, int):
status_code = 500
if 400 <= status_code < 500:
raise MonascaInvalidParametersException(msg)
else:
raise MonascaException(msg)
except Exception as e:
LOG.exception(e)
msg = '%s: %s' % (e.__class__.__name__, e)
raise MonascaException(msg)
return _inner()
def validate(self, max_retries=0):
"""Performs basic **connection** validation of a sqlalchemy engine."""
def _retry_on_exception(exc):
LOG.warning("Engine connection (validate) failed due to '%s'", exc)
if isinstance(exc, sa_exc.OperationalError) and \
_is_db_connection_error(six.text_type(exc.args[0])):
# We may be able to fix this by retrying...
return True
if isinstance(exc, (sa_exc.TimeoutError,
sa_exc.ResourceClosedError,
sa_exc.DisconnectionError)):
# We may be able to fix this by retrying...
return True
# Other failures we likely can't fix by retrying...
return False
@tenacity.retry(
stop=tenacity.stop_after_attempt(max(0, int(max_retries))),
wait=tenacity.wait_exponential(),
reraise=True,
retry=tenacity.retry_if_exception(_retry_on_exception)
)
def _try_connect(engine):
# See if we can make a connection happen.
#
# NOTE(harlowja): note that even though we are connecting
# once it does not mean that we will be able to connect in
# the future, so this is more of a sanity test and is not
# complete connection insurance.
with contextlib.closing(engine.connect()):
pass
_try_connect(self._engine)
def retry_on_db_error(func, retry=None):
"""Decorates the given function so that it retries on DB errors.
Note that the decorator retries the function/method only on some
of the DB errors that are considered to be worth retrying, like
deadlocks and disconnections.
:param func: Function to decorate.
:param retry: a Retrying object
:return: Decorated function.
"""
if not retry:
retry = tenacity.Retrying(
retry=tenacity.retry_if_exception_type(_RETRY_ERRORS),
stop=tenacity.stop_after_attempt(50),
wait=tenacity.wait_incrementing(start=0, increment=0.1, max=2)
)
# The `assigned` arg should be empty as some of the default values are not
# supported by simply initialized MagicMocks. The consequence may
# be that the representation will contain the wrapper and not the
# wrapped function.
@functools.wraps(func, assigned=[])
def decorate(*args, **kw):
# Retrying library decorator might potentially run a decorated
# function within a new thread so it's safer not to apply the
# decorator directly to a target method/function because we can
# lose an authentication context.
# The solution is to create one more function and explicitly set
# auth context before calling it (potentially in a new thread).
auth_ctx = context.ctx() if context.has_ctx() else None
return retry.call(_with_auth_context, auth_ctx, func, *args, **kw)
return decorate
def fetch(self, url, verb='get', **kwargs):
"""return response or None in case of failure, try twice"""
@retry(stop=stop_after_attempt(2), wait=wait_fixed(2))
def _fetch(verb='get'):
headers = {
'Authorization': 'apiToken %s' % C.DEFAULT_SHIPPABLE_TOKEN
}
logging.info(u'%s %s' % (verb, url))
http_method = getattr(requests, verb)
resp = http_method(url, headers=headers, **kwargs)
logging.info(u'shippable status code: %s' % resp.status_code)
logging.info(u'shippable reason: %s' % resp.reason)
if resp.status_code not in [200, 302, 400]:
logging.error(u'RC: %s', resp.status_code)
raise TryAgain
return resp
try:
logging.debug(u'%s', url)
return _fetch(verb=verb)
except RetryError as e:
logging.error(e)
def upgrade():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.BoolOpt('skip-gnocchi-resource-types',
help='Skip gnocchi resource-types upgrade.',
default=False),
cfg.IntOpt('retry',
min=0,
help='Number of times to retry on failure. '
'Default is to retry forever.'),
])
service.prepare_service(conf=conf)
if conf.skip_gnocchi_resource_types:
LOG.info("Skipping Gnocchi resource types upgrade")
else:
LOG.debug("Upgrading Gnocchi resource types")
from ceilometer import gnocchi_client
from gnocchiclient import exceptions
if conf.retry is None:
stop = tenacity.stop_never
else:
stop = tenacity.stop_after_attempt(conf.retry)
tenacity.Retrying(
stop=stop,
retry=tenacity.retry_if_exception_type((
exceptions.ConnectionFailure,
exceptions.UnknownConnectionError,
exceptions.ConnectionTimeout,
exceptions.SSLError,
))
)(gnocchi_client.upgrade_resource_types, conf)
def retry_random_upon_exception(exc, delay=0.5, max_delay=5,
max_attempts=DEFAULT_MAX_ATTEMPTS):
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_exception_type(exc),
wait=tenacity.wait_random_exponential(
multiplier=delay, max=max_delay),
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
def _safe_mongo_call(max_retries, retry_interval):
return tenacity.retry(
retry=tenacity.retry_if_exception_type(
pymongo.errors.AutoReconnect),
wait=tenacity.wait_fixed(retry_interval),
stop=(tenacity.stop_after_attempt(max_retries) if max_retries >= 0
else tenacity.stop_never)
)
def retry_on_conflict(func):
wrapper = tenacity.retry(
stop=tenacity.stop_after_attempt(11),
wait=tenacity.wait_random(max=0.002),
retry=tenacity.retry_if_exception_type(exception.ConcurrentTransaction),
reraise=True,
)
return wrapper(func)
def decorator(func):
@tenacity.retry(
retry=tenacity.retry_if_exception(retry_on_retriable_kafka_error),
wait=tenacity.wait_fixed(1),
stop=tenacity.stop_after_attempt(retries),
reraise=True
)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
def retry_upon_none_result(max_attempts, delay=0.5, max_delay=2, random=False):
if random:
wait_func = tenacity.wait_exponential(
multiplier=delay, max=max_delay)
else:
wait_func = tenacity.wait_random_exponential(
multiplier=delay, max=max_delay)
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_result(lambda x: x is None),
wait=wait_func,
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
def get_connection_from_config(conf):
retries = conf.database.max_retries
@tenacity.retry(
wait=tenacity.wait_fixed(conf.database.retry_interval),
stop=(tenacity.stop_after_attempt(retries) if retries >= 0
else tenacity.stop_never),
reraise=True)
def _inner():
url = (getattr(conf.database, 'metering_connection') or
conf.database.connection)
return get_connection(conf, url)
return _inner()
def get_connection_from_config(conf, purpose='metering'):
retries = conf.database.max_retries
@tenacity.retry(
wait=tenacity.wait_fixed(conf.database.retry_interval),
stop=(tenacity.stop_after_attempt(retries) if retries >= 0
else tenacity.stop_never),
reraise=True)
def _inner():
namespace = 'ceilometer.%s.storage' % purpose
url = (getattr(conf.database, '%s_connection' % purpose) or
conf.database.connection)
return get_connection(conf, url, namespace)
return _inner()
def get_connection_from_config(conf):
retries = conf.database.max_retries
@tenacity.retry(
reraise=True,
wait=tenacity.wait_fixed(conf.database.retry_interval),
stop=(tenacity.stop_after_attempt(retries) if retries >= 0
else tenacity.stop_never)
)
def _inner():
url = (conf.database.connection or
getattr(conf.database, 'event_connection', None))
return get_connection(url, conf)
return _inner()
def __init__(self, conf):
super(GnocchiDispatcher, self).__init__(conf)
self.conf = conf
self.filter_service_activity = (
conf.dispatcher_gnocchi.filter_service_activity)
self._ks_client = keystone_client.get_client(conf)
self.resources_definition = self._load_resources_definitions(conf)
self.cache = None
try:
import oslo_cache
oslo_cache.configure(self.conf)
# NOTE(cdent): The default cache backend is a real but
# noop backend. We don't want to use that here because
# we want to avoid the cache pathways entirely if the
# cache has not been configured explicitly.
if self.conf.cache.enabled:
cache_region = oslo_cache.create_region()
self.cache = oslo_cache.configure_cache_region(
self.conf, cache_region)
self.cache.key_mangler = cache_key_mangler
except ImportError:
pass
except oslo_cache.exception.ConfigurationError as exc:
LOG.warning(_LW('unable to configure oslo_cache: %s') % exc)
self._gnocchi_project_id = None
self._gnocchi_project_id_lock = threading.Lock()
self._gnocchi_resource_lock = LockedDefaultDict(threading.Lock)
self._gnocchi = gnocchi_client.get_gnocchiclient(conf)
retries = conf.storage.max_retries
@tenacity.retry(
wait=tenacity.wait_fixed(conf.storage.retry_interval),
stop=(tenacity.stop_after_attempt(retries) if retries >= 0
else tenacity.stop_never),
reraise=True)
def _get_connection():
self._gnocchi.capabilities.list()
try:
_get_connection()
except Exception:
LOG.error(_LE('Failed to connect to Gnocchi.'))
raise
def get_connection_from_config(conf):
retries = conf.database.max_retries
url = conf.database.connection
connection_scheme = urlparse.urlparse(url).scheme
LOG.debug('looking for %(name)r driver in %(namespace)r',
{'name': connection_scheme, 'namespace': _NAMESPACE})
mgr = driver.DriverManager(_NAMESPACE, connection_scheme)
@tenacity.retry(
wait=tenacity.wait_fixed(conf.database.retry_interval),
stop=tenacity.stop_after_attempt(retries if retries >= 0 else 5),
reraise=True)
def _get_connection():
"""Return an open connection to the database."""
return mgr.driver(conf, url)
return _get_connection()
def send_notification(self, target, ctxt, message, version, retry=None):
if retry is None:
retry = self._pika_engine.default_notification_retry_attempts
def on_exception(ex):
if isinstance(ex, (pika_drv_exc.ExchangeNotFoundException,
pika_drv_exc.RoutingException)):
LOG.warning("Problem during sending notification. %s", ex)
try:
self._declare_notification_queue_binding(target)
except pika_drv_exc.ConnectionException as e:
LOG.warning("Problem during declaring notification queue "
"binding. %s", e)
return True
elif isinstance(ex, (pika_drv_exc.ConnectionException,
pika_drv_exc.MessageRejectedException)):
LOG.warning("Problem during sending notification. %s", ex)
return True
else:
return False
if retry:
retrier = tenacity.retry(
stop=(tenacity.stop_never if retry == -1 else
tenacity.stop_after_attempt(retry)),
retry=tenacity.retry_if_exception(on_exception),
wait=tenacity.wait_fixed(
self._pika_engine.notification_retry_delay
)
)
else:
retrier = None
msg = pika_drv_msg.PikaOutgoingMessage(self._pika_engine, message,
ctxt)
return msg.send(
exchange=(
target.exchange or
self._pika_engine.default_notification_exchange
),
routing_key=target.topic,
confirm=True,
mandatory=True,
persistent=self._pika_engine.notification_persistence,
retrier=retrier
)
def send_request(self, socket, request):
if hasattr(request, 'timeout'):
_stop = tenacity.stop_after_delay(request.timeout)
elif request.retry is not None and request.retry > 0:
# no rpc_response_timeout option if notification
_stop = tenacity.stop_after_attempt(request.retry)
else:
# well, now what?
_stop = tenacity.stop_after_delay(60)
@tenacity.retry(retry=tenacity.retry_if_exception_type(zmq.Again),
stop=_stop)
def send_retrying():
if request.msg_type in zmq_names.MULTISEND_TYPES:
for _ in range(socket.connections_count()):
self.sender.send(socket, request)
else:
self.sender.send(socket, request)
return send_retrying()
def retry_upon_exception_exclude_error_codes(
exc, excluded_errors, delay, max_delay, max_attempts):
"""Retry with the configured exponential delay, unless the exception error
code is in the given list
"""
def retry_if_not_error_codes(e):
# return True only for BadRequests without error codes or with error
# codes not in the exclude list
if isinstance(e, exc):
error_code = _get_bad_request_error_code(e)
if error_code and error_code not in excluded_errors:
return True
return False
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_exception(
retry_if_not_error_codes),
wait=tenacity.wait_exponential(
multiplier=delay, max=max_delay),
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.readthedocs.org/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
engine = engine_from_config(config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
connection = tenacity.Retrying(
stop=tenacity.stop_after_attempt(100),
wait=tenacity.wait_random(min=2, max=5),
before=tenacity.before_log(logging.getLogger("tenacity.retry"), logging.DEBUG),
before_sleep=tenacity.before_sleep_log(logging.getLogger("tenacity.retry"), logging.INFO),
after=tenacity.after_log(logging.getLogger("tenacity.retry"), logging.DEBUG)
).call(engine.connect)
context.configure(connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def test_retry_on_conn_error(self, mocked_session):
retry_args = dict(
wait=tenacity.wait_none(),
stop=tenacity.stop_after_attempt(7),
retry=requests.exceptions.ConnectionError
)
def send_and_raise(request, **kwargs):
raise requests.exceptions.ConnectionError
mocked_session().send.side_effect = send_and_raise
# The job failed for some reason
with self.assertRaises(tenacity.RetryError):
self.get_hook.run_with_advanced_retry(
endpoint='v1/test',
_retry_args=retry_args
)
self.assertEqual(
self.get_hook._retry_obj.stop.max_attempt_number + 1,
mocked_session.call_count
)
return compile.parse1(formula_string, theories=theories)
def str2pol(policy_string, theories=None):
return compile.parse(policy_string, theories=theories)
def pol2str(policy):
return " ".join(str(x) for x in policy)
def form2str(formula):
return str(formula)
@tenacity.retry(stop=tenacity.stop_after_attempt(1000),
wait=tenacity.wait_fixed(0.1))
def retry_check_for_last_message(obj):
if not hasattr(obj, "last_msg"):
raise AttributeError("Missing 'last_msg' attribute")
@tenacity.retry(stop=tenacity.stop_after_attempt(1000),
wait=tenacity.wait_fixed(0.1))
def retry_check_for_message_to_arrive(obj):
if not hasattr(obj.msg, "body"):
raise AttributeError("Missing 'body' attribute")
@tenacity.retry(stop=tenacity.stop_after_attempt(1000),
wait=tenacity.wait_fixed(0.1))
def request(self,
method,
additional_headers=None,
retry=True,
timeout=None,
auth=None,
use_gzip_encoding=None,
params=None,
max_attempts=None,
**kwargs):
"""
Make an HTTP request by calling self._request with backoff retry.
:param method: request method
:type method: str
:param additional_headers: additional headers to include in the request
:type additional_headers: dict[str, str]
:param retry: boolean indicating whether to retry if the request fails
:type retry: boolean
:param timeout: timeout in seconds, overrides default_timeout_secs
:type timeout: float
:param timeout: timeout in seconds
:type timeout: float
:param auth: auth scheme for the request
:type auth: requests.auth.AuthBase
:param use_gzip_encoding: boolean indicating whether to pass gzip
encoding in the request headers or not
:type use_gzip_encoding: boolean | None
:param params: additional params to include in the request
:type params: str | dict[str, T] | None
:param max_attempts: maximum number of attempts to try for any request
:type max_attempts: int
:param kwargs: additional arguments to pass to requests.request
:type kwargs: dict[str, T]
:return: HTTP response
:rtype: requests.Response
"""
request = self._request
if retry:
if max_attempts is None:
max_attempts = self.default_max_attempts
# We retry only when it makes sense: either due to a network
# partition (e.g. connection errors) or if the request failed
# due to a server error such as 500s, timeouts, and so on.
request = tenacity.retry(
stop=tenacity.stop_after_attempt(max_attempts),
wait=tenacity.wait_exponential(),
retry=tenacity.retry_if_exception_type((
requests.exceptions.Timeout,
requests.exceptions.ConnectionError,
MesosServiceUnavailableException,
MesosInternalServerErrorException,
)),
reraise=True,
)(request)
try:
return request(
method=method,
additional_headers=additional_headers,
timeout=timeout,
auth=auth,
use_gzip_encoding=use_gzip_encoding,
params=params,
**kwargs
)
# If the request itself failed, an exception subclassed from
# RequestException will be raised. Catch this and reraise as
# MesosException since we want the caller to be able to catch
# and handle this.
except requests.exceptions.RequestException as err:
raise MesosException('Request failed', err)
class VolumeManager(volume_base.VolumeBase):
'''Volume implementation of virtual machines via cinder.'''
def __init__(self):
super(VolumeManager, self).__init__()
# Must initialize cinder api
self._cinder_client = clients.CinderAuth.get_cinder_client(
service_name=CONF.cinder.service_name,
endpoint=CONF.cinder.endpoint,
region=CONF.cinder.region_name,
endpoint_type=CONF.cinder.endpoint_type,
insecure=CONF.cinder.insecure,
cacert=CONF.cinder.ca_certificates_file
)
self.manager = self._cinder_client.volumes
@retry(reraise=True,
stop=stop_after_attempt(CONF.cinder.volume_create_max_retries))
def create_volume_from_image(self, image_id):
"""Create cinder volume
:param image_id: ID of amphora image
:return volume id
"""
volume = self.manager.create(
size=CONF.cinder.volume_size,
volume_type=CONF.cinder.volume_type,
availability_zone=CONF.cinder.availability_zone,
imageRef=image_id)
resource_status = self.manager.get(volume.id).status
status = constants.CINDER_STATUS_AVAILABLE
start = int(time.time())
while resource_status != status:
time.sleep(CONF.cinder.volume_create_retry_interval)
instance_volume = self.manager.get(volume.id)
resource_status = instance_volume.status
if resource_status == constants.CINDER_STATUS_ERROR:
LOG.error('Error creating %s', instance_volume.id)
instance_volume.delete()
raise cinder_exceptions.ResourceInErrorState(
obj=volume, fault_msg='Cannot create volume')
if int(time.time()) - start >= CONF.cinder.volume_create_timeout:
LOG.error('Timed out waiting to create cinder volume %s',
instance_volume.id)
instance_volume.delete()
raise cinder_exceptions.TimeoutException(
obj=volume, action=constants.CINDER_ACTION_CREATE_VOLUME)
return volume.id
def delete_volume(self, volume_id):
"""Get glance image from volume
:param volume_id: ID of amphora boot volume
:return image id
"""
LOG.debug('Deleting cinder volume %s', volume_id)
try:
instance_volume = self.manager.get(volume_id)
try:
instance_volume.delete()
LOG.debug("Deleted volume %s", volume_id)
except Exception:
LOG.exception("Error deleting cinder volume %s",
volume_id)
raise exceptions.VolumeDeleteException()
except cinder_exceptions.NotFound:
LOG.warning("Volume %s not found: assuming already deleted",
volume_id)
def get_image_from_volume(self, volume_id):
"""Get glance image from volume
:param volume_id: ID of amphora boot volume
:return image id
"""
image_id = None
LOG.debug('Get glance image for volume %s', volume_id)
try:
instance_volume = self.manager.get(volume_id)
except cinder_exceptions.NotFound:
LOG.exception("Volume %s not found", volume_id)
raise exceptions.VolumeGetException()
if hasattr(instance_volume, 'volume_image_metadata'):
image_id = instance_volume.volume_image_metadata.get("image_id")
else:
LOG.error("Volume %s has no image metadata", volume_id)
image_id = None
return image_id
elif (ctxt.pull_from_fork and ctxt.pull["base"]["repo"]["private"]
and not ctxt.pull["maintainer_can_modify"]):
return check_api.Result(
check_api.Conclusion.FAILURE,
"Pull request can't be updated with latest base branch changes",
"Mergify needs the permission to update the base branch of the pull request.\n"
"GitHub does not allow a GitHub App to modify base branch for a private fork.\n"
"You cannot `rebase` a pull request from a private fork.",
)
else:
return None
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=0.2),
stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_if_exception_type(BranchUpdateNeedRetry),
)
def _do_rebase(ctxt: context.Context, token: str) -> None:
# NOTE(sileht):
# $ curl https://api.github.com/repos/sileht/repotest/pulls/2 | jq .commits
# 2
# $ git clone https://[email protected]/sileht-tester/repotest \
# --depth=$((2 + 1)) -b sileht/testpr
# $ cd repotest
# $ git remote add upstream https://[email protected]/sileht/repotest.git
# $ git log | grep Date | tail -1
# Date: Fri Mar 30 21:30:26 2018 (10 days ago)
# $ git fetch upstream master --shallow-since="Fri Mar 30 21:30:26 2018"
# $ git rebase upstream/master
# $ git push origin sileht/testpr:sileht/testpr
def launch_instance(instance_key, use_boot_volume=False, vm_name=None,
private_network_name=None, image_name=None,
flavor_name=None, external_network_name=None, meta=None,
userdata=None):
"""Launch an instance.
:param instance_key: Key to collect associated config data with.
:type instance_key: str
:param use_boot_volume: Whether to boot guest from a shared volume.
:type use_boot_volume: boolean
:param vm_name: Name to give guest.
:type vm_name: str
:param private_network_name: Name of private network to attach guest to.
:type private_network_name: str
:param image_name: Image name to use with guest.
:type image_name: str
:param flavor_name: Flavor name to use with guest.
:type flavor_name: str
:param external_network_name: External network to create floating ip from
for guest.
:type external_network_name: str
:param meta: A dict of arbitrary key/value metadata to store for this
server. Both keys and values must be <=255 characters.
:type meta: dict
:param userdata: Configuration to use upon launch, used by cloud-init.
:type userdata: str
:returns: the created instance
:rtype: novaclient.Server
"""
keystone_session = openstack_utils.get_overcloud_keystone_session()
nova_client = openstack_utils.get_nova_session_client(keystone_session)
neutron_client = openstack_utils.get_neutron_session_client(
keystone_session)
# Collect resource information.
vm_name = vm_name or time.strftime("%Y%m%d%H%M%S")
image_name = image_name or boot_tests[instance_key]['image_name']
image = nova_client.glance.find_image(image_name)
flavor_name = flavor_name or boot_tests[instance_key]['flavor_name']
flavor = nova_client.flavors.find(name=flavor_name)
private_network_name = private_network_name or "private"
net = neutron_client.find_resource("network", private_network_name)
nics = [{'net-id': net.get('id')}]
meta = meta or {}
external_network_name = external_network_name or "ext_net"
if use_boot_volume:
bdmv2 = [{
'boot_index': '0',
'uuid': image.id,
'source_type': 'image',
'volume_size': flavor.disk,
'destination_type': 'volume',
'delete_on_termination': True}]
image = None
else:
bdmv2 = None
# Launch instance.
logging.info('Launching instance {}'.format(vm_name))
instance = nova_client.servers.create(
name=vm_name,
image=image,
block_device_mapping_v2=bdmv2,
flavor=flavor,
key_name=nova_utils.KEYPAIR_NAME,
meta=meta,
nics=nics,
userdata=userdata)
# Test Instance is ready.
logging.info('Checking instance is active')
openstack_utils.resource_reaches_status(
nova_client.servers,
instance.id,
expected_status='ACTIVE',
# NOTE(lourot): in some models this may sometimes take more than 15
# minutes. See lp:1945991
wait_iteration_max_time=120,
stop_after_attempt=16)
logging.info('Checking cloud init is complete')
openstack_utils.cloud_init_complete(
nova_client,
instance.id,
boot_tests[instance_key]['bootstring'])
port = openstack_utils.get_ports_from_device_id(
neutron_client,
instance.id)[0]
logging.info('Assigning floating ip.')
ip = openstack_utils.create_floating_ip(
neutron_client,
external_network_name,
port=port)['floating_ip_address']
logging.info('Assigned floating IP {} to {}'.format(ip, vm_name))
try:
for attempt in Retrying(
stop=stop_after_attempt(8),
wait=wait_exponential(multiplier=1, min=2, max=60)):
with attempt:
try:
openstack_utils.ping_response(ip)
except subprocess.CalledProcessError as e:
logging.error('Pinging {} failed with {}'
.format(ip, e.returncode))
logging.error('stdout: {}'.format(e.stdout))
logging.error('stderr: {}'.format(e.stderr))
raise
except RetryError:
raise openstack_exceptions.NovaGuestNoPingResponse()
# Check ssh'ing to instance.
logging.info('Testing ssh access.')
openstack_utils.ssh_test(
username=boot_tests[instance_key]['username'],
ip=ip,
vm_name=vm_name,
password=boot_tests[instance_key].get('password'),
privkey=openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME))
return instance
class GithubGraphQLQuery(object):
log = logging.getLogger(__name__)
def __init__(self, token_getter):
self.url = 'https://api.github.com/graphql'
self.session = requests.session()
# Will get every 25 requests
self.get_rate_limit_rate = 25
self.query_count = 0
# Set an initial value
self.quota_remain = 5000
self.token_getter = token_getter
def get_token(self) -> str:
return self.token_getter.get_token()[0]
def get_headers(self) -> dict:
headers = {
'Authorization': 'token %s' % self.get_token(),
'User-Agent': 'change-metrics/monocle',
}
self.log.debug('request headers: %s' % headers)
return headers
def get_rate_limit(self):
ratelimit = self.getRateLimit()
if ratelimit:
self.quota_remain = ratelimit['remaining']
self.resetat = utils.is8601_to_dt(ratelimit['resetAt'])
self.log.info("Got rate limit data: remain %s resetat %s" %
(self.quota_remain, self.resetat))
# https://developer.github.com/v3/guides/best-practices-for-integrators/#dealing-with-abuse-rate-limits
def wait_for_call(self):
if self.quota_remain <= 150:
until_reset = self.resetat - utils.utcnow()
self.log.info("Quota remain: %s/calls delay until "
"reset: %s/secs waiting ..." %
(self.quota_remain, until_reset.seconds))
sleep(until_reset.seconds + 60)
self.get_rate_limit()
else:
self.log.debug("Sleeping 1 sec to be a good citizen")
sleep(1)
def getRateLimit(self):
qdata = '''{
rateLimit {
limit
cost
remaining
resetAt
}
}'''
data = self.query(qdata, skip_get_rate_limit=True)
if data:
try:
return data['data']['rateLimit']
except KeyError:
self.log.error('No rate limit data: %s' % data)
raise RequestException('No rate limit data: %s' % data)
@retry(
after=after_log(log, logging.INFO),
wait=wait_fixed(10),
stop=stop_after_attempt(30),
retry=retry_if_exception_type(RequestException),
reraise=True,
)
def query(self, qdata, skip_get_rate_limit=False):
if not skip_get_rate_limit:
if self.query_count % self.get_rate_limit_rate == 0:
self.get_rate_limit()
self.wait_for_call()
data = {'query': qdata}
try:
r = self.session.post(url=self.url,
json=data,
headers=self.get_headers(),
timeout=30.3)
except (
requests.exceptions.ConnectionError,
requests.exceptions.ChunkedEncodingError,
):
raise RequestException("Error connecting to the API")
self.query_count += 1
if 'retry-after' in r.headers:
self.log.info('Got Retry-After: %s, sleeping...' %
r.headers['retry-after'])
sleep(int(r.headers['retry-after']))
if not r.status_code != "200":
self.log.error('No ok response code: %s' % r)
raise RequestException("No ok response code: %s" % r.text)
ret = r.json()
if 'Bad credentials' == ret.get('message', ''):
self.log.info('Query forbidden due to bad credentials')
ret = {}
if 'errors' in ret:
self.log.error("Errors in response: %s" % ret)
if (len(ret['errors']) >= 1 and 'message' in ret['errors'][0]
and 'timeout' in ret['errors'][0]['message']):
raise RequestTimeout(ret['errors'][0]['message'])
if len(ret['errors']) >= 1:
if all([
error for error in ret['errors']
if 'The additions count for this commit is unavailable'
in error['message']
]):
# This errors are not critical, PRs data are complete, w/o
# the failing commit(s). So return the data to the caller and
# move on.
return ret
is_forbiden = any([error['type'] == 'FORBIDDEN']
for error in ret['errors'])
if is_forbiden:
# Do not raise to not retrigger tenacity
self.log.info('Query forbidden due to unsuffcient token ACLs')
ret = {}
else:
raise RequestException("Errors in response: %s" %
ret['errors'])
return ret
@six.wraps(callable_)
def wrapper(*a, **kw):
loop = asyncio.get_event_loop()
return loop.run_until_complete(callable_(*a, **kw))
return wrapper
@retry
@asyncio.coroutine
def _retryable_coroutine(thing):
yield from asyncio.sleep(0.00001)
return thing.go()
@retry(stop=stop_after_attempt(2))
@asyncio.coroutine
def _retryable_coroutine_with_2_attempts(thing):
yield from asyncio.sleep(0.00001)
thing.go()
class TestAsync(unittest.TestCase):
@asynctest
def test_retry(self):
assert asyncio.iscoroutinefunction(_retryable_coroutine)
thing = NoIOErrorAfterCount(5)
yield from _retryable_coroutine(thing)
assert thing.counter == thing.count
@asynctest
class NeutronNetworkingTest(unittest.TestCase):
"""Ensure that openstack instances have valid networking."""
RESOURCE_PREFIX = 'zaza-neutrontests'
@classmethod
def setUpClass(cls):
"""Run class setup for running Neutron API Networking tests."""
cls.keystone_session = (
openstack_utils.get_overcloud_keystone_session())
cls.nova_client = (openstack_utils.get_nova_session_client(
cls.keystone_session))
# NOTE(fnordahl): in the event of a test failure we do not want to run
# tear down code as it will make debugging a problem virtually
# impossible. To alleviate each test method will set the
# `run_tearDown` instance variable at the end which will let us run
# tear down only when there were no failure.
cls.run_tearDown = False
@classmethod
def tearDown(cls):
"""Remove test resources."""
if cls.run_tearDown:
logging.info('Running teardown')
for server in cls.nova_client.servers.list():
if server.name.startswith(cls.RESOURCE_PREFIX):
openstack_utils.delete_resource(cls.nova_client.servers,
server.id,
msg="server")
def test_instances_have_networking(self):
"""Validate North/South and East/West networking."""
guest.launch_instance(glance_setup.LTS_IMAGE_NAME,
vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX))
guest.launch_instance(glance_setup.LTS_IMAGE_NAME,
vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX))
instance_1 = self.nova_client.servers.find(
name='{}-ins-1'.format(self.RESOURCE_PREFIX))
instance_2 = self.nova_client.servers.find(
name='{}-ins-2'.format(self.RESOURCE_PREFIX))
def verify(stdin, stdout, stderr):
"""Validate that the SSH command exited 0."""
self.assertEqual(stdout.channel.recv_exit_status(), 0)
# Verify network from 1 to 2
self.validate_instance_can_reach_other(instance_1, instance_2, verify)
# Verify network from 2 to 1
self.validate_instance_can_reach_other(instance_2, instance_1, verify)
# Validate tenant to external network routing
self.validate_instance_can_reach_router(instance_1, verify)
self.validate_instance_can_reach_router(instance_2, verify)
# If we get here, it means the tests passed
self.run_tearDown = True
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
reraise=True,
stop=tenacity.stop_after_attempt(8))
def validate_instance_can_reach_other(self, instance_1, instance_2,
verify):
"""
Validate that an instance can reach a fixed and floating of another.
:param instance_1: The instance to check networking from
:type instance_1: nova_client.Server
:param instance_2: The instance to check networking from
:type instance_2: nova_client.Server
"""
floating_1 = floating_ips_from_instance(instance_1)[0]
floating_2 = floating_ips_from_instance(instance_2)[0]
address_2 = fixed_ips_from_instance(instance_2)[0]
username = guest.boot_tests['bionic']['username']
password = guest.boot_tests['bionic'].get('password')
privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME)
openstack_utils.ssh_command(username,
floating_1,
'instance-1',
'ping -c 1 {}'.format(address_2),
password=password,
privkey=privkey,
verify=verify)
openstack_utils.ssh_command(username,
floating_1,
'instance-1',
'ping -c 1 {}'.format(floating_2),
password=password,
privkey=privkey,
verify=verify)
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60),
reraise=True,
stop=tenacity.stop_after_attempt(8))
def validate_instance_can_reach_router(self, instance, verify):
"""
Validate that an instance can reach it's primary gateway.
We make the assumption that the router's IP is 192.168.0.1
as that's the network that is setup in
neutron.setup.basic_overcloud_network which is used in all
Zaza Neutron validations.
:param instance: The instance to check networking from
:type instance: nova_client.Server
"""
address = floating_ips_from_instance(instance)[0]
username = guest.boot_tests['bionic']['username']
password = guest.boot_tests['bionic'].get('password')
privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME)
openstack_utils.ssh_command(username,
address,
'instance',
'ping -c 1 192.168.0.1',
password=password,
privkey=privkey,
verify=verify)
pass
class PodLauncher(LoggingMixin):
"""Launches PODS"""
def __init__(self,
kube_client: client.CoreV1Api = None,
in_cluster: bool = True,
cluster_context: Optional[str] = None,
extract_xcom: bool = False):
"""
Creates the launcher.
:param kube_client: kubernetes client
:param in_cluster: whether we are in cluster
:param cluster_context: context of the cluster
:param extract_xcom: whether we should extract xcom
"""
super().__init__()
self._client = kube_client or get_kube_client(
in_cluster=in_cluster, cluster_context=cluster_context)
self._watch = watch.Watch()
self.extract_xcom = extract_xcom
def run_pod_async(self, pod: V1Pod, **kwargs):
"""Runs POD asynchronously"""
pod_mutation_hook(pod)
sanitized_pod = self._client.api_client.sanitize_for_serialization(pod)
json_pod = json.dumps(sanitized_pod, indent=2)
self.log.debug('Pod Creation Request: \n%s', json_pod)
try:
resp = self._client.create_namespaced_pod(
body=sanitized_pod, namespace=pod.metadata.namespace, **kwargs)
self.log.debug('Pod Creation Response: %s', resp)
except Exception as e:
self.log.exception(
'Exception when attempting '
'to create Namespaced Pod: %s', json_pod)
raise e
return resp
def delete_pod(self, pod: V1Pod):
"""Deletes POD"""
try:
self._client.delete_namespaced_pod(pod.metadata.name,
pod.metadata.namespace,
body=client.V1DeleteOptions())
except ApiException as e:
# If the pod is already deleted
if e.status != 404:
raise
def run_pod(self,
pod: V1Pod,
startup_timeout: int = 120,
get_logs: bool = True) -> Tuple[State, Optional[str]]:
"""
Launches the pod synchronously and waits for completion.
:param pod:
:param startup_timeout: Timeout for startup of the pod (if pod is pending for too long, fails task)
:param get_logs: whether to query k8s for logs
:return:
"""
resp = self.run_pod_async(pod)
curr_time = dt.now()
if resp.status.start_time is None:
while self.pod_not_started(pod):
delta = dt.now() - curr_time
if delta.total_seconds() >= startup_timeout:
raise AirflowException("Pod took too long to start")
time.sleep(1)
self.log.debug('Pod not yet started')
return self._monitor_pod(pod, get_logs)
def _monitor_pod(self, pod: V1Pod,
get_logs: bool) -> Tuple[State, Optional[str]]:
if get_logs:
logs = self.read_pod_logs(pod)
for line in logs:
self.log.info(line)
result = None
if self.extract_xcom:
while self.base_container_is_running(pod):
self.log.info('Container %s has state %s', pod.metadata.name,
State.RUNNING)
time.sleep(2)
result = self._extract_xcom(pod)
self.log.info(result)
result = json.loads(result)
while self.pod_is_running(pod):
self.log.info('Pod %s has state %s', pod.metadata.name,
State.RUNNING)
time.sleep(2)
return self._task_status(self.read_pod(pod)), result
def _task_status(self, event):
self.log.info('Event: %s had an event of type %s', event.metadata.name,
event.status.phase)
status = self.process_status(event.metadata.name, event.status.phase)
return status
def pod_not_started(self, pod: V1Pod):
"""Tests if pod has not started"""
state = self._task_status(self.read_pod(pod))
return state == State.QUEUED
def pod_is_running(self, pod: V1Pod):
"""Tests if pod is running"""
state = self._task_status(self.read_pod(pod))
return state not in (State.SUCCESS, State.FAILED)
def base_container_is_running(self, pod: V1Pod):
"""Tests if base container is running"""
event = self.read_pod(pod)
status = next(
iter(
filter(lambda s: s.name == 'base',
event.status.container_statuses)), None)
if not status:
return False
return status.state.running is not None
@tenacity.retry(stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_exponential(),
reraise=True)
def read_pod_logs(self, pod: V1Pod):
"""Reads log from the POD"""
try:
return self._client.read_namespaced_pod_log(
name=pod.metadata.name,
namespace=pod.metadata.namespace,
container='base',
follow=True,
tail_lines=10,
_preload_content=False)
except BaseHTTPError as e:
raise AirflowException(
'There was an error reading the kubernetes API: {}'.format(e))
@tenacity.retry(stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_exponential(),
reraise=True)
def read_pod_events(self, pod):
"""Reads events from the POD"""
try:
return self._client.list_namespaced_event(
namespace=pod.metadata.namespace,
field_selector="involvedObject.name={}".format(
pod.metadata.name))
except BaseHTTPError as e:
raise AirflowException(
'There was an error reading the kubernetes API: {}'.format(e))
@tenacity.retry(stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_exponential(),
reraise=True)
def read_pod(self, pod: V1Pod):
"""Read POD information"""
try:
return self._client.read_namespaced_pod(pod.metadata.name,
pod.metadata.namespace)
except BaseHTTPError as e:
raise AirflowException(
'There was an error reading the kubernetes API: {}'.format(e))
def _extract_xcom(self, pod: V1Pod):
resp = kubernetes_stream(self._client.connect_get_namespaced_pod_exec,
pod.metadata.name,
pod.metadata.namespace,
container=PodDefaults.SIDECAR_CONTAINER_NAME,
command=['/bin/sh'],
stdin=True,
stdout=True,
stderr=True,
tty=False,
_preload_content=False)
try:
result = self._exec_pod_command(
resp, 'cat {}/return.json'.format(PodDefaults.XCOM_MOUNT_PATH))
self._exec_pod_command(resp, 'kill -s SIGINT 1')
finally:
resp.close()
if result is None:
raise AirflowException(
'Failed to extract xcom from pod: {}'.format(
pod.metadata.name))
return result
def _exec_pod_command(self, resp, command):
if resp.is_open():
self.log.info('Running command... %s\n', command)
resp.write_stdin(command + '\n')
while resp.is_open():
resp.update(timeout=1)
if resp.peek_stdout():
return resp.read_stdout()
if resp.peek_stderr():
self.log.info(resp.read_stderr())
break
return None
def process_status(self, job_id, status):
"""Process status information for the JOB"""
status = status.lower()
if status == PodStatus.PENDING:
return State.QUEUED
elif status == PodStatus.FAILED:
self.log.info('Event with job id %s Failed', job_id)
return State.FAILED
elif status == PodStatus.SUCCEEDED:
self.log.info('Event with job id %s Succeeded', job_id)
return State.SUCCESS
elif status == PodStatus.RUNNING:
return State.RUNNING
else:
self.log.info('Event: Invalid state %s on job %s', status, job_id)
return State.FAILED
try:
return int(value)
except (TypeError, ValueError):
return default
def sfloat(value, default):
if not isinstance(default, int):
raise TypeError("Default value must be of float type")
try:
return float(value)
except (TypeError, ValueError):
return default
@retry(stop=stop_after_attempt(5), wait=wait_exponential(min=1, max=10))
async def get_walkoff_auth_header(session, token=None, timeout=5 * 60):
url = config.API_URI.rstrip('/') + '/walkoff/api'
logger.debug("Attempting to refresh WALKOFF JWT")
if token is None:
key = config.get_from_file(config.INTERNAL_KEY_PATH)
async with session.post(url + "/auth/login", json={"username": config.WALKOFF_USERNAME,
"password": key}, timeout=timeout) as resp:
resp_json = await resp.json()
token = resp_json["refresh_token"]
logger.debug("Successfully logged into WALKOFF")
headers = {"Authorization": f"Bearer {token}"}
async with session.post(url + "/auth/refresh", headers=headers, timeout=timeout) as resp:
resp_json = await resp.json()
access_token = resp_json["access_token"]
def test_manila_share(self):
"""Test that Manila + Ganesha shares can be accessed on two instances.
1. create a share
2. Spawn two servers
3. mount it on both
4. write a file on one
5. read it on the other
6. profit
"""
# Create a share
share = self.manila_client.shares.create(share_type='cephfsnfstype',
name='cephnfsshare1',
share_proto="nfs",
size=1)
# Spawn Servers
instance_1 = guest.launch_instance(glance_setup.LTS_IMAGE_NAME,
vm_name='{}-ins-1'.format(
self.RESOURCE_PREFIX),
userdata=self.INSTANCE_USERDATA)
instance_2 = guest.launch_instance(glance_setup.LTS_IMAGE_NAME,
vm_name='{}-ins-2'.format(
self.RESOURCE_PREFIX),
userdata=self.INSTANCE_USERDATA)
fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0]
fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0]
share.allow(access_type='ip', access=fip_1, access_level='rw')
share.allow(access_type='ip', access=fip_2, access_level='rw')
# Mount Share
username = guest.boot_tests['bionic']['username']
password = guest.boot_tests['bionic'].get('password')
privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME)
# Write a file on instance_1
def verify_setup(stdin, stdout, stderr):
status = stdout.channel.recv_exit_status()
self.assertEqual(status, 0)
mount_path = share.export_locations[0]
for attempt in Retrying(stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1,
min=2,
max=10)):
with attempt:
openstack_utils.ssh_command(
username,
fip_1,
'instance-1',
'sudo mkdir -p /mnt/ceph && '
'sudo mount -t nfs -o nfsvers=4.1,proto=tcp '
'{} /mnt/ceph && '
'echo "test" | sudo tee /mnt/ceph/test'.format(mount_path),
password=password,
privkey=privkey,
verify=verify_setup)
for attempt in Retrying(stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1,
min=2,
max=10)):
with attempt:
# Setup that file on instance_2
openstack_utils.ssh_command(
username,
fip_2,
'instance-2',
'sudo mkdir -p /mnt/ceph && '
'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp '
'{} /mnt/ceph'.format(mount_path),
password=password,
privkey=privkey,
verify=verify_setup)
def verify(stdin, stdout, stderr):
status = stdout.channel.recv_exit_status()
self.assertEqual(status, 0)
out = ""
for line in iter(stdout.readline, ""):
out += line
self.assertEqual(out, "test\n")
openstack_utils.ssh_command(
username,
fip_2,
'instance-2',
'sudo cat /mnt/ceph/test'.format(mount_path),
password=password,
privkey=privkey,
verify=verify)
class Comms_master:
"""
"""
def __init__(self, commsffl):
"""
"""
#self.comms = Comms_master(commsffl)
#self.context_master = context_master
#self.task_name = task_name
self.name = 'pycloudmessenger'
#self.commsffl = ffl.Factory.aggregator(self.context_master, task_name=task_name)
self.commsffl = commsffl
workers = self.commsffl.get_participants()
self.workers_ids = list(workers.keys())
def send(self, message, destiny):
try:
with self.commsffl:
# self.send_to maps between worker_id and pseudo_id
self.commsffl.send(message, destiny)
except Exception as err:
print('\n')
print('*' * 80)
print('Pycloudmessenger ERROR at send: %s' % err)
print('*' * 80)
print('\n')
raise
def broadcast(self, message, receivers_list=None):
# receivers_list are not used here, pycloudmessenger already knows all the recipients
try:
with self.commsffl:
self.commsffl.send(message)
except Exception as err:
print('\n')
print('*' * 80)
print('Pycloudmessenger ERROR at broadcast: %s' % err)
print('*' * 80)
print('\n')
raise
def receive(self, timeout=1):
try:
with self.commsffl:
packet = self.commsffl.receive(timeout)
message = packet.content
pseudo_id = packet.notification['participant']
#sender_ = str(self.workers_addresses_cloud.index(pseudo_id))
#sender = message['sender']
#message.update({'pseudo_id': pseudo_id})
#message.update({'sender_': sender})
message.update({'sender': pseudo_id})
except Exception as err:
if 'pycloudmessenger.ffl.fflapi.TimedOutException' not in str(
type(err)): # we skip the normal timeouts
print('\n')
print('*' * 80)
print('Pycloudmessenger ERROR at receive: %s' % err)
print('*' * 80)
print('\n')
else:
message = None
raise
return message
@tenacity.retry(stop=tenacity.stop_after_attempt(5),
wait=tenacity.wait_random(min=1, max=3))
def receive_poms_123(self, timeout=10):
with self.commsffl:
packet = self.commsffl.receive(timeout)
return packet
import math
import operator
import os.path
import random
import re
import shutil
import uuid
import time
import tenacity
from .lib import mkdir, jsonify, toiter, STRING_TYPES, sip, toabs, first
retry = tenacity.retry(
reraise=True,
stop=tenacity.stop_after_attempt(4),
wait=tenacity.wait_random_exponential(0.5, 60.0),
)
@retry
def read_file(path, mode='rt', lock=False, block=False):
f = open(path, mode)
if lock:
f = read_lock_file(f)
data = f.read()
f.close()
return data
@retry
def write_file(
path, file, mode='wt',
class PartitionCoordinator(object):
"""Workload partitioning coordinator.
This class uses the `tooz` library to manage group membership.
To ensure that the other agents know this agent is still alive,
the `heartbeat` method should be called periodically.
Coordination errors and reconnects are handled under the hood, so the
service using the partition coordinator need not care whether the
coordination backend is down. The `extract_my_subset` will simply return an
empty iterable in this case.
"""
def __init__(self, conf, my_id=None):
self.conf = conf
self.backend_url = self.conf.coordination.backend_url
self._coordinator = None
self._groups = set()
self._my_id = my_id or \
encodeutils.safe_encode(uuidutils.generate_uuid())
def start(self):
if self.backend_url:
try:
self._coordinator = tooz.coordination.get_coordinator(
self.backend_url, self._my_id)
self._coordinator.start()
LOG.info('Coordination backend started successfully.')
except tooz.coordination.ToozError:
LOG.exception('Error connecting to coordination backend.')
def stop(self):
if not self._coordinator:
return
for group in list(self._groups):
self.leave_group(group)
try:
self._coordinator.stop()
except tooz.coordination.ToozError:
LOG.exception('Error connecting to coordination backend.')
finally:
self._coordinator = None
def is_active(self):
return self._coordinator is not None
def heartbeat(self):
if self._coordinator:
if not self._coordinator.is_started:
# re-connect
self.start()
try:
self._coordinator.heartbeat()
except tooz.coordination.ToozError:
LOG.exception('Error sending a heartbeat to coordination '
'backend.')
def join_group(self, group_id):
if (not self._coordinator or not self._coordinator.is_started
or not group_id):
return
@tenacity.retry(wait=tenacity.wait_exponential(
multiplier=self.conf.coordination.retry_backoff,
max=self.conf.coordination.max_retry_interval),
retry=tenacity.retry_if_exception_type(
ErrorJoiningPartitioningGroup))
def _inner():
try:
join_req = self._coordinator.join_group(group_id)
join_req.get()
LOG.info('Joined partitioning group %s', group_id)
except tooz.coordination.MemberAlreadyExist:
return
except tooz.coordination.GroupNotCreated:
create_grp_req = self._coordinator.create_group(group_id)
try:
create_grp_req.get()
except tooz.coordination.GroupAlreadyExist:
pass
raise ErrorJoiningPartitioningGroup()
except tooz.coordination.ToozError:
LOG.exception(
'Error joining partitioning group %s,'
' re-trying', group_id)
raise ErrorJoiningPartitioningGroup()
self._groups.add(group_id)
return _inner()
def leave_group(self, group_id):
if group_id not in self._groups:
return
if self._coordinator:
self._coordinator.leave_group(group_id)
self._groups.remove(group_id)
LOG.info('Left partitioning group %s', group_id)
def _get_members(self, group_id):
if not self._coordinator:
return [self._my_id]
while True:
get_members_req = self._coordinator.get_members(group_id)
try:
return get_members_req.get()
except tooz.coordination.GroupNotCreated:
self.join_group(group_id)
@tenacity.retry(
wait=tenacity.wait_random(max=2),
stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_if_exception_type(MemberNotInGroupError),
reraise=True)
def extract_my_subset(self, group_id, universal_set):
"""Filters an iterable, returning only objects assigned to this agent.
We have a list of objects and get a list of active group members from
`tooz`. We then hash all the objects into buckets and return only
the ones that hashed into *our* bucket.
"""
if not group_id or not self.is_active():
return universal_set
if group_id not in self._groups:
self.join_group(group_id)
try:
members = self._get_members(group_id)
LOG.debug('Members of group: %s, Me: %s', members, self._my_id)
if self._my_id not in members:
LOG.warning('Cannot extract tasks because agent failed to '
'join group properly. Rejoining group.')
self.join_group(group_id)
members = self._get_members(group_id)
if self._my_id not in members:
raise MemberNotInGroupError(group_id, members, self._my_id)
LOG.debug('Members of group: %s, Me: %s', members, self._my_id)
hr = HashRing(members)
LOG.debug('Universal set: %s', universal_set)
my_subset = [
v for v in universal_set if hr.get_node(str(v)) == self._my_id
]
LOG.debug('My subset: %s', my_subset)
return my_subset
except tooz.coordination.ToozError:
LOG.exception('Error getting group membership info from '
'coordination backend.')
return []
class ExchangeInterface():
"""Interface for performing queries against exchange API's
"""
def __init__(self, exchange_config):
"""Initializes ExchangeInterface class
Args:
exchange_config (dict): A dictionary containing configuration for the exchanges.
"""
self.logger = structlog.get_logger()
self.exchanges = {}
# Loads the exchanges using ccxt.
for exchange in exchange_config:
if exchange_config[exchange]['required']['enabled']:
new_exchange = getattr(ccxt, exchange)({
"enableRateLimit": True
})
# sets up api permissions for user if given
if new_exchange:
if 'key' in exchange_config[exchange]['optional']:
new_exchange.apiKey = exchange_config[exchange]['optional']['key']
if 'secret' in exchange_config[exchange]['optional']:
new_exchange.secret = exchange_config[exchange]['optional'][
'secret']
if 'username' in exchange_config[exchange]['optional']:
new_exchange.username = exchange_config[exchange]['optional'][
'username']
if 'password' in exchange_config[exchange]['optional']:
new_exchange.password = exchange_config[exchange]['optional'][
'password']
self.exchanges[new_exchange.id] = new_exchange
else:
self.logger.warn("Unable to load exchange %s", new_exchange)
def override_exchange_config(self):
"""Enables all exchanges regardless of user configuration. Useful for the UI layer.
"""
for exchange in ccxt.exchanges:
self.exchanges[exchange] = getattr(ccxt, exchange)({
"enableRateLimit": True
})
@retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
def get_historical_data(self, market_pair, exchange, time_unit, start_date=None, max_days=100):
"""Get historical OHLCV for a symbol pair
Args:
market_pair (str): Contains the symbol pair to operate on i.e. BURST/BTC
exchange (str): Contains the exchange to fetch the historical data from.
time_unit (str): A string specifying the ccxt time unit i.e. 5m or 1d.
start_date (int, optional): Timestamp in milliseconds.
max_days (int, optional): Defaults to 100. Maximum number of days to fetch data for
if start date is not specified.
Returns:
list: Contains a list of lists which contain timestamp, open, high, low, close, volume.
"""
if not start_date:
max_days_date = datetime.now() - timedelta(days=max_days)
start_date = int(max_days_date.replace(tzinfo=timezone.utc).timestamp() * 1000)
historical_data = self.exchanges[exchange].fetch_ohlcv(
market_pair,
timeframe=time_unit,
since=start_date
)
time.sleep(self.exchanges[exchange].rateLimit / 1000)
return historical_data
@retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
def get_account_markets(self, exchange):
"""Get the symbol pairs listed within a users account.
Args:
exchange (str): Contains the exchange to fetch the data from.
Returns:
dict: A dictionary containing market data for the symbol pairs.
"""
account_markets = {}
account_markets.update(self.exchanges[exchange].fetch_balance())
time.sleep(self.exchanges[exchange].rateLimit / 1000)
return account_markets
@retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
def get_markets_for_exchange(self, exchange):
"""Get market data for all symbol pairs listed on the given exchange.
Args:
exchange (str): Contains the exchange to fetch the data from
Returns:
dict: A dictionary containing market data for all symbol pairs.
"""
exchange_markets = self.exchanges[exchange].load_markets()
return exchange_markets
@retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
def get_exchange_markets(self):
"""Get market data for all symbol pairs listed on all configured exchanges.
Returns:
dict: A dictionary containing market data for all symbol pairs.
"""
exchange_markets = {}
for exchange in self.exchanges:
exchange_markets[exchange] = self.exchanges[exchange].load_markets()
time.sleep(self.exchanges[exchange].rateLimit / 1000)
return exchange_markets
@retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
def get_symbol_markets(self, market_pairs):
"""Get market data for specific symbols on all configured exchanges.
Args:
market_pairs (list): The symbol pairs you want to retrieve market data for.
Returns:
dict: A dictionary containing market data for requested symbol pairs.
"""
symbol_markets = {}
for exchange in self.exchanges:
self.exchanges[exchange].load_markets()
symbol_markets[exchange] = {}
for market_pair in market_pairs:
symbol_markets[exchange][market_pair] = self.exchanges[exchange].markets[
market_pair]
time.sleep(self.exchanges[exchange].rateLimit / 1000)
return symbol_markets
@retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
def get_order_book(self, market_pair, exchange):
"""Retrieve the order information for a particular symbol pair.
Args:
market_pair (str): Contains the symbol pair to operate on i.e. BURST/BTC
exchange (str): Contains the exchange to fetch the data from.
Returns:
dict: A dictionary containing bid, ask and other order information on a pair.
"""
return self.exchanges[exchange].fetch_order_book(market_pair)
@retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
def get_open_orders(self):
"""Get the users currently open orders on all configured exchanges.
Returns:
dict: A dictionary containing open order information.
"""
open_orders = {}
for exchange in self.exchanges:
open_orders[exchange] = self.exchanges[exchange].fetch_open_orders()
time.sleep(self.exchanges[exchange].rateLimit / 1000)
return open_orders
@retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
def cancel_order(self, exchange, order_id):
"""Cancels an open order on a particular exchange.
Args:
exchange (str): Contains the exchange to cancel the order on.
order_id (str): The order id you want to cancel.
"""
self.exchanges[exchange].cancel_order(order_id)
time.sleep(self.exchanges[exchange].rateLimit / 1000)
@retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
def get_quote_symbols(self, exchange):
"""Get a list of quote symbols on an exchange.
Args:
exchange (str): Contains the exchange to fetch the data from.
Returns:
list: List of quote symbols on an exchange.
"""
quote_symbols = []
for market_pair in self.exchanges[exchange].markets:
_, quote_symbol = market_pair.split('/')
if not quote_symbol in quote_symbols:
quote_symbols.append(quote_symbol)
return quote_symbols
@retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3))
def get_btc_value(self, exchange, base_symbol, volume):
btc_value = 0
market_pair = base_symbol + "/BTC"
order_book = self.get_order_book(market_pair, exchange)
bid = order_book['bids'][0][0] if order_book['bids'] else None
if bid:
btc_value = bid * volume
return btc_value
class URLFile:
_tlocal = threading.local()
def __init__(self, url, debug=False, cache=None):
self._url = url
self._pos = 0
self._length = None
self._local_file = None
self._debug = debug
# True by default, false if FILEREADER_CACHE is defined, but can be overwritten by the cache input
self._force_download = not int(os.environ.get("FILEREADER_CACHE", "0"))
if cache is not None:
self._force_download = not cache
try:
self._curl = self._tlocal.curl
except AttributeError:
self._curl = self._tlocal.curl = pycurl.Curl()
mkdirs_exists_ok(CACHE_DIR)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
if self._local_file is not None:
os.remove(self._local_file.name)
self._local_file.close()
self._local_file = None
@retry(wait=wait_random_exponential(multiplier=1, max=5),
stop=stop_after_attempt(3),
reraise=True)
def get_length_online(self):
c = self._curl
c.reset()
c.setopt(pycurl.NOSIGNAL, 1)
c.setopt(pycurl.TIMEOUT_MS, 500000)
c.setopt(pycurl.FOLLOWLOCATION, True)
c.setopt(pycurl.URL, self._url)
c.setopt(c.NOBODY, 1)
c.perform()
length = int(c.getinfo(c.CONTENT_LENGTH_DOWNLOAD))
c.reset()
return length
def get_length(self):
if self._length is not None:
return self._length
file_length_path = os.path.join(CACHE_DIR,
hash_256(self._url) + "_length")
if os.path.exists(file_length_path) and not self._force_download:
with open(file_length_path) as file_length:
content = file_length.read()
self._length = int(content)
return self._length
self._length = self.get_length_online()
if not self._force_download:
with atomic_write_in_dir(file_length_path,
mode="w") as file_length:
file_length.write(str(self._length))
return self._length
def read(self, ll=None):
if self._force_download:
return self.read_aux(ll=ll)
file_begin = self._pos
file_end = self._pos + ll if ll is not None else self.get_length()
# We have to align with chunks we store. Position is the begginiing of the latest chunk that starts before or at our file
position = (file_begin // CHUNK_SIZE) * CHUNK_SIZE
response = b""
while True:
self._pos = position
chunk_number = self._pos / CHUNK_SIZE
file_name = hash_256(self._url) + "_" + str(chunk_number)
full_path = os.path.join(CACHE_DIR, str(file_name))
data = None
# If we don't have a file, download it
if not os.path.exists(full_path):
data = self.read_aux(ll=CHUNK_SIZE)
with atomic_write_in_dir(full_path,
mode="wb") as new_cached_file:
new_cached_file.write(data)
else:
with open(full_path, "rb") as cached_file:
data = cached_file.read()
response += data[max(0, file_begin -
position):min(CHUNK_SIZE, file_end -
position)]
position += CHUNK_SIZE
if position >= file_end:
self._pos = file_end
return response
@retry(wait=wait_random_exponential(multiplier=1, max=5),
stop=stop_after_attempt(3),
reraise=True)
def read_aux(self, ll=None):
download_range = False
headers = ["Connection: keep-alive"]
if self._pos != 0 or ll is not None:
if ll is None:
end = self.get_length() - 1
else:
end = min(self._pos + ll, self.get_length()) - 1
if self._pos >= end:
return b""
headers.append(f"Range: bytes={self._pos}-{end}")
download_range = True
dats = BytesIO()
c = self._curl
c.setopt(pycurl.URL, self._url)
c.setopt(pycurl.WRITEDATA, dats)
c.setopt(pycurl.NOSIGNAL, 1)
c.setopt(pycurl.TIMEOUT_MS, 500000)
c.setopt(pycurl.HTTPHEADER, headers)
c.setopt(pycurl.FOLLOWLOCATION, True)
if self._debug:
print("downloading", self._url)
def header(x):
if b'MISS' in x:
print(x.strip())
c.setopt(pycurl.HEADERFUNCTION, header)
def test(debug_type, debug_msg):
print(" debug(%d): %s" % (debug_type, debug_msg.strip()))
c.setopt(pycurl.VERBOSE, 1)
c.setopt(pycurl.DEBUGFUNCTION, test)
t1 = time.time()
c.perform()
if self._debug:
t2 = time.time()
if t2 - t1 > 0.1:
print(f"get {self._url} {headers!r} {t2 - t1:.f} slow")
response_code = c.getinfo(pycurl.RESPONSE_CODE)
if response_code == 416: # Requested Range Not Satisfiable
raise Exception(
f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}"
)
if download_range and response_code != 206: # Partial Content
raise Exception(
f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}"
)
if (not download_range) and response_code != 200: # OK
raise Exception(
f"Error {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}"
)
ret = dats.getvalue()
self._pos += len(ret)
return ret
def seek(self, pos):
self._pos = pos
@property
def name(self):
"""Returns a local path to file with the URLFile's contents.
This can be used to interface with modules that require local files.
"""
if self._local_file is None:
_, ext = os.path.splitext(urllib.parse.urlparse(self._url).path)
local_fd, local_path = tempfile.mkstemp(suffix=ext)
try:
os.write(local_fd, self.read())
local_file = open(local_path, "rb")
except Exception:
os.remove(local_path)
raise
finally:
os.close(local_fd)
self._local_file = local_file
self.read = self._local_file.read
self.seek = self._local_file.seek
return self._local_file.name
class PodManager(LoggingMixin):
"""
Helper class for creating, monitoring, and otherwise interacting with Kubernetes pods
for use with the KubernetesPodOperator
"""
def __init__(
self,
kube_client: client.CoreV1Api = None,
in_cluster: bool = True,
cluster_context: Optional[str] = None,
):
"""
Creates the launcher.
:param kube_client: kubernetes client
:param in_cluster: whether we are in cluster
:param cluster_context: context of the cluster
"""
super().__init__()
self._client = kube_client or get_kube_client(in_cluster=in_cluster, cluster_context=cluster_context)
self._watch = watch.Watch()
def run_pod_async(self, pod: V1Pod, **kwargs) -> V1Pod:
"""Runs POD asynchronously"""
sanitized_pod = self._client.api_client.sanitize_for_serialization(pod)
json_pod = json.dumps(sanitized_pod, indent=2)
self.log.debug('Pod Creation Request: \n%s', json_pod)
try:
resp = self._client.create_namespaced_pod(
body=sanitized_pod, namespace=pod.metadata.namespace, **kwargs
)
self.log.debug('Pod Creation Response: %s', resp)
except Exception as e:
self.log.exception(
'Exception when attempting to create Namespaced Pod: %s', str(json_pod).replace("\n", " ")
)
raise e
return resp
def delete_pod(self, pod: V1Pod) -> None:
"""Deletes POD"""
try:
self._client.delete_namespaced_pod(
pod.metadata.name, pod.metadata.namespace, body=client.V1DeleteOptions()
)
except ApiException as e:
# If the pod is already deleted
if e.status != 404:
raise
@tenacity.retry(
stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_random_exponential(),
reraise=True,
retry=tenacity.retry_if_exception(should_retry_start_pod),
)
def create_pod(self, pod: V1Pod) -> V1Pod:
"""Launches the pod asynchronously."""
return self.run_pod_async(pod)
def await_pod_start(self, pod: V1Pod, startup_timeout: int = 120) -> None:
"""
Waits for the pod to reach phase other than ``Pending``
:param pod:
:param startup_timeout: Timeout (in seconds) for startup of the pod
(if pod is pending for too long, fails task)
:return:
"""
curr_time = datetime.now()
while True:
remote_pod = self.read_pod(pod)
if remote_pod.status.phase != PodPhase.PENDING:
break
self.log.warning("Pod not yet started: %s", pod.metadata.name)
delta = datetime.now() - curr_time
if delta.total_seconds() >= startup_timeout:
msg = (
f"Pod took longer than {startup_timeout} seconds to start. "
"Check the pod events in kubernetes to determine why."
)
raise PodLaunchFailedException(msg)
time.sleep(1)
def follow_container_logs(self, pod: V1Pod, container_name: str) -> PodLoggingStatus:
warnings.warn(
"Method `follow_container_logs` is deprecated. Use `fetch_container_logs` instead"
"with option `follow=True`.",
DeprecationWarning,
)
return self.fetch_container_logs(pod=pod, container_name=container_name, follow=True)
def fetch_container_logs(
self, pod: V1Pod, container_name: str, *, follow=False, since_time: Optional[DateTime] = None
) -> PodLoggingStatus:
"""
Follows the logs of container and streams to airflow logging.
Returns when container exits.
"""
def consume_logs(*, since_time: Optional[DateTime] = None, follow: bool = True) -> Optional[DateTime]:
"""
Tries to follow container logs until container completes.
For a long-running container, sometimes the log read may be interrupted
Such errors of this kind are suppressed.
Returns the last timestamp observed in logs.
"""
timestamp = None
try:
logs = self.read_pod_logs(
pod=pod,
container_name=container_name,
timestamps=True,
since_seconds=(
math.ceil((pendulum.now() - since_time).total_seconds()) if since_time else None
),
follow=follow,
)
for raw_line in logs:
line = raw_line.decode('utf-8', errors="backslashreplace")
timestamp, message = self.parse_log_line(line)
self.log.info(message)
except BaseHTTPError as e:
self.log.warning(
"Reading of logs interrupted with error %r; will retry. "
"Set log level to DEBUG for traceback.",
e,
)
self.log.debug(
"Traceback for interrupted logs read for pod %r",
pod.metadata.name,
exc_info=True,
)
return timestamp or since_time
# note: `read_pod_logs` follows the logs, so we shouldn't necessarily *need* to
# loop as we do here. But in a long-running process we might temporarily lose connectivity.
# So the looping logic is there to let us resume following the logs.
last_log_time = since_time
while True:
last_log_time = consume_logs(since_time=last_log_time, follow=follow)
if not self.container_is_running(pod, container_name=container_name):
return PodLoggingStatus(running=False, last_log_time=last_log_time)
if not follow:
return PodLoggingStatus(running=True, last_log_time=last_log_time)
else:
self.log.warning(
'Pod %s log read interrupted but container %s still running',
pod.metadata.name,
container_name,
)
time.sleep(1)
def await_container_completion(self, pod: V1Pod, container_name: str) -> None:
while not self.container_is_running(pod=pod, container_name=container_name):
time.sleep(1)
def await_pod_completion(self, pod: V1Pod) -> V1Pod:
"""
Monitors a pod and returns the final state
:param pod: pod spec that will be monitored
:return: Tuple[State, Optional[str]]
"""
while True:
remote_pod = self.read_pod(pod)
if remote_pod.status.phase in PodPhase.terminal_states:
break
self.log.info('Pod %s has phase %s', pod.metadata.name, remote_pod.status.phase)
time.sleep(2)
return remote_pod
def parse_log_line(self, line: str) -> Tuple[Optional[DateTime], str]:
"""
Parse K8s log line and returns the final state
:param line: k8s log line
:return: timestamp and log message
:rtype: Tuple[str, str]
"""
split_at = line.find(' ')
if split_at == -1:
self.log.error(
"Error parsing timestamp (no timestamp in message %r). "
"Will continue execution but won't update timestamp",
line,
)
return None, line
timestamp = line[:split_at]
message = line[split_at + 1 :].rstrip()
try:
last_log_time = cast(DateTime, pendulum.parse(timestamp))
except ParserError:
self.log.error("Error parsing timestamp. Will continue execution but won't update timestamp")
return None, line
return last_log_time, message
def container_is_running(self, pod: V1Pod, container_name: str) -> bool:
"""Reads pod and checks if container is running"""
remote_pod = self.read_pod(pod)
return container_is_running(pod=remote_pod, container_name=container_name)
@tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
def read_pod_logs(
self,
pod: V1Pod,
container_name: str,
tail_lines: Optional[int] = None,
timestamps: bool = False,
since_seconds: Optional[int] = None,
follow=True,
) -> Iterable[bytes]:
"""Reads log from the POD"""
additional_kwargs = {}
if since_seconds:
additional_kwargs['since_seconds'] = since_seconds
if tail_lines:
additional_kwargs['tail_lines'] = tail_lines
try:
return self._client.read_namespaced_pod_log(
name=pod.metadata.name,
namespace=pod.metadata.namespace,
container=container_name,
follow=follow,
timestamps=timestamps,
_preload_content=False,
**additional_kwargs,
)
except BaseHTTPError:
self.log.exception('There was an error reading the kubernetes API.')
raise
@tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
def read_pod_events(self, pod: V1Pod) -> "CoreV1EventList":
"""Reads events from the POD"""
try:
return self._client.list_namespaced_event(
namespace=pod.metadata.namespace, field_selector=f"involvedObject.name={pod.metadata.name}"
)
except BaseHTTPError as e:
raise AirflowException(f'There was an error reading the kubernetes API: {e}')
@tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
def read_pod(self, pod: V1Pod) -> V1Pod:
"""Read POD information"""
try:
return self._client.read_namespaced_pod(pod.metadata.name, pod.metadata.namespace)
except BaseHTTPError as e:
raise AirflowException(f'There was an error reading the kubernetes API: {e}')
def extract_xcom(self, pod: V1Pod) -> str:
"""Retrieves XCom value and kills xcom sidecar container"""
with closing(
kubernetes_stream(
self._client.connect_get_namespaced_pod_exec,
pod.metadata.name,
pod.metadata.namespace,
container=PodDefaults.SIDECAR_CONTAINER_NAME,
command=['/bin/sh'],
stdin=True,
stdout=True,
stderr=True,
tty=False,
_preload_content=False,
)
) as resp:
result = self._exec_pod_command(resp, f'cat {PodDefaults.XCOM_MOUNT_PATH}/return.json')
self._exec_pod_command(resp, 'kill -s SIGINT 1')
if result is None:
raise AirflowException(f'Failed to extract xcom from pod: {pod.metadata.name}')
return result
def _exec_pod_command(self, resp, command: str) -> Optional[str]:
res = None
if resp.is_open():
self.log.info('Running command... %s\n', command)
resp.write_stdin(command + '\n')
while resp.is_open():
resp.update(timeout=1)
while resp.peek_stdout():
res = res + resp.read_stdout() if res else resp.read_stdout()
error_res = None
while resp.peek_stderr():
error_res = error_res + resp.read_stderr() if error_res else resp.read_stderr()
if error_res:
self.log.info("stderr from command: %s", error_res)
break
if res:
return res
return res
def send(self, target, ctxt, message, wait_for_reply=None, timeout=None,
retry=None):
with timeutils.StopWatch(duration=timeout) as stopwatch:
if retry is None:
retry = self._pika_engine.default_rpc_retry_attempts
exchange = self._pika_engine.get_rpc_exchange_name(
target.exchange
)
def on_exception(ex):
if isinstance(ex, pika_drv_exc.ExchangeNotFoundException):
# it is desired to create exchange because if we sent to
# exchange which is not exists, we get ChannelClosed
# exception and need to reconnect
try:
self._declare_rpc_exchange(exchange, stopwatch)
except pika_drv_exc.ConnectionException as e:
LOG.warning("Problem during declaring exchange. %s", e)
return True
elif isinstance(ex, (pika_drv_exc.ConnectionException,
exceptions.MessageDeliveryFailure)):
LOG.warning("Problem during message sending. %s", ex)
return True
else:
return False
if retry:
retrier = tenacity.retry(
stop=(tenacity.stop_never if retry == -1 else
tenacity.stop_after_attempt(retry)),
retry=tenacity.retry_if_exception(on_exception),
wait=tenacity.wait_fixed(self._pika_engine.rpc_retry_delay)
)
else:
retrier = None
if target.fanout:
return self.cast_all_workers(
exchange, target.topic, ctxt, message, stopwatch, retrier
)
routing_key = self._pika_engine.get_rpc_queue_name(
target.topic, target.server, retrier is None
)
msg = pika_drv_msg.RpcPikaOutgoingMessage(self._pika_engine,
message, ctxt)
try:
reply = msg.send(
exchange=exchange,
routing_key=routing_key,
reply_listener=(
self._reply_listener if wait_for_reply else None
),
stopwatch=stopwatch,
retrier=retrier
)
except pika_drv_exc.ExchangeNotFoundException as ex:
try:
self._declare_rpc_exchange(exchange, stopwatch)
except pika_drv_exc.ConnectionException as e:
LOG.warning("Problem during declaring exchange. %s", e)
raise ex
if reply is not None:
if reply.failure is not None:
raise reply.failure
return reply.result
return connection
def is_disconnection_exception(e):
if not libvirt:
return False
return (isinstance(e, libvirt.libvirtError)
and e.get_error_code() in (libvirt.VIR_ERR_SYSTEM_ERROR,
libvirt.VIR_ERR_INTERNAL_ERROR)
and e.get_error_domain() in (libvirt.VIR_FROM_REMOTE,
libvirt.VIR_FROM_RPC))
retry_on_disconnect = tenacity.retry(
retry=tenacity.retry_if_exception(is_disconnection_exception),
stop=tenacity.stop_after_attempt(2))
def raise_nodata_if_unsupported(method):
def inner(in_self, instance, *args, **kwargs):
try:
return method(in_self, instance, *args, **kwargs)
except libvirt.libvirtError as e:
# NOTE(sileht): At this point libvirt connection error
# have been reraise as tenacity.RetryError()
msg = _('Failed to inspect instance %(instance_uuid)s stats, '
'can not get info from libvirt: %(error)s') % {
"instance_uuid": instance.id,
"error": e}
raise virt_inspector.NoDataException(msg)
return inner
def retry_on_stale_data_error(func):
wrapper = tenacity.retry(
stop=tenacity.stop_after_attempt(3),
retry=tenacity.retry_if_exception_type(exc.StaleDataError),
reraise=True)
return wrapper(func)
# Note, we yield the options via list_opts to enable generation of the
# sample heat.conf, but we don't register these options directly via
# cfg.CONF.register*, it's done via ks_loading.register_auth_conf_options
# Note, only auth_type = v3password is expected to work, example config:
# [trustee]
# auth_type = password
# auth_url = http://192.168.1.2:35357
# username = heat
# password = password
# user_domain_id = default
PASSWORD_PLUGIN = 'password' # nosec Bandit B105
TRUSTEE_CONF_GROUP = 'trustee'
ks_loading.register_auth_conf_options(cfg.CONF, TRUSTEE_CONF_GROUP)
retry_on_connection_timeout = tenacity.retry(
stop=tenacity.stop_after_attempt(cfg.CONF.client_retry_limit + 1),
wait=tenacity.wait_random(max=2),
retry=tenacity.retry_if_exception_type(
(ksa_exceptions.ConnectFailure, ksa_exceptions.DiscoveryFailure)),
reraise=True)
def list_opts():
trustee_opts = ks_loading.get_auth_common_conf_options()
trustee_opts.extend(
ks_loading.get_auth_plugin_conf_options(PASSWORD_PLUGIN))
yield TRUSTEE_CONF_GROUP, trustee_opts
def _moved_attr(new_name):
def getter(self):
async def generate_game(self):
with tempfile.TemporaryDirectory() as tmp:
settings_file_path = os.path.join(tmp, "settings.json")
self.hash = ''.join(
random.choices(string.ascii_letters + string.digits, k=12))
self.settings['outputpath'] = tmp
self.settings['outputname'] = self.hash
self.settings['create_rom'] = True
self.settings['create_spoiler'] = True
self.settings['calc_playthrough'] = False
self.settings['rom'] = os.environ.get('ALTTP_ROM')
self.settings['enemizercli'] = os.path.join(
os.environ.get('ENEMIZER_HOME'), 'EnemizerCLI.Core')
# set some defaults we do NOT want to change ever
self.settings['count'] = 1
self.settings['multi'] = 1
self.settings['names'] = ""
self.settings['race'] = not self.spoilers
with open(settings_file_path, "w") as f:
json.dump(self.settings, f)
attempts = 0
try:
async for attempt in AsyncRetrying(
stop=stop_after_attempt(10),
retry=retry_if_exception_type(Exception)):
with attempt:
attempts += 1
proc = await asyncio.create_subprocess_exec(
'python3',
'DungeonRandomizer.py',
'--settingsfile',
settings_file_path,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=os.environ.get('DOOR_RANDO_HOME'))
stdout, stderr = await proc.communicate()
logging.info(stdout.decode())
if proc.returncode > 0:
raise Exception(
f'Exception while generating game: {stderr.decode()}'
)
except RetryError as e:
raise e.last_attempt._exception from e
self.attempts = attempts
self.patch_name = "DR_" + self.settings['outputname'] + ".bps"
self.rom_name = "DR_" + self.settings['outputname'] + ".sfc"
self.spoiler_name = "DR_" + self.settings[
'outputname'] + "_Spoiler.txt"
rom_path = os.path.join(tmp, self.rom_name)
patch_path = os.path.join(tmp, self.patch_name)
spoiler_path = os.path.join(tmp, self.spoiler_name)
proc = await asyncio.create_subprocess_exec(
os.path.join('utils', 'flips'),
'--create',
'--bps-delta',
os.environ.get("ALTTP_ROM"),
rom_path,
patch_path,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
stdout, stderr = await proc.communicate()
logging.info(stdout.decode())
if proc.returncode > 0:
raise Exception(
f'Exception while while creating patch: {stderr.decode()}')
async with aiofiles.open(patch_path, "rb") as f:
patchfile = await f.read()
session = aioboto3.Session()
async with session.client('s3') as s3:
await s3.put_object(
Bucket=os.environ.get('SAHASRAHBOT_BUCKET'),
Key=f"patch/{self.patch_name}",
Body=patchfile,
ACL='public-read')
async with aiofiles.open(spoiler_path, "rb") as f:
self.spoilerfile = await f.read()
async with session.client('s3') as s3:
await s3.put_object(
Bucket=os.environ.get('SAHASRAHBOT_BUCKET'),
Key=f"spoiler/{self.spoiler_name}",
Body=gzip.compress(self.spoilerfile),
ACL='public-read' if self.spoilers else 'private',
ContentEncoding='gzip',
ContentDisposition='attachment')
class GitRepo(object):
"""
Class for work with repositories
"""
def __init__(self, root_repo_dir, repo_name, branch, url, commit_id=None):
"""
:param root_repo_dir: Directory where repositories will clone
:param repo_name: Name of repository
:param branch: Branch of repository
:param commit_id: Commit ID
"""
self.name = repo_name
self.branch = branch
self.url = url
self.commit_id = commit_id
self.local_repo_dir = root_repo_dir / repo_name
self.repo = None
self.log = logging.getLogger()
def prepare_repo(self):
"""
Preparing repository for build
Include cloning and updating repo to remote state
:return: None
"""
self.log.info('-' * 50)
self.log.info("Getting repo " + self.name)
self.clone()
self.repo = git.Repo(str(self.local_repo_dir))
self.hard_reset()
self.clean()
self.checkout("master", silent=True)
self.fetch()
self.hard_reset('FETCH_HEAD')
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def clone(self):
"""
Clone repo
:return: None
"""
# checking correctness of git repository
# if dir is not repository, it will be removed
if self.local_repo_dir.exists():
try:
git.Repo(str(self.local_repo_dir))
except git.InvalidGitRepositoryError:
self.log.info('Remove broken repo %s', self.local_repo_dir)
remove_directory(self.local_repo_dir)
if not self.local_repo_dir.exists():
self.log.info("Clone repo " + self.name)
git.Git().clone(self.url, str(self.local_repo_dir))
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def fetch(self):
"""
Fetch repo
:return: None
"""
self.log.info("Fetch repo %s to %s", self.name, self.branch)
self.repo.remotes.origin.fetch(self.branch)
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def hard_reset(self, reset_to=None):
"""
Hard reset repo
:param reset_to: Commit ID or branch. If None - hard reset to HEAD
:return: None
"""
self.log.info("Hard reset repo " + self.name)
if reset_to:
self.repo.git.reset('--hard', reset_to)
else:
self.repo.git.reset('--hard')
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def checkout(self, branch=None, silent=False):
"""
Checkout to certain state
:param branch: Branch of repo. If None - checkout to commit ID from class variable commit_id
:param silent: Flag for getting time of commit
(set to True only if commit_id does not exist)
:type silent: Boolean
:return: None
"""
checkout_to = branch if branch else self.commit_id
self.log.info("Checkout repo %s to %s", self.name, checkout_to)
self.repo.git.checkout(checkout_to, force=True)
if not silent:
# error raises after checkout to master if we try
# to get time of triggered commit_id before fetching repo
# (commit does not exist in local repository yet)
committed_date = self.repo.commit(self.commit_id).committed_date
self.log.info("Committed date: %s",
datetime.fromtimestamp(committed_date))
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def clean(self):
"""
Clean repo
:return: None
"""
self.log.info("Clean repo " + self.name)
self.repo.git.clean('-xdf')
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def pull(self):
"""
Pull repo
:return: None
"""
self.log.info("Pull repo " + self.name)
self.repo.git.pull()
def revert_commit_by_time(self, commit_time):
"""
Sets commit by time.
If commit date <= certain time,
commit sets to class variable commit_id.
:param commit_time: datetime
:return: None
"""
self.commit_id = str(
next(
self.repo.iter_commits(rev='master',
until=commit_time,
max_count=1)))
def get_time(self, commit_id=None):
"""
Get datetime of commit
:param commit_id: Commit ID
:return: datetime
"""
commit = commit_id if commit_id else self.commit_id
return self.repo.commit(commit).committed_date
class GoogleAPIResource(Resource):
# Names of the get method of the root resource
get_method = "get"
required_resource_data = ['name']
# jmespath expression for getting labels
resource_labels_path = "resource.labels"
# Other properties of a resource we might need to perform evaluations, such as iam policy
resource_components = {}
# If a resource is not in a ready state, we can't update it. If we retrieve
# it, and the state changes, updates will be rejected because the ETAG will
# have changed. If a resource defines readiness criteria, the get() call
# will wait until the resource is in a ready state to return
#
# Key/Value to check to see if a resource is ready
readiness_key = None
readiness_value = None
readiness_terminal_values = []
def __init__(self, client_kwargs=None, **resource_data):
if client_kwargs is None:
client_kwargs = {}
# Set some defaults
self._service = None
self._resource_metadata = None
self._full_resource_name = None
# Load and validate additional resource data
self._resource_data = resource_data
self._validate_resource_data()
# Store the client kwargs to pass to any new clients
self._client_kwargs = client_kwargs
self._ancestry = None
def _validate_resource_data(self):
''' Verify we have all the required data for this resource '''
if not all(arg in self._resource_data
for arg in self.required_resource_data):
raise ResourceException(
'Missing data required for resource creation. Expected data: {}; Got: {}'
.format(','.join(self.required_resource_data),
','.join(self._resource_data.keys())))
@staticmethod
def _extract_cai_name_data(name):
''' Attempt to get identifiable information out of a Cloud Asset Inventory-formatted resource_name '''
# Most resources need only a subset of these fields to query the google apis
fields = {
'project_id': r'/projects/([^\/]+)/',
'location': r'/(?:locations|regions|zones)/([^\/]+)/',
'name': r'([^\/]+)$',
# Less-common resource data
# AppEngine
'app': r'/apps/([^\/]+)/',
'service': r'/services/([^\/]+)/',
'version': r'/versions/([^\/]+)/',
# NodePools
'cluster': r'/clusters/([^\/]+)/',
# ServiceAccounts
'service_account': r'serviceAccounts/([^\/]+)/',
}
resource_data = {}
# Extract available resource data from resource name
for field_name in fields:
m = re.search(fields[field_name], name)
if m:
resource_data[field_name] = m.group(1)
return resource_data
@classmethod
def subclass_by_type(cls, resource_type):
mapper = {
res_cls.resource_type: res_cls
for res_cls in cls.__subclasses__()
}
try:
return mapper[resource_type]
except KeyError:
raise ResourceException(
'Unrecognized resource type: {}'.format(resource_type))
@classmethod
def from_resource_data(cls,
*,
resource_type,
client_kwargs=None,
**resource_data):
if client_kwargs is None:
client_kwargs = {}
res_cls = cls.subclass_by_type(resource_type)
return res_cls(client_kwargs=client_kwargs, **resource_data)
@staticmethod
def from_cai_data(resource_name,
resource_type,
project_id=None,
client_kwargs=None):
''' Attempt to return the appropriate resource using Cloud Asset Inventory-formatted resource info '''
if client_kwargs is None:
client_kwargs = {}
res_cls = GoogleAPIResource.subclass_by_type(resource_type)
resource_data = GoogleAPIResource._extract_cai_name_data(resource_name)
# if the project_id was passed, and its wasnt found in the resource name, add it
if project_id and 'project_id' not in resource_data:
resource_data['project_id'] = project_id
return res_cls(client_kwargs=client_kwargs, **resource_data)
def to_dict(self):
details = self._resource_data.copy()
details.update({
'resource_type': self.resource_type,
})
try:
details['full_resource_name'] = self.full_resource_name()
except Exception:
details['full_resource_name'] = None
return details
def type(self):
return self.resource_type
def full_resource_name(self):
if self._full_resource_name is None:
self.gen_full_resource_name()
return self._full_resource_name
# Google's documentation describes what it calls a 'full resource name' for
# resources. None of the API's seem to implement it (except Cloud Asset
# Inventory). This attempts to generate it from the discovery-based api
# client's generated http request url.
#
# If we inject it into the resource, we can use it in policy evaluation to
# simplify the structure of our policies
def gen_full_resource_name(self):
method = getattr(self.service, self.get_method)
uri = method(**self._get_request_args()).uri
uri_parsed = urlparse(uri)
domain = uri_parsed.netloc
path_segments = uri_parsed.path[1:].split('/')
# CAI uses cloudsql.googleapis.com in their full_resource_name, so we need to detect all
# bad incarnations and replace them
bad_sql_names = ['sql', 'sqladmin']
# First we need the name of the api
if domain.startswith("www."):
# we need to get the api name from the path
api_name = path_segments.pop(0)
api_name = 'cloudsql' if api_name in bad_sql_names else api_name
else:
# the api name is the first segment of the domain
api_name = domain.split('.')[0]
# the sql api is now returning sqladmin.googleapis.com/sql/<ver>
# and the CAI docs state the FRN for sql instances should start
# with //cloudsql.googleapis.com/ so lets replace all odd sql ones
# and rely on the code below to catch duplicates
path_segments[0] = 'cloudsql' if path_segments[
0] in bad_sql_names else path_segments[0]
api_name = 'cloudsql' if api_name in bad_sql_names else api_name
# occasionally the compute api baseUrl is returned as
# compute.googleapis.com/compute, in which case we need to remove
# the duplicated api reference
# also addresses the sql issue mentioned above
if api_name == path_segments[0]:
path_segments.pop(0)
# Remove the version from the path
path_segments.pop(0)
# Remove method from the last path segment
if ":" in path_segments[-1]:
path_segments[-1] = path_segments[-1].split(":")[0]
# Annoying resource-specific fixes
# The url for buckets is `/b/` instead of `/buckets/`. Initially this fixed that
# Unfortunately, CAI omits the collection name, rather than follow Google's API design doc
# So we just remove the collection name
#
# https://cloud.google.com/apis/design/resource_names
# See: https://issuetracker.google.com/issues/131586763
#
if api_name == 'storage' and path_segments[0] == 'b':
path_segments.pop(0)
# Replace with this if they fix CAI:
# path_segments[0] = "buckets"
if api_name == 'bigtableadmin':
api_name = 'bigtable'
resource_path = "/".join(path_segments)
self._full_resource_name = "//{}.googleapis.com/{}".format(
api_name, resource_path)
def _get_component(self, component):
method_name = self.resource_components[component]
# Many components take the same request signature, but allow for custom request
# args if needed. Fall back to default args if the expected function doesn't exist
if hasattr(self, f'_get_{component}_request_args'):
req_arg_method = getattr(self, f'_get_{component}_request_args')
else:
req_arg_method = getattr(self, '_get_request_args')
method = getattr(self.service, method_name)
component_metadata = method(**req_arg_method()).execute()
return component_metadata
def get(self, refresh=True):
if not refresh and self._resource_metadata:
return self._resource_metadata
method = getattr(self.service, self.get_method)
# If the resource has readiness criteria, wait for it
if self.readiness_key and self.readiness_value:
waiter = Waiter(method, **self._get_request_args())
asset = waiter.wait(self.readiness_key,
self.readiness_value,
terminal_values=self.readiness_terminal_values,
interval=10,
retries=90)
else:
asset = method(**self._get_request_args()).execute()
resp = {
'type': self.type(),
'name': self.full_resource_name(),
}
resp['resource'] = asset
for c in self.resource_components:
resp[c] = self._get_component(c)
self._resource_metadata = resp
return self._resource_metadata
# Determine what remediation steps to take, allow for future remediation specifications
def remediate(self, remediation):
# Check for an update spec version, default to version 1
remediation_spec = remediation.get('_remediation_spec', "")
if remediation_spec in ['v2beta1', 'v2']:
required_keys = ['method', 'params']
for step in remediation.get('steps', []):
if not all(k in step for k in required_keys):
raise InvalidRemediationSpecStep()
method_name = step.get('method')
params = step.get('params')
self._call_method(method_name, params)
else:
raise UnsupportedRemediationSpec(
"The specified remediation spec is not supported")
@tenacity.retry(retry=tenacity.retry_if_exception(is_retryable_exception),
wait=tenacity.wait_random_exponential(multiplier=5,
max=20),
stop=tenacity.stop_after_attempt(15))
def _call_method(self, method_name, params):
''' Call the requested method on the resource '''
method = getattr(self.service, method_name)
return method(**params).execute()
@property
def ancestry(self):
if self._ancestry:
return self._ancestry
# attempt to fill in the resource's ancestry
# if the target project has the cloudresourcemanager api disabled, this will fail
# if the resource_data doesn't include the project_id (ex: with storage buckets) this will also fail
try:
resource_manager_projects = build_subresource(
'cloudresourcemanager.projects', 'v1', **self._client_kwargs)
resp = resource_manager_projects.getAncestry(
projectId=self.project_id).execute()
# Reformat getAncestry response to be a list of resource names
self._ancestry = [
f"//cloudresourcemanager.googleapis.com/{ancestor['resourceId']['type']}s/{ancestor['resourceId']['id']}"
for ancestor in resp.get('ancestor')
]
except Exception:
# This call is best-effort. Any failures should be caught
pass
return self._ancestry
@property
def organization(self):
ancestry = self.ancestry
if not ancestry:
return None
return next((ancestor for ancestor in ancestry if ancestor.startswith(
'//cloudresourcemanager.googleapis.com/organizations/')), None)
@property
def client_kwargs(self):
return self._client_kwargs
@client_kwargs.setter
def client_kwargs(self, client_kwargs):
# Invalidate service/parent because client_kwargs changed
self._service = None
self._client_kwargs = client_kwargs
@property
def service(self):
if self._service is None:
full_resource_path = "{}.{}".format(self.service_name,
self.resource_path)
self._service = build_subresource(full_resource_path, self.version,
**self._client_kwargs)
return self._service
@property
def labels(self):
labels = jmespath.search(self.resource_labels_path,
self.get(refresh=False)) or {}
if not isinstance(labels, dict):
raise TypeError("Unexpected label format")
return labels
@property
def project_id(self):
return self._resource_data.get('project_id')
from typing import Any, List
from googleapiclient.errors import HttpError
from tenacity import TryAgain, retry, retry_if_exception_type, stop_after_attempt, wait_exponential
from dispatch.decorators import apply, counter, timer
from dispatch.plugins.bases import ParticipantGroupPlugin
from dispatch.plugins.dispatch_google import groups as google_group_plugin
from dispatch.plugins.dispatch_google.common import get_service
from dispatch.plugins.dispatch_google.config import GOOGLE_USER_OVERRIDE, GOOGLE_DOMAIN
log = logging.getLogger(__name__)
@retry(
stop=stop_after_attempt(3),
retry=retry_if_exception_type(TryAgain),
wait=wait_exponential(multiplier=1, min=2, max=5),
)
def make_call(client: Any,
func: Any,
delay: int = None,
propagate_errors: bool = False,
**kwargs):
"""Make an google client api call."""
try:
data = getattr(client, func)(**kwargs).execute()
if delay:
time.sleep(delay)
class Buda(ExchangeClient):
def __init__(self, public_key=None, secret_key=None):
self.name = 'Buda'
# if not read_only and (public_key is None or secret_key is None):
# public_key = input('Enter your public key: ')
# secret_key = input('Enter your private key: ')
self.base_uri = 'https://www.buda.com/api'
self.api_type = 'REST'
if public_key and secret_key:
self.auth = BudaHMACAuth(public_key, secret_key)
self.timeout = 5
super().__init__(
read_only=True if not (public_key and secret_key) else False)
@retry(stop=stop_after_attempt(number_of_attempts), wait=wait_fixed(0.2))
def get_book(self, pair):
# try:
response = requests.get(
f"{self.base_uri}/v2/markets/{pair.ticker}/order_book",
timeout=self.timeout)
book = response.json()['order_book']
# except JSONDecodeError:
# print(response)
asks = [{'amount': x[1], 'price': x[0]} for x in book['asks']]
bids = [{'amount': x[1], 'price': x[0]} for x in book['bids']]
return {ASK: asks, BID: bids}
@retry(stop=stop_after_attempt(number_of_attempts))
def get_active_orders(self, pair):
result = {ASK: [], BID: []}
try:
response = requests.get(
f"{self.base_uri}/v2/markets/{pair.ticker}/orders",
auth=self.auth,
timeout=self.timeout).json()
orders = response['orders']
except KeyError:
print(response)
for order in orders:
if order['state'] == 'canceled':
continue
side = ASK if order['type'].lower() == 'ask' else BID
result[side].append(
Order(order['limit'][0],
side,
order['amount'][0],
order_id=order['id'],
pair=pair))
return result
@retry(retry=retry_if_exception(is_not_local_exception),
stop=stop_after_attempt(number_of_attempts))
def cancel_order(self, order):
requests.put(f"{self.base_uri}/v2/orders/{order.order_id}",
auth=self.auth,
json={
'state': 'canceling'
},
timeout=self.timeout).json()
@retry(retry=retry_if_exception(is_not_local_exception),
stop=stop_after_attempt(number_of_attempts))
def get_balance(self, currency):
try:
response = requests.get(f"{self.base_uri}/v2/balances",
auth=self.auth,
timeout=self.timeout)
balances = response.json()['balances']
except KeyError:
print(response.content)
try:
search_result = next(
item for item in balances
if item["id"].lower() == currency.symbol.lower())
except StopIteration:
raise currency_doesnt_exist
return [
search_result['available_amount'][0],
search_result['frozen_amount'][0]
]
@retry(retry=retry_if_exception(is_not_local_exception),
stop=stop_after_attempt(number_of_attempts))
def create_order(self, pair, amount, side, limit_price=None):
body = {}
body['price_type'] = 'LIMIT' if limit_price else 'MARKET'
body['amount'] = truncate(amount, 5)
if limit_price:
body['limit'] = limit_price
body['type'] = 'Ask' if side is ASK else 'Bid'
response = requests.post(
f"{self.base_uri}/v2/markets/{pair.ticker}/orders",
json=body,
auth=self.auth,
timeout=self.timeout).json()
def subscribe(self, pair):
self._register_pair_and_currencies(pair)
if self.api_type == 'REST':
self.__start_threads__(pair)
def unsubscribe(self, pair):
pass
@retry(retry=retry_if_exception(is_not_local_exception),
stop=stop_after_attempt(number_of_attempts))
def get_list_of_currencies_and_pairs(self, auto_register=False):
response = requests.get(f"{self.base_uri}/v2/markets",
timeout=self.timeout).json()
pairs = response['markets']
list_of_currencies = set([])
list_of_pairs = []
for pair in pairs:
try:
base_curr = quote_curr = None
for curr in list_of_currencies:
if pair['base_currency'] == curr.symbol:
base_curr = curr
if pair['quote_currency'] == curr.symbol:
quote_curr = curr
if not base_curr:
base_curr = base.exchange.Currency(
name=pair['base_currency'],
symbol=pair['base_currency'],
exchange_client=self)
if not quote_curr:
quote_curr = base.exchange.Currency(
name=pair['quote_currency'],
symbol=pair['quote_currency'],
exchange_client=self)
except currency_doesnt_exist:
continue
pair = base.exchange.Pair(
ticker=pair['id'],
quote=quote_curr,
base=base_curr,
minimum_step=pair['minimum_order_amount'][0],
exchange_client=self)
list_of_pairs.append(pair)
list_of_currencies.add(quote_curr)
list_of_currencies.add(base_curr)
if auto_register is True:
for pair in list_of_pairs:
self._register_pair_and_currencies(pair)
return list(list_of_currencies), list_of_pairs
def get_history(self, pair):
response = requests.get(f"{self.base_uri}/v1/trade/{pair.ticker}/",
headers=self.headers,
timeout=self.timeout).json()
print(response)
class ControllerWorker(base_taskflow.BaseTaskFlowEngine):
def __init__(self):
self._amphora_flows = amphora_flows.AmphoraFlows()
self._health_monitor_flows = health_monitor_flows.HealthMonitorFlows()
self._lb_flows = load_balancer_flows.LoadBalancerFlows()
self._listener_flows = listener_flows.ListenerFlows()
self._member_flows = member_flows.MemberFlows()
self._pool_flows = pool_flows.PoolFlows()
self._l7policy_flows = l7policy_flows.L7PolicyFlows()
self._l7rule_flows = l7rule_flows.L7RuleFlows()
self._amphora_repo = repo.AmphoraRepository()
self._amphora_health_repo = repo.AmphoraHealthRepository()
self._health_mon_repo = repo.HealthMonitorRepository()
self._lb_repo = repo.LoadBalancerRepository()
self._listener_repo = repo.ListenerRepository()
self._member_repo = repo.MemberRepository()
self._pool_repo = repo.PoolRepository()
self._l7policy_repo = repo.L7PolicyRepository()
self._l7rule_repo = repo.L7RuleRepository()
self._flavor_repo = repo.FlavorRepository()
super(ControllerWorker, self).__init__()
@tenacity.retry(
retry=(
tenacity.retry_if_result(_is_provisioning_status_pending_update) |
tenacity.retry_if_exception_type()),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
def _get_db_obj_until_pending_update(self, repo, id):
return repo.get(db_apis.get_session(), id=id)
def create_amphora(self):
"""Creates an Amphora.
This is used to create spare amphora.
:returns: amphora_id
"""
try:
create_amp_tf = self._taskflow_load(
self._amphora_flows.get_create_amphora_flow(),
store={constants.BUILD_TYPE_PRIORITY:
constants.LB_CREATE_SPARES_POOL_PRIORITY,
constants.FLAVOR: None}
)
with tf_logging.DynamicLoggingListener(create_amp_tf, log=LOG):
create_amp_tf.run()
return create_amp_tf.storage.fetch('amphora')
except Exception as e:
LOG.error('Failed to create an amphora due to: {}'.format(str(e)))
def delete_amphora(self, amphora_id):
"""Deletes an existing Amphora.
:param amphora_id: ID of the amphora to delete
:returns: None
:raises AmphoraNotFound: The referenced Amphora was not found
"""
amphora = self._amphora_repo.get(db_apis.get_session(),
id=amphora_id)
delete_amp_tf = self._taskflow_load(self._amphora_flows.
get_delete_amphora_flow(),
store={constants.AMPHORA: amphora})
with tf_logging.DynamicLoggingListener(delete_amp_tf,
log=LOG):
delete_amp_tf.run()
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
def create_health_monitor(self, health_monitor_id):
"""Creates a health monitor.
:param pool_id: ID of the pool to create a health monitor on
:returns: None
:raises NoResultFound: Unable to find the object
"""
health_mon = self._health_mon_repo.get(db_apis.get_session(),
id=health_monitor_id)
if not health_mon:
LOG.warning('Failed to fetch %s %s from DB. Retrying for up to '
'60 seconds.', 'health_monitor', health_monitor_id)
raise db_exceptions.NoResultFound
pool = health_mon.pool
listeners = pool.listeners
pool.health_monitor = health_mon
load_balancer = pool.load_balancer
create_hm_tf = self._taskflow_load(
self._health_monitor_flows.get_create_health_monitor_flow(),
store={constants.HEALTH_MON: health_mon,
constants.POOL: pool,
constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer})
with tf_logging.DynamicLoggingListener(create_hm_tf,
log=LOG):
create_hm_tf.run()
def delete_health_monitor(self, health_monitor_id):
"""Deletes a health monitor.
:param pool_id: ID of the pool to delete its health monitor
:returns: None
:raises HMNotFound: The referenced health monitor was not found
"""
health_mon = self._health_mon_repo.get(db_apis.get_session(),
id=health_monitor_id)
pool = health_mon.pool
listeners = pool.listeners
load_balancer = pool.load_balancer
delete_hm_tf = self._taskflow_load(
self._health_monitor_flows.get_delete_health_monitor_flow(),
store={constants.HEALTH_MON: health_mon,
constants.POOL: pool,
constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer})
with tf_logging.DynamicLoggingListener(delete_hm_tf,
log=LOG):
delete_hm_tf.run()
def update_health_monitor(self, health_monitor_id, health_monitor_updates):
"""Updates a health monitor.
:param pool_id: ID of the pool to have it's health monitor updated
:param health_monitor_updates: Dict containing updated health monitor
:returns: None
:raises HMNotFound: The referenced health monitor was not found
"""
health_mon = None
try:
health_mon = self._get_db_obj_until_pending_update(
self._health_mon_repo, health_monitor_id)
except tenacity.RetryError as e:
LOG.warning('Health monitor did not go into %s in 60 seconds. '
'This either due to an in-progress Octavia upgrade '
'or an overloaded and failing database. Assuming '
'an upgrade is in progress and continuing.',
constants.PENDING_UPDATE)
health_mon = e.last_attempt.result()
pool = health_mon.pool
listeners = pool.listeners
pool.health_monitor = health_mon
load_balancer = pool.load_balancer
update_hm_tf = self._taskflow_load(
self._health_monitor_flows.get_update_health_monitor_flow(),
store={constants.HEALTH_MON: health_mon,
constants.POOL: pool,
constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer,
constants.UPDATE_DICT: health_monitor_updates})
with tf_logging.DynamicLoggingListener(update_hm_tf,
log=LOG):
update_hm_tf.run()
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
def create_listener(self, listener_id):
"""Creates a listener.
:param listener_id: ID of the listener to create
:returns: None
:raises NoResultFound: Unable to find the object
"""
listener = self._listener_repo.get(db_apis.get_session(),
id=listener_id)
if not listener:
LOG.warning('Failed to fetch %s %s from DB. Retrying for up to '
'60 seconds.', 'listener', listener_id)
raise db_exceptions.NoResultFound
load_balancer = listener.load_balancer
create_listener_tf = self._taskflow_load(self._listener_flows.
get_create_listener_flow(),
store={constants.LOADBALANCER:
load_balancer,
constants.LISTENERS:
[listener]})
with tf_logging.DynamicLoggingListener(create_listener_tf,
log=LOG):
create_listener_tf.run()
def delete_listener(self, listener_id):
"""Deletes a listener.
:param listener_id: ID of the listener to delete
:returns: None
:raises ListenerNotFound: The referenced listener was not found
"""
listener = self._listener_repo.get(db_apis.get_session(),
id=listener_id)
load_balancer = listener.load_balancer
delete_listener_tf = self._taskflow_load(
self._listener_flows.get_delete_listener_flow(),
store={constants.LOADBALANCER: load_balancer,
constants.LISTENER: listener})
with tf_logging.DynamicLoggingListener(delete_listener_tf,
log=LOG):
delete_listener_tf.run()
def update_listener(self, listener_id, listener_updates):
"""Updates a listener.
:param listener_id: ID of the listener to update
:param listener_updates: Dict containing updated listener attributes
:returns: None
:raises ListenerNotFound: The referenced listener was not found
"""
listener = None
try:
listener = self._get_db_obj_until_pending_update(
self._listener_repo, listener_id)
except tenacity.RetryError as e:
LOG.warning('Listener did not go into %s in 60 seconds. '
'This either due to an in-progress Octavia upgrade '
'or an overloaded and failing database. Assuming '
'an upgrade is in progress and continuing.',
constants.PENDING_UPDATE)
listener = e.last_attempt.result()
load_balancer = listener.load_balancer
update_listener_tf = self._taskflow_load(self._listener_flows.
get_update_listener_flow(),
store={constants.LISTENER:
listener,
constants.LOADBALANCER:
load_balancer,
constants.UPDATE_DICT:
listener_updates,
constants.LISTENERS:
[listener]})
with tf_logging.DynamicLoggingListener(update_listener_tf, log=LOG):
update_listener_tf.run()
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
def create_load_balancer(self, load_balancer_id, flavor=None):
"""Creates a load balancer by allocating Amphorae.
First tries to allocate an existing Amphora in READY state.
If none are available it will attempt to build one specifically
for this load balancer.
:param load_balancer_id: ID of the load balancer to create
:returns: None
:raises NoResultFound: Unable to find the object
"""
lb = self._lb_repo.get(db_apis.get_session(), id=load_balancer_id)
if not lb:
LOG.warning('Failed to fetch %s %s from DB. Retrying for up to '
'60 seconds.', 'load_balancer', load_balancer_id)
raise db_exceptions.NoResultFound
# TODO(johnsom) convert this to octavia_lib constant flavor
# once octavia is transitioned to use octavia_lib
store = {constants.LOADBALANCER_ID: load_balancer_id,
constants.BUILD_TYPE_PRIORITY:
constants.LB_CREATE_NORMAL_PRIORITY,
constants.FLAVOR: flavor}
topology = lb.topology
store[constants.UPDATE_DICT] = {
constants.TOPOLOGY: topology
}
create_lb_flow = self._lb_flows.get_create_load_balancer_flow(
topology=topology, listeners=lb.listeners)
create_lb_tf = self._taskflow_load(create_lb_flow, store=store)
with tf_logging.DynamicLoggingListener(create_lb_tf, log=LOG):
create_lb_tf.run()
def delete_load_balancer(self, load_balancer_id, cascade=False):
"""Deletes a load balancer by de-allocating Amphorae.
:param load_balancer_id: ID of the load balancer to delete
:returns: None
:raises LBNotFound: The referenced load balancer was not found
"""
lb = self._lb_repo.get(db_apis.get_session(),
id=load_balancer_id)
if cascade:
(flow,
store) = self._lb_flows.get_cascade_delete_load_balancer_flow(lb)
else:
(flow, store) = self._lb_flows.get_delete_load_balancer_flow(lb)
store.update({constants.LOADBALANCER: lb,
constants.SERVER_GROUP_ID: lb.server_group_id})
delete_lb_tf = self._taskflow_load(flow, store=store)
with tf_logging.DynamicLoggingListener(delete_lb_tf,
log=LOG):
delete_lb_tf.run()
def update_load_balancer(self, load_balancer_id, load_balancer_updates):
"""Updates a load balancer.
:param load_balancer_id: ID of the load balancer to update
:param load_balancer_updates: Dict containing updated load balancer
:returns: None
:raises LBNotFound: The referenced load balancer was not found
"""
lb = None
try:
lb = self._get_db_obj_until_pending_update(
self._lb_repo, load_balancer_id)
except tenacity.RetryError as e:
LOG.warning('Load balancer did not go into %s in 60 seconds. '
'This either due to an in-progress Octavia upgrade '
'or an overloaded and failing database. Assuming '
'an upgrade is in progress and continuing.',
constants.PENDING_UPDATE)
lb = e.last_attempt.result()
listeners, _ = self._listener_repo.get_all(
db_apis.get_session(),
load_balancer_id=load_balancer_id)
update_lb_tf = self._taskflow_load(
self._lb_flows.get_update_load_balancer_flow(),
store={constants.LOADBALANCER: lb,
constants.LISTENERS: listeners,
constants.UPDATE_DICT: load_balancer_updates})
with tf_logging.DynamicLoggingListener(update_lb_tf,
log=LOG):
update_lb_tf.run()
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
def create_member(self, member_id):
"""Creates a pool member.
:param member_id: ID of the member to create
:returns: None
:raises NoSuitablePool: Unable to find the node pool
"""
member = self._member_repo.get(db_apis.get_session(),
id=member_id)
if not member:
LOG.warning('Failed to fetch %s %s from DB. Retrying for up to '
'60 seconds.', 'member', member_id)
raise db_exceptions.NoResultFound
pool = member.pool
listeners = pool.listeners
load_balancer = pool.load_balancer
create_member_tf = self._taskflow_load(self._member_flows.
get_create_member_flow(),
store={constants.MEMBER: member,
constants.LISTENERS:
listeners,
constants.LOADBALANCER:
load_balancer,
constants.POOL: pool})
with tf_logging.DynamicLoggingListener(create_member_tf,
log=LOG):
create_member_tf.run()
def delete_member(self, member_id):
"""Deletes a pool member.
:param member_id: ID of the member to delete
:returns: None
:raises MemberNotFound: The referenced member was not found
"""
member = self._member_repo.get(db_apis.get_session(),
id=member_id)
pool = member.pool
listeners = pool.listeners
load_balancer = pool.load_balancer
delete_member_tf = self._taskflow_load(
self._member_flows.get_delete_member_flow(),
store={constants.MEMBER: member, constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer, constants.POOL: pool}
)
with tf_logging.DynamicLoggingListener(delete_member_tf,
log=LOG):
delete_member_tf.run()
def batch_update_members(self, old_member_ids, new_member_ids,
updated_members):
old_members = [self._member_repo.get(db_apis.get_session(), id=mid)
for mid in old_member_ids]
new_members = [self._member_repo.get(db_apis.get_session(), id=mid)
for mid in new_member_ids]
updated_members = [
(self._member_repo.get(db_apis.get_session(), id=m.get('id')), m)
for m in updated_members]
if old_members:
pool = old_members[0].pool
elif new_members:
pool = new_members[0].pool
else:
pool = updated_members[0][0].pool
listeners = pool.listeners
load_balancer = pool.load_balancer
batch_update_members_tf = self._taskflow_load(
self._member_flows.get_batch_update_members_flow(
old_members, new_members, updated_members),
store={constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer,
constants.POOL: pool})
with tf_logging.DynamicLoggingListener(batch_update_members_tf,
log=LOG):
batch_update_members_tf.run()
def update_member(self, member_id, member_updates):
"""Updates a pool member.
:param member_id: ID of the member to update
:param member_updates: Dict containing updated member attributes
:returns: None
:raises MemberNotFound: The referenced member was not found
"""
member = None
try:
member = self._get_db_obj_until_pending_update(
self._member_repo, member_id)
except tenacity.RetryError as e:
LOG.warning('Member did not go into %s in 60 seconds. '
'This either due to an in-progress Octavia upgrade '
'or an overloaded and failing database. Assuming '
'an upgrade is in progress and continuing.',
constants.PENDING_UPDATE)
member = e.last_attempt.result()
pool = member.pool
listeners = pool.listeners
load_balancer = pool.load_balancer
update_member_tf = self._taskflow_load(self._member_flows.
get_update_member_flow(),
store={constants.MEMBER: member,
constants.LISTENERS:
listeners,
constants.LOADBALANCER:
load_balancer,
constants.POOL:
pool,
constants.UPDATE_DICT:
member_updates})
with tf_logging.DynamicLoggingListener(update_member_tf,
log=LOG):
update_member_tf.run()
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
def create_pool(self, pool_id):
"""Creates a node pool.
:param pool_id: ID of the pool to create
:returns: None
:raises NoResultFound: Unable to find the object
"""
pool = self._pool_repo.get(db_apis.get_session(),
id=pool_id)
if not pool:
LOG.warning('Failed to fetch %s %s from DB. Retrying for up to '
'60 seconds.', 'pool', pool_id)
raise db_exceptions.NoResultFound
listeners = pool.listeners
load_balancer = pool.load_balancer
create_pool_tf = self._taskflow_load(self._pool_flows.
get_create_pool_flow(),
store={constants.POOL: pool,
constants.LISTENERS:
listeners,
constants.LOADBALANCER:
load_balancer})
with tf_logging.DynamicLoggingListener(create_pool_tf,
log=LOG):
create_pool_tf.run()
def delete_pool(self, pool_id):
"""Deletes a node pool.
:param pool_id: ID of the pool to delete
:returns: None
:raises PoolNotFound: The referenced pool was not found
"""
pool = self._pool_repo.get(db_apis.get_session(),
id=pool_id)
load_balancer = pool.load_balancer
listeners = pool.listeners
delete_pool_tf = self._taskflow_load(
self._pool_flows.get_delete_pool_flow(),
store={constants.POOL: pool, constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer})
with tf_logging.DynamicLoggingListener(delete_pool_tf,
log=LOG):
delete_pool_tf.run()
def update_pool(self, pool_id, pool_updates):
"""Updates a node pool.
:param pool_id: ID of the pool to update
:param pool_updates: Dict containing updated pool attributes
:returns: None
:raises PoolNotFound: The referenced pool was not found
"""
pool = None
try:
pool = self._get_db_obj_until_pending_update(
self._pool_repo, pool_id)
except tenacity.RetryError as e:
LOG.warning('Pool did not go into %s in 60 seconds. '
'This either due to an in-progress Octavia upgrade '
'or an overloaded and failing database. Assuming '
'an upgrade is in progress and continuing.',
constants.PENDING_UPDATE)
pool = e.last_attempt.result()
listeners = pool.listeners
load_balancer = pool.load_balancer
update_pool_tf = self._taskflow_load(self._pool_flows.
get_update_pool_flow(),
store={constants.POOL: pool,
constants.LISTENERS:
listeners,
constants.LOADBALANCER:
load_balancer,
constants.UPDATE_DICT:
pool_updates})
with tf_logging.DynamicLoggingListener(update_pool_tf,
log=LOG):
update_pool_tf.run()
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
def create_l7policy(self, l7policy_id):
"""Creates an L7 Policy.
:param l7policy_id: ID of the l7policy to create
:returns: None
:raises NoResultFound: Unable to find the object
"""
l7policy = self._l7policy_repo.get(db_apis.get_session(),
id=l7policy_id)
if not l7policy:
LOG.warning('Failed to fetch %s %s from DB. Retrying for up to '
'60 seconds.', 'l7policy', l7policy_id)
raise db_exceptions.NoResultFound
listeners = [l7policy.listener]
load_balancer = l7policy.listener.load_balancer
create_l7policy_tf = self._taskflow_load(
self._l7policy_flows.get_create_l7policy_flow(),
store={constants.L7POLICY: l7policy,
constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer})
with tf_logging.DynamicLoggingListener(create_l7policy_tf,
log=LOG):
create_l7policy_tf.run()
def delete_l7policy(self, l7policy_id):
"""Deletes an L7 policy.
:param l7policy_id: ID of the l7policy to delete
:returns: None
:raises L7PolicyNotFound: The referenced l7policy was not found
"""
l7policy = self._l7policy_repo.get(db_apis.get_session(),
id=l7policy_id)
load_balancer = l7policy.listener.load_balancer
listeners = [l7policy.listener]
delete_l7policy_tf = self._taskflow_load(
self._l7policy_flows.get_delete_l7policy_flow(),
store={constants.L7POLICY: l7policy,
constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer})
with tf_logging.DynamicLoggingListener(delete_l7policy_tf,
log=LOG):
delete_l7policy_tf.run()
def update_l7policy(self, l7policy_id, l7policy_updates):
"""Updates an L7 policy.
:param l7policy_id: ID of the l7policy to update
:param l7policy_updates: Dict containing updated l7policy attributes
:returns: None
:raises L7PolicyNotFound: The referenced l7policy was not found
"""
l7policy = None
try:
l7policy = self._get_db_obj_until_pending_update(
self._l7policy_repo, l7policy_id)
except tenacity.RetryError as e:
LOG.warning('L7 policy did not go into %s in 60 seconds. '
'This either due to an in-progress Octavia upgrade '
'or an overloaded and failing database. Assuming '
'an upgrade is in progress and continuing.',
constants.PENDING_UPDATE)
l7policy = e.last_attempt.result()
listeners = [l7policy.listener]
load_balancer = l7policy.listener.load_balancer
update_l7policy_tf = self._taskflow_load(
self._l7policy_flows.get_update_l7policy_flow(),
store={constants.L7POLICY: l7policy,
constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer,
constants.UPDATE_DICT: l7policy_updates})
with tf_logging.DynamicLoggingListener(update_l7policy_tf,
log=LOG):
update_l7policy_tf.run()
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
def create_l7rule(self, l7rule_id):
"""Creates an L7 Rule.
:param l7rule_id: ID of the l7rule to create
:returns: None
:raises NoResultFound: Unable to find the object
"""
l7rule = self._l7rule_repo.get(db_apis.get_session(),
id=l7rule_id)
if not l7rule:
LOG.warning('Failed to fetch %s %s from DB. Retrying for up to '
'60 seconds.', 'l7rule', l7rule_id)
raise db_exceptions.NoResultFound
l7policy = l7rule.l7policy
listeners = [l7policy.listener]
load_balancer = l7policy.listener.load_balancer
create_l7rule_tf = self._taskflow_load(
self._l7rule_flows.get_create_l7rule_flow(),
store={constants.L7RULE: l7rule,
constants.L7POLICY: l7policy,
constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer})
with tf_logging.DynamicLoggingListener(create_l7rule_tf,
log=LOG):
create_l7rule_tf.run()
def delete_l7rule(self, l7rule_id):
"""Deletes an L7 rule.
:param l7rule_id: ID of the l7rule to delete
:returns: None
:raises L7RuleNotFound: The referenced l7rule was not found
"""
l7rule = self._l7rule_repo.get(db_apis.get_session(),
id=l7rule_id)
l7policy = l7rule.l7policy
load_balancer = l7policy.listener.load_balancer
listeners = [l7policy.listener]
delete_l7rule_tf = self._taskflow_load(
self._l7rule_flows.get_delete_l7rule_flow(),
store={constants.L7RULE: l7rule,
constants.L7POLICY: l7policy,
constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer})
with tf_logging.DynamicLoggingListener(delete_l7rule_tf,
log=LOG):
delete_l7rule_tf.run()
def update_l7rule(self, l7rule_id, l7rule_updates):
"""Updates an L7 rule.
:param l7rule_id: ID of the l7rule to update
:param l7rule_updates: Dict containing updated l7rule attributes
:returns: None
:raises L7RuleNotFound: The referenced l7rule was not found
"""
l7rule = None
try:
l7rule = self._get_db_obj_until_pending_update(
self._l7rule_repo, l7rule_id)
except tenacity.RetryError as e:
LOG.warning('L7 rule did not go into %s in 60 seconds. '
'This either due to an in-progress Octavia upgrade '
'or an overloaded and failing database. Assuming '
'an upgrade is in progress and continuing.',
constants.PENDING_UPDATE)
l7rule = e.last_attempt.result()
l7policy = l7rule.l7policy
listeners = [l7policy.listener]
load_balancer = l7policy.listener.load_balancer
update_l7rule_tf = self._taskflow_load(
self._l7rule_flows.get_update_l7rule_flow(),
store={constants.L7RULE: l7rule,
constants.L7POLICY: l7policy,
constants.LISTENERS: listeners,
constants.LOADBALANCER: load_balancer,
constants.UPDATE_DICT: l7rule_updates})
with tf_logging.DynamicLoggingListener(update_l7rule_tf,
log=LOG):
update_l7rule_tf.run()
def _perform_amphora_failover(self, amp, priority):
"""Internal method to perform failover operations for an amphora.
:param amp: The amphora to failover
:param priority: The create priority
:returns: None
"""
stored_params = {constants.FAILED_AMPHORA: amp,
constants.LOADBALANCER_ID: amp.load_balancer_id,
constants.BUILD_TYPE_PRIORITY: priority, }
if amp.status == constants.DELETED:
LOG.warning('Amphora %s is marked DELETED in the database but '
'was submitted for failover. Deleting it from the '
'amphora health table to exclude it from health '
'checks and skipping the failover.', amp.id)
self._amphora_health_repo.delete(db_apis.get_session(),
amphora_id=amp.id)
return
if (CONF.house_keeping.spare_amphora_pool_size == 0) and (
CONF.nova.enable_anti_affinity is False):
LOG.warning("Failing over amphora with no spares pool may "
"cause delays in failover times while a new "
"amphora instance boots.")
# if we run with anti-affinity we need to set the server group
# as well
lb = self._amphora_repo.get_lb_for_amphora(
db_apis.get_session(), amp.id)
if CONF.nova.enable_anti_affinity and lb:
stored_params[constants.SERVER_GROUP_ID] = lb.server_group_id
if lb and lb.flavor_id:
stored_params[constants.FLAVOR] = (
self._flavor_repo.get_flavor_metadata_dict(
db_apis.get_session(), lb.flavor_id))
else:
stored_params[constants.FLAVOR] = {}
failover_amphora_tf = self._taskflow_load(
self._amphora_flows.get_failover_flow(
role=amp.role, load_balancer=lb),
store=stored_params)
with tf_logging.DynamicLoggingListener(failover_amphora_tf, log=LOG):
failover_amphora_tf.run()
def failover_amphora(self, amphora_id):
"""Perform failover operations for an amphora.
:param amphora_id: ID for amphora to failover
:returns: None
:raises AmphoraNotFound: The referenced amphora was not found
"""
try:
amp = self._amphora_repo.get(db_apis.get_session(),
id=amphora_id)
if not amp:
LOG.warning("Could not fetch Amphora %s from DB, ignoring "
"failover request.", amphora_id)
return
self._perform_amphora_failover(
amp, constants.LB_CREATE_FAILOVER_PRIORITY)
if amp.load_balancer_id:
LOG.info("Mark ACTIVE in DB for load balancer id: %s",
amp.load_balancer_id)
self._lb_repo.update(
db_apis.get_session(), amp.load_balancer_id,
provisioning_status=constants.ACTIVE)
except Exception as e:
try:
self._lb_repo.update(
db_apis.get_session(), amp.load_balancer_id,
provisioning_status=constants.ERROR)
except Exception:
LOG.error("Unable to revert LB status to ERROR.")
with excutils.save_and_reraise_exception():
LOG.error("Failover exception: %s", e)
def failover_loadbalancer(self, load_balancer_id):
"""Perform failover operations for a load balancer.
:param load_balancer_id: ID for load balancer to failover
:returns: None
:raises LBNotFound: The referenced load balancer was not found
"""
# Note: This expects that the load balancer is already in
# provisioning_status=PENDING_UPDATE state
try:
lb = self._lb_repo.get(db_apis.get_session(),
id=load_balancer_id)
# Exclude amphora already deleted
amps = [a for a in lb.amphorae if a.status != constants.DELETED]
for amp in amps:
# failover amphora in backup role
# Note: this amp may not currently be the backup
# TODO(johnsom) Change this to query the amp state
# once the amp API supports it.
if amp.role == constants.ROLE_BACKUP:
self._perform_amphora_failover(
amp, constants.LB_CREATE_ADMIN_FAILOVER_PRIORITY)
for amp in amps:
# failover everyhting else
if amp.role != constants.ROLE_BACKUP:
self._perform_amphora_failover(
amp, constants.LB_CREATE_ADMIN_FAILOVER_PRIORITY)
self._lb_repo.update(
db_apis.get_session(), load_balancer_id,
provisioning_status=constants.ACTIVE)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.error("LB %(lbid)s failover exception: %(exc)s",
{'lbid': load_balancer_id, 'exc': e})
self._lb_repo.update(
db_apis.get_session(), load_balancer_id,
provisioning_status=constants.ERROR)
def amphora_cert_rotation(self, amphora_id):
"""Perform cert rotation for an amphora.
:param amphora_id: ID for amphora to rotate
:returns: None
:raises AmphoraNotFound: The referenced amphora was not found
"""
amp = self._amphora_repo.get(db_apis.get_session(),
id=amphora_id)
LOG.info("Start amphora cert rotation, amphora's id is: %s", amp.id)
certrotation_amphora_tf = self._taskflow_load(
self._amphora_flows.cert_rotate_amphora_flow(),
store={constants.AMPHORA: amp,
constants.AMPHORA_ID: amp.id})
with tf_logging.DynamicLoggingListener(certrotation_amphora_tf,
log=LOG):
certrotation_amphora_tf.run()
def update_amphora_agent_config(self, amphora_id):
"""Update the amphora agent configuration.
Note: This will update the amphora agent configuration file and
update the running configuration for mutatable configuration
items.
:param amphora_id: ID of the amphora to update.
:returns: None
"""
LOG.info("Start amphora agent configuration update, amphora's id "
"is: %s", amphora_id)
amp = self._amphora_repo.get(db_apis.get_session(), id=amphora_id)
lb = self._amphora_repo.get_lb_for_amphora(db_apis.get_session(),
amphora_id)
flavor = {}
if lb.flavor_id:
flavor = self._flavor_repo.get_flavor_metadata_dict(
db_apis.get_session(), lb.flavor_id)
update_amphora_tf = self._taskflow_load(
self._amphora_flows.update_amphora_config_flow(),
store={constants.AMPHORA: amp,
constants.FLAVOR: flavor})
with tf_logging.DynamicLoggingListener(update_amphora_tf,
log=LOG):
update_amphora_tf.run()
import logging
from tenacity import after_log, before_log, retry, stop_after_attempt, wait_fixed
from app.db.session import SessionLocal
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
max_tries = 60 * 5 # 5 minutes
wait_seconds = 1
@retry(
stop=stop_after_attempt(max_tries),
wait=wait_fixed(wait_seconds),
before=before_log(logger, logging.INFO),
after=after_log(logger, logging.WARN),
)
def init() -> None:
try:
db = SessionLocal()
# Try to create session to check if DB is awake
db.execute("SELECT 1")
except Exception as e:
logger.error(e)
raise e
def main() -> None:
logger.info("Initializing service")
class FaultyHttpMock:
@retry(retry=retry_if_exception_type(SnapshotHttpException),
stop=stop_after_attempt(1))
async def get_orderbook_snapshot(self, product_id, http_timeout):
raise SnapshotHttpException
class FileUploader:
UPLOAD_RETRY_COUNT = 5
def __init__(self, file_retriever, upload_path, dbx):
self._dbx = dbx
self._upload_path = upload_path
self._file_retriever = file_retriever
self._buffer = io.BytesIO()
self._total_bytes = 0
self._session = None
@property
def total_bytes(self):
return self._total_bytes
@retry(stop=stop_after_attempt(UPLOAD_RETRY_COUNT),
retry=retry_if_exception_type(requests.exceptions.ReadTimeout),
after=after_log(logging.getLogger(), logging.WARNING))
def upload(self):
self._file_retriever.read(self._buffer.write)
bytes = self._buffer.getvalue()
assert self._total_bytes + len(bytes) == self._file_retriever.size
self._dbx.files_upload(bytes,
path=self._upload_path,
mode=WriteMode('overwrite'),
client_modified=self._file_retriever.modified)
self._total_bytes += len(bytes)
def upload_in_chunks(self, chunk_size):
self._file_retriever.read(lambda data: self._on_read(data, chunk_size))
assert self._session is not None
bytes = self._buffer.getvalue()
assert self._total_bytes + len(bytes) == self._file_retriever.size
self._upload_session(bytes, finish=True)
self._total_bytes += len(bytes)
logging.info(
'chunk %s bytes uploaded. upload finished, total: %s bytes' %
(len(bytes), self._total_bytes))
def _on_read(self, data, chunk_size):
self._buffer.write(data)
if self._buffer.tell() > chunk_size:
bytes = self._buffer.getvalue()
self._upload_session(bytes, finish=False)
self._total_bytes += len(bytes)
logging.info('chunk %s bytes uploaded, total: %s bytes' %
(len(bytes), self._total_bytes))
self._buffer.truncate(0)
self._buffer.seek(0)
@retry(stop=stop_after_attempt(UPLOAD_RETRY_COUNT),
retry=retry_if_exception_type(requests.exceptions.ReadTimeout),
after=after_log(logging.getLogger(), logging.WARNING))
def _upload_session(self, bytes, finish):
if self._session is None:
start_result = self._dbx.files_upload_session_start(bytes)
self._session = start_result.session_id
elif finish:
cursor = UploadSessionCursor(self._session, self._total_bytes)
info = CommitInfo(path=self._upload_path,
mode=WriteMode('overwrite'),
client_modified=self._file_retriever.modified)
self._dbx.files_upload_session_finish(bytes, cursor, info)
else:
cursor = UploadSessionCursor(self._session, self._total_bytes)
self._dbx.files_upload_session_append_v2(bytes, cursor)
def reply(self, reply=None, failure=None):
"""Send back reply to the RPC client
:param reply: Dictionary, reply. In case of exception should be None
:param failure: Tuple, should be a sys.exc_info() tuple.
Should be None if RPC request was successfully processed.
:return RpcReplyPikaIncomingMessage, message with reply
"""
if self.reply_q is None:
return
reply_outgoing_message = RpcReplyPikaOutgoingMessage(
self._pika_engine, self.msg_id, reply=reply, failure_info=failure,
content_type=self._content_type,
)
def on_exception(ex):
if isinstance(ex, pika_drv_exc.ConnectionException):
LOG.warning(
"Connectivity related problem during reply sending. %s",
ex
)
return True
else:
return False
if self._pika_engine.rpc_reply_retry_attempts:
retrier = tenacity.retry(
stop=(
tenacity.stop_never
if self._pika_engine.rpc_reply_retry_attempts == -1 else
tenacity.stop_after_attempt(
self._pika_engine.rpc_reply_retry_attempts
)
),
retry=tenacity.retry_if_exception(on_exception),
wait=tenacity.wait_fixed(
self._pika_engine.rpc_reply_retry_delay
)
)
else:
retrier = None
try:
timeout = (None if self.expiration_time is None else
max(self.expiration_time - time.time(), 0))
with timeutils.StopWatch(duration=timeout) as stopwatch:
reply_outgoing_message.send(
reply_q=self.reply_q,
stopwatch=stopwatch,
retrier=retrier
)
LOG.debug(
"Message [id:'%s'] replied to '%s'.", self.msg_id, self.reply_q
)
except Exception:
LOG.exception(
"Message [id:'%s'] wasn't replied to : %s", self.msg_id,
self.reply_q
)
# limitations under the License.
"""Encapsulate glance-simplestreams-sync testing."""
import json
import logging
import requests
import tenacity
import zaza.model as zaza_model
import zaza.openstack.charm_tests.test_utils as test_utils
import zaza.openstack.utilities.openstack as openstack_utils
@tenacity.retry(
retry=tenacity.retry_if_result(lambda images: len(images) < 4),
wait=tenacity.wait_fixed(6), # interval between retries
stop=tenacity.stop_after_attempt(100)) # retry times
def retry_image_sync(glance_client):
"""Wait for image sync with retry."""
# convert generator to list
return list(glance_client.images.list())
@tenacity.retry(retry=tenacity.retry_if_exception_type(
json.decoder.JSONDecodeError),
wait=tenacity.wait_fixed(10),
reraise=True,
stop=tenacity.stop_after_attempt(10))
def get_product_streams(url):
"""Get product streams json data with retry."""
# There is a race between the images being available in glance and any
# metadata being written. Use tenacity to avoid this race.
class FunctionVersionsController(rest.RestController):
_custom_actions = {
'scale_up': ['POST'],
'scale_down': ['POST'],
'detach': ['POST'],
}
def __init__(self, *args, **kwargs):
self.type = 'function_version'
self.storage_provider = storage_base.load_storage_provider(CONF)
self.engine_client = rpc.get_engine_client()
super(FunctionVersionsController, self).__init__(*args, **kwargs)
@tenacity.retry(wait=tenacity.wait_fixed(1),
stop=tenacity.stop_after_attempt(30),
reraise=True,
retry=tenacity.retry_if_exception_type(
exc.EtcdLockException))
def _create_function_version(self, project_id, function_id, **kwargs):
with etcd_util.get_function_version_lock(function_id) as lock:
if not lock.is_acquired():
raise exc.EtcdLockException(
"Etcd: failed to acquire version lock for function %s." %
function_id)
with db_api.transaction():
# Get latest function package md5 and version number
func_db = db_api.get_function(function_id, insecure=False)
if func_db.code['source'] != constants.PACKAGE_FUNCTION:
raise exc.NotAllowedException(
"Function versioning only allowed for %s type "
"function." % constants.PACKAGE_FUNCTION)
l_md5 = func_db.code['md5sum']
l_version = func_db.latest_version
if len(func_db.versions) >= constants.MAX_VERSION_NUMBER:
raise exc.NotAllowedException(
'Can not exceed maximum number(%s) of versions' %
constants.MAX_VERSION_NUMBER)
# Check if the latest package changed since last version
changed = self.storage_provider.changed_since(
project_id, function_id, l_md5, l_version)
if not changed:
raise exc.NotAllowedException(
'Function package not changed since the latest '
'version %s.' % l_version)
LOG.info("Creating %s, function_id: %s, old_version: %d",
self.type, function_id, l_version)
# Create new version and copy package.
self.storage_provider.copy(project_id, function_id, l_md5,
l_version)
version = db_api.increase_function_version(
function_id, l_version, **kwargs)
func_db.latest_version = l_version + 1
LOG.info("New version %d for function %s created.", l_version + 1,
function_id)
return version
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(resources.FunctionVersion,
types.uuid,
body=resources.FunctionVersion,
status_code=201)
def post(self, function_id, body):
"""Create a new version for the function.
Only allow to create version for package type function.
The supported boy params:
- description: Optional. The description of the new version.
"""
ctx = context.get_ctx()
acl.enforce('function_version:create', ctx)
params = body.to_dict()
values = {
'description': params.get('description'),
}
# Try to create a new function version within lock and db transaction
try:
version = self._create_function_version(ctx.project_id,
function_id, **values)
except exc.EtcdLockException as e:
LOG.exception(str(e))
# Reraise a generic exception as the end users should not know
# the underlying details.
raise exc.QinlingException('Internal server error.')
return resources.FunctionVersion.from_db_obj(version)
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(resources.FunctionVersions, types.uuid)
def get_all(self, function_id):
"""Get all the versions of the given function.
Admin user can get all versions for the normal user's function.
"""
acl.enforce('function_version:get_all', context.get_ctx())
LOG.info("Getting versions for function %s.", function_id)
# Getting function and versions needs to happen in a db transaction
with db_api.transaction():
func_db = db_api.get_function(function_id)
db_versions = func_db.versions
versions = [
resources.FunctionVersion.from_db_obj(v) for v in db_versions
]
return resources.FunctionVersions(function_versions=versions)
@rest_utils.wrap_pecan_controller_exception
@pecan.expose()
@pecan.expose('json')
def get(self, function_id, version):
"""Get function version or download function version package.
This method can support HTTP request using either
'Accept:application/json' or no 'Accept' header.
"""
ctx = context.get_ctx()
acl.enforce('function_version:get', ctx)
download = strutils.bool_from_string(
pecan.request.GET.get('download', False))
version = int(version)
version_db = db_api.get_function_version(function_id, version)
if not download:
LOG.info("Getting version %s for function %s.", version,
function_id)
pecan.override_template('json')
return resources.FunctionVersion.from_db_obj(version_db).to_dict()
LOG.info("Downloading version %s for function %s.", version,
function_id)
f = self.storage_provider.retrieve(version_db.project_id,
function_id,
None,
version=version)
if isinstance(f, collections.Iterable):
pecan.response.app_iter = f
else:
pecan.response.app_iter = FileIter(f)
pecan.response.headers['Content-Type'] = 'application/zip'
pecan.response.headers['Content-Disposition'] = (
'attachment; filename="%s_%s"' % (function_id, version))
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(None, types.uuid, int, status_code=204)
def delete(self, function_id, version):
"""Delete a specific function version.
- The version should not being used by any job
- The version should not being used by any webhook
- Admin user can not delete normal user's version
"""
ctx = context.get_ctx()
acl.enforce('function_version:delete', ctx)
LOG.info("Deleting version %s of function %s.", version, function_id)
with db_api.transaction():
version_db = db_api.get_function_version(function_id,
version,
insecure=False)
latest_version = version_db.function.latest_version
version_jobs = db_api.get_jobs(
function_id=version_db.function_id,
function_version=version_db.version_number,
status={'nin': ['done', 'cancelled']})
if len(version_jobs) > 0:
raise exc.NotAllowedException(
'The function version is still associated with running '
'job(s).')
version_webhook = db_api.get_webhooks(
function_id=version_db.function_id,
function_version=version_db.version_number,
)
if len(version_webhook) > 0:
raise exc.NotAllowedException(
'The function version is still associated with webhook.')
filters = rest_utils.get_filters(
function_id=version_db.function_id,
function_version=version_db.version_number)
version_aliases = db_api.get_function_aliases(**filters)
if len(version_aliases) > 0:
raise exc.NotAllowedException(
'The function version is still associated with alias.')
# Delete resources for function version
self.engine_client.delete_function(function_id, version=version)
etcd_util.delete_function(function_id, version=version)
self.storage_provider.delete(ctx.projectid,
function_id,
None,
version=version)
db_api.delete_function_version(function_id, version)
if latest_version == version:
version_db.function.latest_version = latest_version - 1
LOG.info("Version %s of function %s deleted.", version, function_id)
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(None,
types.uuid,
int,
body=resources.ScaleInfo,
status_code=202)
def scale_up(self, function_id, version, scale):
"""Scale up the workers for function version execution.
This is admin only operation. The load monitoring of execution
depends on the monitoring solution of underlying orchestrator.
"""
acl.enforce('function_version:scale_up', context.get_ctx())
func_db = db_api.get_function(function_id)
# If version=0, it's equivalent to /functions/<funcion-id>/scale_up
if version > 0:
db_api.get_function_version(function_id, version)
params = scale.to_dict()
LOG.info('Starting to scale up function %s(version %s), params: %s',
function_id, version, params)
self.engine_client.scaleup_function(function_id,
runtime_id=func_db.runtime_id,
version=version,
count=params['count'])
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(None,
types.uuid,
int,
body=resources.ScaleInfo,
status_code=202)
def scale_down(self, function_id, version, scale):
"""Scale down the workers for function version execution.
This is admin only operation. The load monitoring of execution
depends on the monitoring solution of underlying orchestrator.
"""
acl.enforce('function_version:scale_down', context.get_ctx())
db_api.get_function(function_id)
params = scale.to_dict()
# If version=0, it's equivalent to /functions/<funcion-id>/scale_down
if version > 0:
db_api.get_function_version(function_id, version)
workers = etcd_util.get_workers(function_id, version=version)
if len(workers) <= 1:
LOG.info('No need to scale down function %s(version %s)',
function_id, version)
return
LOG.info('Starting to scale down function %s(version %s), params: %s',
function_id, version, params)
self.engine_client.scaledown_function(function_id,
version=version,
count=params['count'])
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(None, types.uuid, int, status_code=202)
def detach(self, function_id, version):
"""Detach the function version from its underlying workers.
This is admin only operation, which gives admin user a safe way to
clean up the underlying resources allocated for the function version.
"""
acl.enforce('function_version:detach', context.get_ctx())
db_api.get_function(function_id)
# If version=0, it's equivalent to /functions/<funcion-id>/detach
if version > 0:
db_api.get_function_version(function_id, version)
LOG.info('Starting to detach function %s(version %s)', function_id,
version)
# Delete allocated resources in orchestrator and etcd keys.
self.engine_client.delete_function(function_id, version=version)
etcd_util.delete_function(function_id, version=version)
render_message_template,
)
from dispatch.plugins.bases import EmailPlugin
from dispatch.plugins.dispatch_google import gmail as google_gmail_plugin
from dispatch.plugins.dispatch_google.common import get_service
from dispatch.plugins.dispatch_google.config import (
GOOGLE_USER_OVERRIDE,
GOOGLE_SERVICE_ACCOUNT_DELEGATED_ACCOUNT,
)
from .filters import env
log = logging.getLogger(__name__)
@retry(stop=stop_after_attempt(3))
def send_message(service, message):
"""Sends an email message."""
return service.users().messages().send(userId="me", body=message).execute()
def create_html_message(recipient: str, subject: str, body: str) -> Dict:
"""Creates a message for an email."""
message = MIMEText(body, "html")
if GOOGLE_USER_OVERRIDE:
recipient = GOOGLE_USER_OVERRIDE
log.warning("GOOGLE_USER_OVERIDE set. Using override.")
message["to"] = recipient
message["from"] = GOOGLE_SERVICE_ACCOUNT_DELEGATED_ACCOUNT
class InstaLooter(object):
"""A brutal Instagram looter that raids without API tokens.
"""
@CachedClassProperty
@classmethod
def _cachefs(cls):
"""~fs.base.FS: the cache filesystem.
"""
url = "usercache://{}:{}:{}".format(__appname__, __author__, __version__)
return fs.open_fs(url, create=True)
@CachedClassProperty
@classmethod
def _user_agents(cls):
"""~fake_useragent.UserAgent: a collection of fake user-agents.
"""
filename = 'fake_useragent_{}.json'.format(fake_useragent.VERSION)
return fake_useragent.UserAgent(
path=cls._cachefs.getsyspath(filename),
safe_attrs=['__name__', '__objclass__'])
# str: The name of the cookie file in the cache filesystem
_COOKIE_FILE = "cookies.txt"
@classmethod
def _init_session(cls, session=None):
# type: (Optional[Session]) -> Session
"""Initialise the given session and load class cookies to its jar.
Arguments:
session (~requests.Session, optional): a `requests`
session, or `None` to create a new one.
Returns:
~requests.Session: an initialised session instance.
"""
session = session or Session()
# Load cookies
session.cookies = LWPCookieJar(
cls._cachefs.getsyspath(cls._COOKIE_FILE))
try:
typing.cast(FileCookieJar, session.cookies).load()
except IOError:
pass
typing.cast(FileCookieJar, session.cookies).clear_expired_cookies()
return session
@classmethod
def _login(cls, username, password, session=None):
# type: (str, str, Optional[Session]) -> None
"""Login with provided credentials and session.
Arguments:
username (str): the username to log in with.
password (str): the password to log in with.
session (~requests.Session, optional): the session to use,
or `None` to create a new session.
Note:
Code taken from LevPasha/instabot.py
"""
session = cls._init_session(session)
headers = copy.deepcopy(session.headers)
homepage = "https://www.instagram.com/"
login_url = "https://www.instagram.com/accounts/login/ajax/"
data = {'username': username, 'password': password}
try:
session.headers.update({
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive',
'Content-Length': '0',
'Host': 'www.instagram.com',
'Origin': 'https://www.instagram.com',
'Referer': 'https://www.instagram.com',
'User-Agent': cls._user_agents.firefox,
'X-Instagram-AJAX': '1',
'X-Requested-With': 'XMLHttpRequest'
})
with session.get(homepage) as res:
token = get_shared_data(res.text)['config']['csrf_token']
session.headers.update({'X-CSRFToken': token})
time.sleep(5 * random.random()) # nosec
with session.post(login_url, data, allow_redirects=True) as login:
token = next(c.value for c in login.cookies if c.name == 'csrftoken')
session.headers.update({'X-CSRFToken': token})
if not login.ok:
raise SystemError("Login error: check your connection")
data = json.loads(login.text)
if not data.get('authenticated', False):
raise ValueError('Login error: check your login data')
time.sleep(5 * random.random()) # nosec
with session.get(homepage) as res:
if res.text.find(username) == -1:
raise ValueError('Login error: check your login data')
try:
typing.cast(FileCookieJar, session.cookies).save()
except IOError:
pass
finally:
session.headers = headers
@classmethod
def _logout(cls, session=None):
# type: (Optional[Session]) -> None
"""Log out from current session.
Also deletes the eventual cookie file left in the cache directory,
to prevent new connections from using the old session ID.
Arguments:
session (~requests.Session): the session to use, or `None`
to create a new session.
Note:
Code taken from LevPasha/instabot.py
"""
session = cls._init_session(session)
sessionid = cls._sessionid(session)
if sessionid is not None:
url = "https://www.instagram.com/accounts/logout/"
session.post(url, data={"csrfmiddlewaretoken": sessionid})
if cls._cachefs.exists(cls._COOKIE_FILE):
cls._cachefs.remove(cls._COOKIE_FILE)
@classmethod
def _logged_in(cls, session=None):
# type: (Optional[Session]) -> bool
"""Check if there is an open Instagram session.
Arguments:
session (~requests.Session): the session to use, or `None`
to create a new session.
Returns:
bool: `True` if there's an active session, `False` otherwise.
"""
return cls._sessionid(session) is not None
@classmethod
def _sessionid(cls, session=None):
# type: (Optional[Session]) -> Optional[Text]
"""Get the ID of the currently opened Instagram session.
Arguments:
session (~requests.Session): the session to use, or `None`
to create a new session.
Returns:
str or None: the session ID, if any, or `None`.
"""
_session = cls._init_session(session)
_cookies = typing.cast(FileCookieJar, _session.cookies)
return next((ck.value for ck in _cookies
if ck.domain == ".instagram.com"
and ck.name == "ds_user_id"
and ck.path == "/"), None)
def __init__(self,
add_metadata=False, # type: bool
get_videos=False, # type: bool
videos_only=False, # type: bool
jobs=16, # type: int
template="{id}", # type: Text
dump_json=False, # type: bool
dump_only=False, # type: bool
extended_dump=False, # type: bool
session=None # type: Optional[Session]
):
# type: (...) -> None
"""Create a new looter instance.
Arguments:
add_metadata (bool): Add date and comment metadata to
the downloaded pictures.
get_videos (bool): Also get the videos from the given target.
videos_only (bool): Only download videos (implies
``get_videos=True``).
jobs (bool): the number of parallel threads to use to
download media (12 or more is advised to have a true parallel
download of media files).
template (str): a filename format, in Python new-style-formatting
format. See the the :ref:`Template` page of the documentation
for available keys.
dump_json (bool): Save each resource metadata to a
JSON file next to the actual image/video.
dump_only (bool): Only save metadata and discard the actual
resource.
extended_dump (bool): Attempt to fetch as much metadata as
possible, at the cost of more time. Set to `True` if, for
instance, you always want the top comments to be downloaded
in the dump.
session (~requests.Session or None): a `requests` session,
or `None` to create a new one.
"""
self.add_metadata = add_metadata
self.get_videos = get_videos or videos_only
self.videos_only = videos_only
self.jobs = jobs
self.namegen = NameGenerator(template)
self.dump_only = dump_only
self.dump_json = dump_json or dump_only
self.extended_dump = extended_dump
self.session = self._init_session(session)
atexit.register(self.session.close)
# Set a fake User-Agent
if self.session.headers['User-Agent'].startswith('python-requests'):
self.session.headers['User-Agent'] = self._user_agents.firefox
# Get CSRFToken and RHX
with self.session.get('https://www.instagram.com/') as res:
token = get_shared_data(res.text)['config']['csrf_token']
self.session.headers['X-CSRFToken'] = token
self.rhx = get_shared_data(res.text)['rhx_gis']
@abc.abstractmethod
def pages(self):
# type: () -> Iterator[Dict[Text, Any]]
"""Obtain an iterator over Instagram post pages.
Returns:
PageIterator: an iterator over the instagram post pages.
"""
return NotImplemented
def _medias(self,
pages_iterator, # type: Iterable[Dict[Text, Any]]
timeframe=None # type: Optional[_Timeframe]
):
# type: (...) -> Iterator[Dict[Text, Any]]
"""Obtain an iterator over the medias of the given pages iterator.
Arguments:
pages_iterator (Iterator): an iterator over the Instagram
pages, returned by `InstaLooter.pages`
Returns:
MediasIterator: an iterator over the medias in every pages.
"""
if timeframe is not None:
return TimedMediasIterator(pages_iterator, timeframe)
return MediasIterator(pages_iterator)
def medias(self, timeframe=None):
# type: (Optional[_Timeframe]) -> Iterator[Dict[Text, Any]]
"""Obtain an iterator over the Instagram medias.
Wraps the iterator returned by `InstaLooter.pages` to seamlessly
iterate over the medias of all the pages.
Returns:
MediasIterator: an iterator over the medias in every pages.
"""
return self._medias(self.pages(), timeframe)
@tenacity.retry(tenacity.stop_after_attempt(10), tenacity.wait_exponential(1, 10))
def get_post_info(self, code):
# type: (str) -> dict
"""Get media information from a given post code.
Arguments:
code (str): the code of the post (can be obtained either
from the ``shortcode`` attribute of media dictionaries, or
from a post URL: ``https://www.instagram.com/p/<code>/``)
Returns:
dict: a media dictionaries, in the format used by Instagram.
"""
url = "https://www.instagram.com/p/{}/".format(code)
with self.session.get(url) as res:
data = get_shared_data(res.text)
return data['entry_data']['PostPage'][0]['graphql']['shortcode_media']
def download_pictures(self,
destination, # type: Union[str, fs.base.FS]
media_count=None, # type: Optional[int]
timeframe=None, # type: Optional[_Timeframe]
new_only=False, # type: bool
pgpbar_cls=None, # type: Optional[Type[ProgressBar]]
dlpbar_cls=None # type: Optional[Type[ProgressBar]]
):
# type: (...) -> int
"""Download all the pictures to the provided destination.
Actually a shortcut for `.download` with ``condition`` set
to accept only images.
"""
return self.download(
destination,
condition=lambda media: not media["is_video"],
media_count=media_count,
timeframe=timeframe,
new_only=new_only,
pgpbar_cls=pgpbar_cls,
dlpbar_cls=dlpbar_cls,
)
def download_videos(self,
destination, # type: Union[str, fs.base.FS]
media_count=None, # type: Optional[int]
timeframe=None, # type: Optional[_Timeframe]
new_only=False, # type: bool
pgpbar_cls=None, # type: Optional[Type[ProgressBar]]
dlpbar_cls=None, # type: Optional[Type[ProgressBar]]
):
# type: (...) -> int
"""Download all videos to the provided destination.
Actually a shortcut for `.download` with ``condition`` set
to accept only videos.
"""
return self.download(
destination,
condition=lambda media: media["is_video"],
media_count=media_count,
timeframe=timeframe,
new_only=new_only,
pgpbar_cls=pgpbar_cls,
dlpbar_cls=dlpbar_cls,
)
def download(self,
destination, # type: Union[str, fs.base.FS]
condition=None, # type: Optional[Callable[[dict], bool]]
media_count=None, # type: Optional[int]
timeframe=None, # type: Optional[_Timeframe]
new_only=False, # type: bool
pgpbar_cls=None, # type: Optional[Type[ProgressBar]]
dlpbar_cls=None, # type: Optional[Type[ProgressBar]]
):
# type: (...) -> int
"""Download all medias passing ``condition`` to destination.
Arguments:
destination (~fs.base.FS or str): the filesystem where to
store the downloaded files, as a filesystem instance or
FS URL.
condition (function): the condition to filter the
medias with. If `None` is given, a function is created using
the ``get_videos`` and ``videos_only`` passed at object
initialisation.
media_count (int or None): the maximum number of medias
to download. Leave to ``None`` to download everything from
the target. *Note that more files can be downloaded, since
a post with multiple images/videos is considered to be a
single media*.
timeframe (tuple or None): a tuple of two `~datetime.datetime`
objects to enforce a time frame (the first item must be
more recent). Leave to `None` to ignore times.
new_only (bool): stop media discovery when already
downloaded medias are encountered.
pgpbar_cls (type or None): an optional `~.pbar.ProgressBar`
subclass to use to display page scraping progress.
dlpbar_cls (type or None): an optional `~.pbar.ProgressBar`
subclass to use to display file download progress.
Returns:
int: the number of queued medias.
May not be equal to the number of downloaded medias if some
errors occurred during background download.
"""
# Open the destination filesystem
destination, close_destination = self._init_destfs(destination)
# Create an iterator over the pages with an optional progress bar
pages_iterator = self.pages() # type: Iterable[Dict[Text, Any]]
pages_iterator = pgpbar = self._init_pbar(pages_iterator, pgpbar_cls)
# Create an iterator over the medias
medias_iterator = self._medias(iter(pages_iterator), timeframe)
# Create the media download bar from a dummy iterator
dlpbar = self._init_pbar(
six.moves.range(length_hint(medias_iterator)), dlpbar_cls)
# Start a group of workers
workers, queue = self._init_workers(
dlpbar if dlpbar_cls is not None else None, destination)
# Make sure exiting the main thread will shutdown workers
atexit.register(self._shutdown_workers, workers)
# Queue all medias
medias_queued = self._fill_media_queue(
queue, destination, medias_iterator, media_count,
new_only, condition)
# Once queuing the medias is fininished, finish the page progress bar
# and set a new maximum on the download progress bar.
if pgpbar_cls is not None:
pgpbar.finish() # type: ignore
if dlpbar_cls is not None:
dlpbar.set_maximum(medias_queued) # type: ignore
# If no medias were queued, issue a warning
# TODO: refine warning depending on download parameters
if medias_queued == 0:
warnings.warn("No medias found.")
# Add poison pills to the queue and wait for workers to finish
self._poison_workers(workers, queue)
self._join_workers(workers, queue)
# Once downloading is finished, finish the download progress bar
# and close the destination if needed.
if dlpbar_cls is not None:
dlpbar.finish() # type: ignore
if close_destination:
destination.close()
return medias_queued
def login(self, username, password):
# type: (str, str) -> None
"""Log the instance in using the given credentials.
Arguments:
username (str): the username to log in with.
password (str): the password to log in with.
"""
self._login(username, password, session=self.session)
def logout(self):
# type: () -> None
"""Log the instance out from the currently opened session.
"""
self._logout(session=self.session)
def logged_in(self):
# type: () -> bool
"""Check if there's an open Instagram session.
"""
return self._logged_in(self.session)
def _init_pbar(self,
it, # type: Iterable[_T]
pbar_cls=None, # type: Optional[Type[ProgressBar]]
):
# type: (...) -> Iterable[_T]
"""Wrap an iterable within a `ProgressBar`.
Arguments:
it (~collections.Iterable): an iterable to wrap.
pgpbar_cls (type or None): an optional `ProgressBar` subclass
to use, or `None` to avoid using a progress bar.
Returns:
~collections.Iterable: the wrapped iterable.
"""
if pbar_cls is not None:
if not issubclass(pbar_cls, ProgressBar):
raise TypeError("pbar must implement the ProgressBar interface !")
maximum = length_hint(it)
it = pbar = pbar_cls(it)
pbar.set_maximum(maximum)
pbar.set_lock(threading.RLock())
return it
def _init_destfs(self, destination, create=True):
# type: (Union[str, fs.base.FS], bool) -> Tuple[fs.base.FS, bool]
"""Open a filesystem either from a FS URL or filesystem instance.
Arguments:
destination (~fs.base.FS or str): the destination filesystem
to open, as a filesystem instance or FS URL.
create (bool): whether or not to create a new
filesystem if it does not exist.
Returns:
(~fs.base.FS, bool): the open FS, and whether to close it.
"""
close_destination = False
if isinstance(destination, six.binary_type):
destination = destination.decode('utf-8')
if isinstance(destination, six.text_type):
destination = fs.open_fs(destination, create=create)
close_destination = True
if not isinstance(destination, fs.base.FS):
raise TypeError("<destination> must be a FS URL or FS instance.")
return destination, close_destination
def _fill_media_queue(self,
queue, # type: Queue
destination, # type: fs.base.FS
medias_iter, # type: Iterable[Any]
media_count=None, # type: Optional[int]
new_only=False, # type: bool
condition=None, # type: Optional[Callable[[dict], bool]]
):
# type: (...) -> int
"""Fill the download queue with medias from the provided iterator.
Arguments:
queue (~queue.Queue): the download queue to fill.
destination (~fs.base.FS): the filesystem where to download
the files.
medias_iterator (~collections.Iterable): an iterable over the
Instagram medias to download.
media_count (int or None): the maximum number of new medias to
download, or ``None`` to download all discoverable medias.
new_only (bool): stop media discovery when a media that was
already downloaded is encountered.
condition (function or None): the condition to filter the medias
with. If `None` is given, a function is created using the
``get_videos`` and ``videos_only`` passed at object
initialisation.
Returns:
int: the number of queued medias.
May not be equal to the number of downloaded medias if some
errors occurred during downloads.
"""
# Create a condition from parameters if needed
if condition is not None:
_condition = condition # type: Callable[[dict], bool]
else:
if self.videos_only:
def _condition(media): return media['is_video']
elif not self.get_videos:
def _condition(media): return not media['is_video']
else:
def _condition(media): return True
# Queue all media filling the condition
medias_queued = 0
for media in six.moves.filter(_condition, medias_iter):
# Check if the whole post info is required
if self.namegen.needs_extended(media) or media["__typename"] != "GraphImage":
media = self.get_post_info(media['shortcode'])
# Check that sidecar children fit the condition
if media['__typename'] == "GraphSidecar":
# Check that each node fits the condition
for sidecar in media['edge_sidecar_to_children']['edges'][:]:
if not _condition(sidecar['node']):
media['edge_sidecar_to_children']['edges'].remove(sidecar)
# Check that the nodelist is not depleted
if not media['edge_sidecar_to_children']['edges']:
continue
# Check that the file does not exist
# FIXME: not working well with sidecar
if new_only and destination.exists(self.namegen.file(media)):
break
# Put the medias in the queue
queue.put(media)
medias_queued += 1
if media_count is not None and medias_queued >= media_count:
break
return medias_queued
# WORKERS UTILS
def _init_workers(self,
pbar, # type: Union[ProgressBar, Iterable, None]
destination, # type: fs.base.FS
):
# type: (...) -> Tuple[List[InstaDownloader], Queue]
workers = [] # type: List[InstaDownloader]
queue = Queue() # type: Queue
for _ in six.moves.range(self.jobs):
worker = InstaDownloader(
queue=queue,
destination=destination,
namegen=self.namegen,
add_metadata=self.add_metadata,
dump_json=self.dump_json,
dump_only=self.dump_only,
pbar=pbar,
session=self.session)
worker.start()
workers.append(worker)
return workers, queue
def _poison_workers(self, workers, queue):
# type: (List[InstaDownloader], Queue) -> None
for worker in workers:
queue.put(None)
def _join_workers(self, workers, queue):
# type: (List[InstaDownloader], Queue) -> None
if any(w.is_alive() for w in workers):
for worker in workers:
worker.join()
def _shutdown_workers(self, workers):
# type: (List[InstaDownloader]) -> None
for worker in workers:
worker.terminate()
class ServiceCoordinator(object):
"""Service coordinator.
This class uses the `tooz` library to manage group membership.
To ensure that the other agents know this agent is still alive,
the `heartbeat` method should be called periodically.
"""
def __init__(self, my_id=None):
self._coordinator = None
self._my_id = six.b(my_id or utils.get_process_identifier())
self._started = False
def start(self):
backend_url = cfg.CONF.coordination.backend_url
if backend_url:
try:
self._coordinator = tooz.coordination.get_coordinator(
backend_url, self._my_id)
self._coordinator.start(start_heart=True)
self._started = True
LOG.info('Coordination backend started successfully.')
except tooz.coordination.ToozError as e:
self._started = False
LOG.exception(
'Error connecting to coordination backend. '
'%s', six.text_type(e))
def stop(self):
if not self.is_active():
return
try:
self._coordinator.stop()
except tooz.coordination.ToozError:
LOG.warning('Error connecting to coordination backend.')
finally:
self._coordinator = None
self._started = False
def is_active(self):
return self._coordinator and self._started
@tenacity.retry(stop=tenacity.stop_after_attempt(5))
def join_group(self, group_id):
if not self.is_active() or not group_id:
return
try:
join_req = self._coordinator.join_group(six.b(group_id))
join_req.get()
LOG.info('Joined service group:%s, member:%s', group_id,
self._my_id)
return
except tooz.coordination.MemberAlreadyExist:
return
except tooz.coordination.GroupNotCreated as e:
create_grp_req = self._coordinator.create_group(six.b(group_id))
try:
create_grp_req.get()
except tooz.coordination.GroupAlreadyExist:
pass
# Re-raise exception to join group again.
raise e
def leave_group(self, group_id):
if self.is_active():
self._coordinator.leave_group(six.b(group_id))
LOG.info('Left service group:%s, member:%s', group_id, self._my_id)
def get_members(self, group_id):
"""Gets members of coordination group.
ToozError exception must be handled when this function is invoded, we
leave it to the invoker for the handling decision.
"""
if not self.is_active():
return []
get_members_req = self._coordinator.get_members(six.b(group_id))
try:
members = get_members_req.get()
LOG.debug('Members of group %s: %s', group_id, members)
return members
except tooz.coordination.GroupNotCreated:
LOG.warning('Group %s does not exist.', group_id)
return []
before_sleep_log,
retry,
retry_if_exception_type,
stop_after_attempt,
wait_fixed,
)
from ESSArch_Core.fixity.checksum import calculate_checksum
MB = 1024 * 1024
DEFAULT_BLOCK_SIZE = 10 * MB
logger = logging.getLogger('essarch.storage.copy')
@retry(retry=retry_if_exception_type(RequestException), reraise=True, stop=stop_after_attempt(5),
wait=wait_fixed(60), before_sleep=before_sleep_log(logger, logging.DEBUG))
def copy_chunk_remotely(src, dst, offset, file_size, requests_session, upload_id=None, block_size=DEFAULT_BLOCK_SIZE):
filename = os.path.basename(src)
with open(src, 'rb') as srcf:
srcf.seek(offset)
chunk = srcf.read(block_size)
start = offset
end = offset + block_size - 1
if end > file_size:
end = file_size - 1
HTTP_CONTENT_RANGE = 'bytes %s-%s/%s' % (start, end, file_size)
