def validate(self, max_retries=0):
"""Performs basic **connection** validation of a sqlalchemy engine."""
def _retry_on_exception(exc):
LOG.warning("Engine connection (validate) failed due to '%s'", exc)
if isinstance(exc, sa_exc.OperationalError) and \
_is_db_connection_error(six.text_type(exc.args[0])):
# We may be able to fix this by retrying...
return True
if isinstance(exc, (sa_exc.TimeoutError,
sa_exc.ResourceClosedError,
sa_exc.DisconnectionError)):
# We may be able to fix this by retrying...
return True
# Other failures we likely can't fix by retrying...
return False
@tenacity.retry(
stop=tenacity.stop_after_attempt(max(0, int(max_retries))),
wait=tenacity.wait_exponential(),
reraise=True,
retry=tenacity.retry_if_exception(_retry_on_exception)
)
def _try_connect(engine):
# See if we can make a connection happen.
#
# NOTE(harlowja): note that even though we are connecting
# once it does not mean that we will be able to connect in
# the future, so this is more of a sanity test and is not
# complete connection insurance.
with contextlib.closing(engine.connect()):
pass
_try_connect(self._engine)
def retry_upon_exception(exc, delay, max_delay, max_attempts):
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_exception_type(exc),
wait=tenacity.wait_exponential(
multiplier=delay, max=max_delay),
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
def join_group(self, group_id):
if (not self._coordinator or not self._coordinator.is_started
or not group_id):
return
@tenacity.retry(
wait=tenacity.wait_exponential(
multiplier=self.conf.coordination.retry_backoff,
max=self.conf.coordination.max_retry_interval),
retry=tenacity.retry_if_exception_type(
ErrorJoiningPartitioningGroup))
def _inner():
try:
join_req = self._coordinator.join_group(group_id)
join_req.get()
LOG.info(_LI('Joined partitioning group %s'), group_id)
except tooz.coordination.MemberAlreadyExist:
return
except tooz.coordination.GroupNotCreated:
create_grp_req = self._coordinator.create_group(group_id)
try:
create_grp_req.get()
except tooz.coordination.GroupAlreadyExist:
pass
raise ErrorJoiningPartitioningGroup()
except tooz.coordination.ToozError:
LOG.exception(_LE('Error joining partitioning group %s,'
' re-trying'), group_id)
raise ErrorJoiningPartitioningGroup()
self._groups.add(group_id)
return _inner()
def wrapped(*args, **kwargs):
self = args[0]
new_fn = tenacity.retry(
reraise=True,
retry=tenacity.retry_if_result(_ofport_result_pending),
wait=tenacity.wait_exponential(multiplier=0.01, max=1),
stop=tenacity.stop_after_delay(
self.vsctl_timeout))(fn)
return new_fn(*args, **kwargs)
def get_ovn_idls(driver, trigger):
@tenacity.retry(
wait=tenacity.wait_exponential(max=180),
reraise=True)
def get_ovn_idl_retry(cls, driver, trigger):
LOG.info(_LI('Getting %(cls)s for %(trigger)s with retry'),
{'cls': cls.__name__, 'trigger': trigger.im_class.__name__})
return cls(driver, trigger)
nb_ovn_idl = get_ovn_idl_retry(OvsdbNbOvnIdl, driver, trigger)
sb_ovn_idl = get_ovn_idl_retry(OvsdbSbOvnIdl, driver, trigger)
return nb_ovn_idl, sb_ovn_idl
def retry_upon_none_result(max_attempts, delay=0.5, max_delay=2, random=False):
if random:
wait_func = tenacity.wait_exponential(
multiplier=delay, max=max_delay)
else:
wait_func = tenacity.wait_random_exponential(
multiplier=delay, max=max_delay)
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_result(lambda x: x is None),
wait=wait_func,
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
def get_schema_helper(connection, schema_name, retry=True):
try:
return _get_schema_helper(connection, schema_name)
except Exception:
with excutils.save_and_reraise_exception(reraise=False) as ctx:
if not retry:
ctx.reraise = True
# We may have failed due to set-manager not being called
helpers.enable_connection_uri(connection)
# There is a small window for a race, so retry up to a second
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.01),
stop=tenacity.stop_after_delay(1),
reraise=True)
def do_get_schema_helper():
return _get_schema_helper(connection, schema_name)
return do_get_schema_helper()
def get_schema_helper(self):
"""Retrieve the schema helper object from OVSDB"""
# The implementation of this function is same as the base class method
# without the enable_connection_uri() called (since ovs-vsctl won't
# exist on the controller node when using the reference architecture).
try:
helper = idlutils.get_schema_helper(self.connection,
self.schema_name)
except Exception:
# There is a small window for a race, so retry up to a second
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.01),
stop=tenacity.stop_after_delay(1),
reraise=True)
def do_get_schema_helper():
return idlutils.get_schema_helper(self.connection,
self.schema_name)
helper = do_get_schema_helper()
return helper
def idl_factory():
conn = cfg.CONF.OVS.ovsdb_connection
schema_name = 'Open_vSwitch'
try:
helper = idlutils.get_schema_helper(conn, schema_name)
except Exception:
helpers.enable_connection_uri(conn)
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.01),
stop=tenacity.stop_after_delay(1),
reraise=True)
def do_get_schema_helper():
return idlutils.get_schema_helper(conn, schema_name)
helper = do_get_schema_helper()
# TODO(twilson) We should still select only the tables/columns we use
helper.register_all()
return idl.Idl(conn, helper)
def get_schema_helper(self):
"""Retrieve the schema helper object from OVSDB"""
try:
helper = idlutils.get_schema_helper(self.connection,
self.schema_name)
except Exception:
# We may have failed do to set-manager not being called
helpers.enable_connection_uri(self.connection)
# There is a small window for a race, so retry up to a second
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.01),
stop=tenacity.stop_after_delay(1),
reraise=True)
def do_get_schema_helper():
return idlutils.get_schema_helper(self.connection,
self.schema_name)
helper = do_get_schema_helper()
return helper
def retry_upon_exception_exclude_error_codes(
exc, excluded_errors, delay, max_delay, max_attempts):
"""Retry with the configured exponential delay, unless the exception error
code is in the given list
"""
def retry_if_not_error_codes(e):
# return True only for BadRequests without error codes or with error
# codes not in the exclude list
if isinstance(e, exc):
error_code = _get_bad_request_error_code(e)
if error_code and error_code not in excluded_errors:
return True
return False
return tenacity.retry(reraise=True,
retry=tenacity.retry_if_exception(
retry_if_not_error_codes),
wait=tenacity.wait_exponential(
multiplier=delay, max=max_delay),
stop=tenacity.stop_after_attempt(max_attempts),
before=_log_before_retry, after=_log_after_retry)
class GitRepo(object):
"""
Class for work with repositories
"""
def __init__(self, root_repo_dir, repo_name, branch, url, commit_id=None):
"""
:param root_repo_dir: Directory where repositories will clone
:param repo_name: Name of repository
:param branch: Branch of repository
:param commit_id: Commit ID
"""
self.repo_name = repo_name
self.branch_name = branch
self.url = url
self.commit_id = commit_id
self.local_repo_dir = root_repo_dir / repo_name
self.repo = None
self.log = logging.getLogger(self.__class__.__name__)
def prepare_repo(self):
"""
Preparing repository for build
Include cloning and updating repo to remote state
:return: None
"""
self.log.info('-' * 50)
self.log.info("Getting repo " + self.repo_name)
self.clone()
self.repo = git.Repo(str(self.local_repo_dir))
self.hard_reset()
self.clean()
self.checkout(branch_name="master", silent=True)
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def clone(self):
"""
Clone repo
:return: None
"""
# checking correctness of git repository
# if dir is not repository, it will be removed
if self.local_repo_dir.exists():
try:
git.Repo(str(self.local_repo_dir))
except git.InvalidGitRepositoryError:
self.log.info('Remove broken repo %s', self.local_repo_dir)
remove_directory(self.local_repo_dir)
if not self.local_repo_dir.exists():
self.log.info("Clone repo " + self.repo_name)
git.Git().clone(self.url, str(self.local_repo_dir))
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def fetch(self, branch_name=None):
"""
Fetch repo
:return: None
"""
refname = branch_name or self.branch_name
self.log.info("Fetch repo %s to %s", self.repo_name, refname)
self.repo.remotes.origin.fetch(refname)
self.hard_reset('FETCH_HEAD')
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def hard_reset(self, reset_to=None):
"""
Hard reset repo
:param reset_to: Commit ID or branch. If None - hard reset to HEAD
:return: None
"""
self.log.info("Hard reset repo " + self.repo_name)
if reset_to:
self.repo.git.reset('--hard', reset_to)
else:
self.repo.git.reset('--hard')
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def checkout(self, branch_name=None, silent=False):
"""
Checkout to certain state
:param branch_name: Branch of repo.
If None - checkout to commit ID from class variable commit_id
:param silent: Flag for getting time of commit
(set to True only if commit_id does not exist)
:type silent: Boolean
:return: None
"""
checkout_dest = branch_name or self.commit_id
self.log.info("Checkout repo %s to %s", self.repo_name, checkout_dest)
try:
self.repo.git.checkout(checkout_dest, force=True)
except git.exc.GitCommandError:
self.log.exception("Remote branch %s does not exist", checkout_dest)
if str(self.commit_id).lower() == 'head':
self.commit_id = str(self.repo.head.commit)
if not silent:
# error raises after checkout to master if we try
# to get time of triggered commit_id before fetching repo
# (commit does not exist in local repository yet)
committed_date = self.repo.commit(checkout_dest).committed_date
self.log.info("Committed date: %s", datetime.fromtimestamp(committed_date))
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def clean(self):
"""
Clean repo
:return: None
"""
self.log.info("Clean repo " + self.repo_name)
self.repo.git.clean('-xdf')
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
def pull(self):
"""
Pull repo
:return: None
"""
self.log.info("Pull repo " + self.repo_name)
self.repo.git.pull()
def change_repo_state(self, branch_name=None, commit_time=None):
"""
Change the repo state
:param branch_name: name of branch to checkout
:param commit_time: time of commit
:return: None
"""
self.branch_name = branch_name or self.branch_name
self.fetch()
if branch_name:
# Checkout to branch
self.checkout(branch_name=self.branch_name)
if commit_time:
self.revert_commit_by_time(commit_time)
# Checkout to commit id
self.checkout()
def revert_commit_by_time(self, commit_time):
"""
Sets commit by time.
If commit date <= certain time,
commit sets to class variable commit_id.
:param commit_time: timestamp
:return: None
"""
self.commit_id = str(next(self.repo.iter_commits(
until=commit_time, max_count=1)))
self.log.info(f"Revert commit by time to: {datetime.fromtimestamp(commit_time)}")
def get_time(self, commit_id=None):
"""
Get datetime of commit
:param commit_id: Commit ID
:return: datetime
"""
commit = commit_id if commit_id else self.commit_id
return self.repo.commit(commit).committed_date
def is_branch_exist(self, branch_name):
"""
Check if branch exists in repo
:param branch_name: branch name
:return: True if branch exists else false
"""
# Need to fetch all to get remote branches
self.repo.remotes.origin.fetch()
if self.repo.git.branch('--list', f'*/{branch_name}', '--all'):
return True
return False
for rule in current_rules:
if '-i %s' % vif in rule and '--among-src' in rule:
ebtables(['-D', chain] + rule.split())
def _delete_mac_spoofing_protection(vifs, current_rules, table, chain):
# delete the jump rule and then delete the whole chain
jumps = [vif for vif in vifs if _mac_vif_jump_present(vif, current_rules)]
for vif in jumps:
ebtables(['-D', chain, '-i', vif, '-j',
_mac_chain_name(vif)],
table=table)
for vif in vifs:
chain = _mac_chain_name(vif)
if chain_exists(chain, current_rules):
ebtables(['-X', chain], table=table)
# Used to scope ebtables commands in testing
NAMESPACE = None
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=0.01),
retry=tenacity.retry_if_exception(lambda e: e.returncode == 255),
reraise=True)
def ebtables(comm, table='nat'):
execute = ip_lib.IPWrapper(NAMESPACE).netns.execute
return execute(['ebtables', '-t', table, '--concurrent'] + comm,
run_as_root=True)
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Code for configuring swift."""
import logging
import tenacity
import zaza.openstack.utilities.openstack as openstack
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=10, max=300),
reraise=True, stop=tenacity.stop_after_attempt(10),
retry=tenacity.retry_if_exception_type(AssertionError))
def wait_for_region2():
"""Ensure two regions are present."""
keystone_session = openstack.get_overcloud_keystone_session()
keystone_client = (
openstack.get_keystone_session_client(
keystone_session,
client_api_version='3'))
swift_svc_id = keystone_client.services.find(name='swift').id
regions = set([ep.region
for ep in keystone_client.endpoints.list(swift_svc_id)])
logging.info('Checking there are 2 regions. Current count is {}'.format(
len(regions)))
assert len(set(regions)) == 2, "Incorrect number of regions"
table.add_row("kubectl", self.kubectl_version)
table.add_row("gcloud", self.gcloud_version)
table.add_row("cloud_sql_proxy", self.cloud_sql_proxy_version)
table.add_row("mysql", self.mysql_version)
table.add_row("sqlite3", self.sqlite3_version)
table.add_row("psql", self.psql_version)
console.print(table)
class FileIoException(Exception):
"""Raises when error happens in FileIo.io integration"""
@tenacity.retry(
stop=tenacity.stop_after_attempt(5),
wait=tenacity.wait_exponential(multiplier=1, max=10),
retry=tenacity.retry_if_exception_type(FileIoException),
before=tenacity.before_log(log, logging.DEBUG),
after=tenacity.after_log(log, logging.DEBUG),
)
def _upload_text_to_fileio(content):
"""Upload text file to File.io service and return lnk"""
resp = requests.post("https://file.io", data={"text": content})
if not resp.ok:
print(resp.json())
raise FileIoException("Failed to send report to file.io service.")
try:
return resp.json()["link"]
except ValueError as e:
log.debug(e)
raise FileIoException("Failed to send report to file.io service.")
class NovaClientPlugin(microversion_mixin.MicroversionMixin,
client_plugin.ClientPlugin):
deferred_server_statuses = [
'BUILD', 'HARD_REBOOT', 'PASSWORD', 'REBOOT', 'RESCUE', 'RESIZE',
'REVERT_RESIZE', 'SHUTOFF', 'SUSPENDED', 'VERIFY_RESIZE'
]
exceptions_module = exceptions
NOVA_API_VERSION = '2.1'
max_microversion = cfg.CONF.max_nova_api_microversion
service_types = [COMPUTE] = ['compute']
def _get_service_name(self):
return self.COMPUTE
def _create(self, version=None):
if not version:
# TODO(prazumovsky): remove all unexpected calls from tests and
# add default_version after that.
version = self.NOVA_API_VERSION
args = self._get_args(version)
client = nc.Client(version, **args)
return client
def _get_args(self, version):
endpoint_type = self._get_client_option(CLIENT_NAME, 'endpoint_type')
return {
'session':
self.context.keystone_session,
'endpoint_type':
endpoint_type,
'service_type':
self.COMPUTE,
'region_name':
self._get_region_name(),
'connect_retries':
cfg.CONF.client_retry_limit,
'http_log_debug':
self._get_client_option(CLIENT_NAME, 'http_log_debug')
}
def get_max_microversion(self):
if not self.max_microversion:
client = self._create()
self.max_microversion = client.versions.get_current().version
return self.max_microversion
def is_version_supported(self, version):
api_ver = api_versions.get_api_version(version)
max_api_ver = api_versions.get_api_version(self.get_max_microversion())
return max_api_ver >= api_ver
def is_not_found(self, ex):
return isinstance(ex, (exceptions.NotFound, q_exceptions.NotFound))
def is_over_limit(self, ex):
return isinstance(ex, exceptions.OverLimit)
def is_bad_request(self, ex):
return isinstance(ex, exceptions.BadRequest)
def is_conflict(self, ex):
return isinstance(ex, exceptions.Conflict)
def is_unprocessable_entity(self, ex):
http_status = (getattr(ex, 'http_status', None)
or getattr(ex, 'code', None))
return (isinstance(ex, exceptions.ClientException)
and http_status == 422)
@tenacity.retry(stop=tenacity.stop_after_attempt(
max(cfg.CONF.client_retry_limit + 1, 0)),
retry=tenacity.retry_if_exception(
client_plugin.retry_if_connection_err),
reraise=True)
def get_server(self, server):
"""Return fresh server object.
Substitutes Nova's NotFound for Heat's EntityNotFound,
to be returned to user as HTTP error.
"""
try:
return self.client().servers.get(server)
except exceptions.NotFound:
raise exception.EntityNotFound(entity='Server', name=server)
def fetch_server(self, server_id):
"""Fetch fresh server object from Nova.
Log warnings and return None for non-critical API errors.
Use this method in various ``check_*_complete`` resource methods,
where intermittent errors can be tolerated.
"""
server = None
try:
server = self.client().servers.get(server_id)
except exceptions.OverLimit as exc:
LOG.warning(
"Received an OverLimit response when "
"fetching server (%(id)s) : %(exception)s", {
'id': server_id,
'exception': exc
})
except exceptions.ClientException as exc:
if ((getattr(exc, 'http_status', getattr(exc, 'code', None))
in (500, 503))):
LOG.warning(
"Received the following exception when "
"fetching server (%(id)s) : %(exception)s", {
'id': server_id,
'exception': exc
})
else:
raise
return server
def fetch_server_attr(self, server_id, attr):
server = self.fetch_server(server_id)
fetched_attr = getattr(server, attr, None)
return fetched_attr
def refresh_server(self, server):
"""Refresh server's attributes.
Also log warnings for non-critical API errors.
"""
try:
server.get()
except exceptions.OverLimit as exc:
LOG.warning(
"Server %(name)s (%(id)s) received an OverLimit "
"response during server.get(): %(exception)s", {
'name': server.name,
'id': server.id,
'exception': exc
})
except exceptions.ClientException as exc:
if ((getattr(exc, 'http_status', getattr(exc, 'code', None))
in (500, 503))):
LOG.warning(
'Server "%(name)s" (%(id)s) received the '
'following exception during server.get(): '
'%(exception)s', {
'name': server.name,
'id': server.id,
'exception': exc
})
else:
raise
def get_ip(self, server, net_type, ip_version):
"""Return the server's IP of the given type and version."""
if net_type in server.addresses:
for ip in server.addresses[net_type]:
if ip['version'] == ip_version:
return ip['addr']
def get_status(self, server):
"""Return the server's status.
:param server: server object
:returns: status as a string
"""
# Some clouds append extra (STATUS) strings to the status, strip it
return server.status.split('(')[0]
def _check_active(self, server, res_name='Server'):
"""Check server status.
Accepts both server IDs and server objects.
Returns True if server is ACTIVE,
raises errors when server has an ERROR or unknown to Heat status,
returns False otherwise.
:param res_name: name of the resource to use in the exception message
"""
# not checking with is_uuid_like as most tests use strings e.g. '1234'
if isinstance(server, str):
server = self.fetch_server(server)
if server is None:
return False
else:
status = self.get_status(server)
else:
status = self.get_status(server)
if status != 'ACTIVE':
self.refresh_server(server)
status = self.get_status(server)
if status in self.deferred_server_statuses:
return False
elif status == 'ACTIVE':
return True
elif status == 'ERROR':
fault = getattr(server, 'fault', {})
raise exception.ResourceInError(
resource_status=status,
status_reason=_("Message: %(message)s, Code: %(code)s") % {
'message': fault.get('message', _('Unknown')),
'code': fault.get('code', _('Unknown'))
})
else:
raise exception.ResourceUnknownStatus(
resource_status=server.status,
result=_('%s is not active') % res_name)
def find_flavor_by_name_or_id(self, flavor):
"""Find the specified flavor by name or id.
:param flavor: the name of the flavor to find
:returns: the id of :flavor:
"""
return self._find_flavor_id(self.context.tenant_id, flavor)
@os_client.MEMOIZE_FINDER
def _find_flavor_id(self, tenant_id, flavor):
# tenant id in the signature is used for the memoization key,
# that would differentiate similar resource names across tenants.
return self.get_flavor(flavor).id
def get_flavor(self, flavor_identifier):
"""Get the flavor object for the specified flavor name or id.
:param flavor_identifier: the name or id of the flavor to find
:returns: a flavor object with name or id :flavor:
"""
try:
flavor = self.client().flavors.get(flavor_identifier)
except exceptions.NotFound:
flavor = self.client().flavors.find(name=flavor_identifier)
return flavor
def get_host(self, host_name):
"""Get the host id specified by name.
:param host_name: the name of host to find
:returns: the list of match hosts
:raises exception.EntityNotFound:
"""
host_list = self.client().hosts.list()
for host in host_list:
if host.host_name == host_name and host.service == self.COMPUTE:
return host
raise exception.EntityNotFound(entity='Host', name=host_name)
def get_keypair(self, key_name):
"""Get the public key specified by :key_name:
:param key_name: the name of the key to look for
:returns: the keypair (name, public_key) for :key_name:
:raises exception.EntityNotFound:
"""
try:
return self.client().keypairs.get(key_name)
except exceptions.NotFound:
raise exception.EntityNotFound(entity='Key', name=key_name)
def build_userdata(self,
metadata,
userdata=None,
instance_user=None,
user_data_format='HEAT_CFNTOOLS'):
"""Build multipart data blob for CloudInit and Ignition.
Data blob includes user-supplied Metadata, user data, and the required
Heat in-instance configuration.
:param resource: the resource implementation
:type resource: heat.engine.Resource
:param userdata: user data string
:type userdata: str or None
:param instance_user: the user to create on the server
:type instance_user: string
:param user_data_format: Format of user data to return
:type user_data_format: string
:returns: multipart mime as a string
"""
if user_data_format == 'RAW':
return userdata
is_cfntools = user_data_format == 'HEAT_CFNTOOLS'
is_software_config = user_data_format == 'SOFTWARE_CONFIG'
if (is_software_config
and NovaClientPlugin.is_ignition_format(userdata)):
return NovaClientPlugin.build_ignition_data(metadata, userdata)
def make_subpart(content, filename, subtype=None):
if subtype is None:
subtype = os.path.splitext(filename)[0]
if content is None:
content = ''
try:
content.encode('us-ascii')
charset = 'us-ascii'
except UnicodeEncodeError:
charset = 'utf-8'
msg = (text.MIMEText(content, _subtype=subtype, _charset=charset)
if subtype else text.MIMEText(content, _charset=charset))
msg.add_header('Content-Disposition',
'attachment',
filename=filename)
return msg
def read_cloudinit_file(fn):
return pkgutil.get_data('heat',
'cloudinit/%s' % fn).decode('utf-8')
if instance_user:
config_custom_user = '******' % instance_user
# FIXME(shadower): compatibility workaround for cloud-init 0.6.3.
# We can drop this once we stop supporting 0.6.3 (which ships
# with Ubuntu 12.04 LTS).
#
# See bug https://bugs.launchpad.net/heat/+bug/1257410
boothook_custom_user = r"""useradd -m %s
echo -e '%s\tALL=(ALL)\tNOPASSWD: ALL' >> /etc/sudoers
""" % (instance_user, instance_user)
else:
config_custom_user = ''
boothook_custom_user = ''
cloudinit_config = string.Template(
read_cloudinit_file('config')).safe_substitute(
add_custom_user=config_custom_user)
cloudinit_boothook = string.Template(
read_cloudinit_file('boothook.sh')).safe_substitute(
add_custom_user=boothook_custom_user)
attachments = [(cloudinit_config, 'cloud-config'),
(cloudinit_boothook, 'boothook.sh', 'cloud-boothook'),
(read_cloudinit_file('part_handler.py'),
'part-handler.py')]
if is_cfntools:
attachments.append((userdata, 'cfn-userdata', 'x-cfninitdata'))
elif is_software_config:
# attempt to parse userdata as a multipart message, and if it
# is, add each part as an attachment
userdata_parts = None
try:
userdata_parts = email.message_from_string(userdata)
except Exception:
pass
if userdata_parts and userdata_parts.is_multipart():
for part in userdata_parts.get_payload():
attachments.append(
(part.get_payload(), part.get_filename(),
part.get_content_subtype()))
else:
attachments.append((userdata, ''))
if is_cfntools:
attachments.append((read_cloudinit_file('loguserdata.py'),
'loguserdata.py', 'x-shellscript'))
if metadata:
attachments.append(
(jsonutils.dumps(metadata), 'cfn-init-data', 'x-cfninitdata'))
if is_cfntools:
heat_client_plugin = self.context.clients.client_plugin('heat')
cfn_md_url = heat_client_plugin.get_cfn_metadata_server_url()
attachments.append(
(cfn_md_url, 'cfn-metadata-server', 'x-cfninitdata'))
# Create a boto config which the cfntools on the host use to know
# where the cfn API is to be accessed
cfn_url = urlparse.urlparse(cfn_md_url)
is_secure = cfg.CONF.instance_connection_is_secure
vcerts = cfg.CONF.instance_connection_https_validate_certificates
boto_cfg = "\n".join([
"[Boto]", "debug = 0",
"is_secure = %s" % is_secure,
"https_validate_certificates = %s" % vcerts,
"cfn_region_name = heat",
"cfn_region_endpoint = %s" % cfn_url.hostname
])
attachments.append((boto_cfg, 'cfn-boto-cfg', 'x-cfninitdata'))
subparts = [make_subpart(*args) for args in attachments]
mime_blob = multipart.MIMEMultipart(_subparts=subparts)
return mime_blob.as_string()
@staticmethod
def is_ignition_format(userdata):
try:
payload = jsonutils.loads(userdata)
ig = payload.get("ignition")
return True if ig and ig.get("version") else False
except Exception:
return False
@staticmethod
def build_ignition_data(metadata, userdata):
if not metadata:
return userdata
payload = jsonutils.loads(userdata)
encoded_metadata = urlparse.quote(jsonutils.dumps(metadata))
path_list = [
"/var/lib/heat-cfntools/cfn-init-data",
"/var/lib/cloud/data/cfn-init-data"
]
ignition_format_metadata = {
"filesystem": "root",
"group": {
"name": "root"
},
"path": "",
"user": {
"name": "root"
},
"contents": {
"source": "data:," + encoded_metadata,
"verification": {}
},
"mode": 0o640
}
for path in path_list:
storage = payload.setdefault('storage', {})
try:
files = storage.setdefault('files', [])
except AttributeError:
raise ValueError('Ignition "storage" section must be a map')
else:
try:
data = ignition_format_metadata.copy()
data["path"] = path
files.append(data)
except AttributeError:
raise ValueError('Ignition "files" section must be a list')
return jsonutils.dumps(payload)
def check_delete_server_complete(self, server_id):
"""Wait for server to disappear from Nova."""
try:
server = self.fetch_server(server_id)
except Exception as exc:
self.ignore_not_found(exc)
return True
if not server:
return False
task_state_in_nova = getattr(server, 'OS-EXT-STS:task_state', None)
# the status of server won't change until the delete task has done
if task_state_in_nova == 'deleting':
return False
status = self.get_status(server)
if status == 'DELETED':
return True
if status == 'SOFT_DELETED':
self.client().servers.force_delete(server_id)
elif status == 'ERROR':
fault = getattr(server, 'fault', {})
message = fault.get('message', 'Unknown')
code = fault.get('code')
errmsg = _("Server %(name)s delete failed: (%(code)s) "
"%(message)s") % dict(
name=server.name, code=code, message=message)
raise exception.ResourceInError(resource_status=status,
status_reason=errmsg)
return False
def rename(self, server, name):
"""Update the name for a server."""
server.update(name)
def resize(self, server_id, flavor_id):
"""Resize the server."""
server = self.fetch_server(server_id)
if server:
server.resize(flavor_id)
return True
else:
return False
def check_resize(self, server_id, flavor):
"""Verify that a resizing server is properly resized.
If that's the case, confirm the resize, if not raise an error.
"""
server = self.fetch_server(server_id)
# resize operation is asynchronous so the server resize may not start
# when checking server status (the server may stay ACTIVE instead
# of RESIZE).
if not server or server.status in ('RESIZE', 'ACTIVE'):
return False
if server.status == 'VERIFY_RESIZE':
return True
else:
raise exception.Error(
_("Resizing to '%(flavor)s' failed, status '%(status)s'") %
dict(flavor=flavor, status=server.status))
def verify_resize(self, server_id):
server = self.fetch_server(server_id)
if not server:
return False
status = self.get_status(server)
if status == 'VERIFY_RESIZE':
server.confirm_resize()
return True
else:
msg = _("Could not confirm resize of server %s") % server_id
raise exception.ResourceUnknownStatus(result=msg,
resource_status=status)
def check_verify_resize(self, server_id):
server = self.fetch_server(server_id)
if not server:
return False
status = self.get_status(server)
if status == 'ACTIVE':
return True
if status == 'VERIFY_RESIZE':
return False
task_state_in_nova = getattr(server, 'OS-EXT-STS:task_state', None)
# Wait till move out from any resize steps (including resize_finish).
if task_state_in_nova is not None and 'resize' in task_state_in_nova:
return False
else:
msg = _("Confirm resize for server %s failed") % server_id
raise exception.ResourceUnknownStatus(result=msg,
resource_status=status)
def rebuild(self,
server_id,
image_id,
password=None,
preserve_ephemeral=False,
meta=None,
files=None):
"""Rebuild the server and call check_rebuild to verify."""
server = self.fetch_server(server_id)
if server:
server.rebuild(image_id,
password=password,
preserve_ephemeral=preserve_ephemeral,
meta=meta,
files=files)
return True
else:
return False
def check_rebuild(self, server_id):
"""Verify that a rebuilding server is rebuilt.
Raise error if it ends up in an ERROR state.
"""
server = self.fetch_server(server_id)
if server is None or server.status == 'REBUILD':
return False
if server.status == 'ERROR':
raise exception.Error(
_("Rebuilding server failed, status '%s'") % server.status)
else:
return True
def meta_serialize(self, metadata):
"""Serialize non-string metadata values before sending them to Nova."""
if not isinstance(metadata, collections.Mapping):
raise exception.StackValidationFailed(
message=_("nova server metadata needs to be a Map."))
return dict(
(key,
(value if isinstance(value, str) else jsonutils.dumps(value)))
for (key, value) in metadata.items())
def meta_update(self, server, metadata):
"""Delete/Add the metadata in nova as needed."""
metadata = self.meta_serialize(metadata)
current_md = server.metadata
to_del = sorted(set(current_md) - set(metadata))
client = self.client()
if len(to_del) > 0:
client.servers.delete_meta(server, to_del)
client.servers.set_meta(server, metadata)
def server_to_ipaddress(self, server):
"""Return the server's IP address, fetching it from Nova."""
try:
server = self.client().servers.get(server)
except exceptions.NotFound as ex:
LOG.warning('Instance (%(server)s) not found: %(ex)s', {
'server': server,
'ex': ex
})
else:
for n in sorted(server.networks, reverse=True):
if len(server.networks[n]) > 0:
return server.networks[n][0]
@tenacity.retry(stop=tenacity.stop_after_attempt(
max(cfg.CONF.client_retry_limit + 1, 0)),
retry=tenacity.retry_if_exception(
client_plugin.retry_if_connection_err),
reraise=True)
def absolute_limits(self):
"""Return the absolute limits as a dictionary."""
limits = self.client().limits.get()
return dict([(limit.name, limit.value)
for limit in list(limits.absolute)])
def get_console_urls(self, server):
"""Return dict-like structure of server's console urls.
The actual console url is lazily resolved on access.
"""
nc = self.client
class ConsoleUrls(collections.Mapping):
def __init__(self, server):
self.console_method = server.get_console_url
self.support_console_types = [
'novnc', 'xvpvnc', 'spice-html5', 'rdp-html5', 'serial',
'webmks'
]
def __getitem__(self, key):
try:
if key not in self.support_console_types:
raise exceptions.UnsupportedConsoleType(key)
if key == 'webmks':
data = nc().servers.get_console_url(server, key)
else:
data = self.console_method(key)
console_data = data.get('remote_console',
data.get('console'))
url = console_data['url']
except exceptions.UnsupportedConsoleType as ex:
url = ex.message
except Exception as e:
url = _('Cannot get console url: %s') % str(e)
return url
def __len__(self):
return len(self.support_console_types)
def __iter__(self):
return (key for key in self.support_console_types)
return ConsoleUrls(server)
def attach_volume(self, server_id, volume_id, device):
try:
va = self.client().volumes.create_server_volume(
server_id=server_id, volume_id=volume_id, device=device)
except Exception as ex:
if self.is_client_exception(ex):
raise exception.Error(
_("Failed to attach volume %(vol)s to server %(srv)s "
"- %(err)s") % {
'vol': volume_id,
'srv': server_id,
'err': ex
})
else:
raise
return va.id
def detach_volume(self, server_id, attach_id):
# detach the volume using volume_attachment
try:
self.client().volumes.delete_server_volume(server_id, attach_id)
except Exception as ex:
if not (self.is_not_found(ex) or self.is_bad_request(ex)):
raise exception.Error(
_("Could not detach attachment %(att)s "
"from server %(srv)s.") % {
'srv': server_id,
'att': attach_id
})
def check_detach_volume_complete(self, server_id, attach_id):
"""Check that nova server lost attachment.
This check is needed for immediate reattachment when updating:
there might be some time between cinder marking volume as 'available'
and nova removing attachment from its own objects, so we
check that nova already knows that the volume is detached.
"""
try:
self.client().volumes.get_server_volume(server_id, attach_id)
except Exception as ex:
self.ignore_not_found(ex)
LOG.info("Volume %(vol)s is detached from server %(srv)s", {
'vol': attach_id,
'srv': server_id
})
return True
else:
LOG.debug("Server %(srv)s still has attachment %(att)s.", {
'att': attach_id,
'srv': server_id
})
return False
def associate_floatingip(self, server_id, floatingip_id):
iface_list = self.fetch_server(server_id).interface_list()
if len(iface_list) == 0:
raise client_exception.InterfaceNotFound(id=server_id)
if len(iface_list) > 1:
LOG.warning(
"Multiple interfaces found for server %s, "
"using the first one.", server_id)
port_id = iface_list[0].port_id
fixed_ips = iface_list[0].fixed_ips
fixed_address = next(ip['ip_address'] for ip in fixed_ips
if netutils.is_valid_ipv4(ip['ip_address']))
request_body = {
'floatingip': {
'port_id': port_id,
'fixed_ip_address': fixed_address
}
}
self.clients.client('neutron').update_floatingip(
floatingip_id, request_body)
def dissociate_floatingip(self, floatingip_id):
request_body = {
'floatingip': {
'port_id': None,
'fixed_ip_address': None
}
}
self.clients.client('neutron').update_floatingip(
floatingip_id, request_body)
def associate_floatingip_address(self, server_id, fip_address):
fips = self.clients.client('neutron').list_floatingips(
floating_ip_address=fip_address)['floatingips']
if len(fips) == 0:
args = {'ip_address': fip_address}
raise client_exception.EntityMatchNotFound(entity='floatingip',
args=args)
self.associate_floatingip(server_id, fips[0]['id'])
def dissociate_floatingip_address(self, fip_address):
fips = self.clients.client('neutron').list_floatingips(
floating_ip_address=fip_address)['floatingips']
if len(fips) == 0:
args = {'ip_address': fip_address}
raise client_exception.EntityMatchNotFound(entity='floatingip',
args=args)
self.dissociate_floatingip(fips[0]['id'])
def interface_detach(self, server_id, port_id):
with self.ignore_not_found:
server = self.fetch_server(server_id)
if server:
server.interface_detach(port_id)
return True
def interface_attach(self,
server_id,
port_id=None,
net_id=None,
fip=None,
security_groups=None):
server = self.fetch_server(server_id)
if server:
attachment = server.interface_attach(port_id, net_id, fip)
if not port_id and security_groups:
props = {'security_groups': security_groups}
self.clients.client('neutron').update_port(
attachment.port_id, {'port': props})
return True
else:
return False
@tenacity.retry(stop=tenacity.stop_after_attempt(
cfg.CONF.max_interface_check_attempts),
wait=tenacity.wait_exponential(multiplier=0.5, max=12.0),
retry=tenacity.retry_if_result(
client_plugin.retry_if_result_is_false))
def check_interface_detach(self, server_id, port_id):
with self.ignore_not_found:
server = self.fetch_server(server_id)
if server:
interfaces = server.interface_list()
for iface in interfaces:
if iface.port_id == port_id:
return False
return True
@tenacity.retry(stop=tenacity.stop_after_attempt(
cfg.CONF.max_interface_check_attempts),
wait=tenacity.wait_fixed(0.5),
retry=tenacity.retry_if_result(
client_plugin.retry_if_result_is_false))
def check_interface_attach(self, server_id, port_id):
if not port_id:
return True
server = self.fetch_server(server_id)
if server:
interfaces = server.interface_list()
for iface in interfaces:
if iface.port_id == port_id:
return True
return False
class MergifyPull(object):
# NOTE(sileht): Use from_cache/from_event not the constructor directly
g_pull = attr.ib()
installation_id = attr.ib()
_complete = attr.ib(init=False, default=False)
_reviews_required = attr.ib(init=False, default=None)
# Cached attributes
_reviews_ok = attr.ib(init=False, default=None)
_reviews_ko = attr.ib(init=False, default=None)
_required_statuses = attr.ib(
init=False,
default=None,
validator=attr.validators.optional(
attr.validators.instance_of(StatusState)),
)
_mergify_state = attr.ib(
init=False,
default=None,
validator=attr.validators.optional(
attr.validators.instance_of(MergifyState)),
)
_github_state = attr.ib(init=False, default=None)
_github_description = attr.ib(init=False, default=None)
@classmethod
def from_raw(cls, installation_id, installation_token, pull_raw):
g = github.Github(installation_token)
pull = github.PullRequest.PullRequest(g._Github__requester, {},
pull_raw,
completed=True)
return cls(pull, installation_id)
def __attrs_post_init__(self):
self.log = daiquiri.getLogger(__name__, pull_request=self)
self._ensure_mergable_state()
def _ensure_complete(self):
if not self._complete: # pragma: no cover
raise RuntimeError("%s: used an incomplete MergifyPull")
@property
def status(self):
# TODO(sileht): Should be removed at some point. When the cache
# will not have mergify_engine_status key anymore
return {
"mergify_state": self.mergify_state,
"github_state": self.github_state,
"github_description": self.github_description
}
@property
def mergify_state(self):
self._ensure_complete()
return self._mergify_state
@property
def github_state(self):
self._ensure_complete()
return self._github_state
@property
def github_description(self):
self._ensure_complete()
return self._github_description
def complete(self, cache, branch_rule, collaborators):
need_to_be_saved = False
protection = branch_rule["protection"]
if protection["required_pull_request_reviews"]:
self._reviews_required = protection[
"required_pull_request_reviews"][
"required_approving_review_count"]
else:
self._reviews_required = 0
if "mergify_engine_reviews_ok" in cache:
self._reviews_ok = cache["mergify_engine_reviews_ok"]
self._reviews_ko = cache["mergify_engine_reviews_ko"]
else:
need_to_be_saved = True
self._reviews_ok, self._reviews_ko = self._compute_approvals(
branch_rule, collaborators)
if "mergify_engine_required_statuses" in cache:
self._required_statuses = StatusState(
cache["mergify_engine_required_statuses"])
else:
need_to_be_saved = True
self._required_statuses = self._compute_required_statuses(
branch_rule)
if "mergify_engine_status" in cache:
s = cache["mergify_engine_status"]
self._mergify_state = MergifyState(s["mergify_state"])
self._github_state = s["github_state"]
self._github_description = s["github_description"]
else:
need_to_be_saved = True
(self._mergify_state, self._github_state,
self._github_description) = self._compute_status(
branch_rule, collaborators)
self._complete = True
return need_to_be_saved
def refresh(self, branch_rule, collaborators):
# NOTE(sileht): Redownload the PULL to ensure we don't have
# any etag floating around
self._ensure_mergable_state(force=True)
return self.complete({}, branch_rule, collaborators)
def jsonify(self):
raw = copy.copy(self.g_pull.raw_data)
raw["mergify_engine_reviews_ok"] = self._reviews_ok
raw["mergify_engine_reviews_ko"] = self._reviews_ko
raw["mergify_engine_required_statuses"] = self._required_statuses.value
raw["mergify_engine_status"] = {
"mergify_state": self._mergify_state.value,
"github_description": self._github_description,
"github_state": self._github_state
}
return raw
def _get_reviews(self):
# Ignore reviews that are not from someone with write permissions
# And only keep the last review for each user.
return list(
dict((review.user.login, review)
for review in self.g_pull.get_reviews()
if (review._rawData['author_association'] in (
"COLLABORATOR", "MEMBER", "OWNER"))).values())
def to_dict(self):
reviews = self._get_reviews()
statuses = self._get_checks()
# FIXME(jd) pygithub does 2 HTTP requests whereas 1 is enough!
review_requested_users, review_requested_teams = (
self.g_pull.get_review_requests())
return {
"assignee": [a.login for a in self.g_pull.assignees],
"label": [l.name for l in self.g_pull.labels],
"review-requested":
([u.login for u in review_requested_users] +
["@" + t.slug for t in review_requested_teams]),
"author":
self.g_pull.user.login,
"merged-by":
(self.g_pull.merged_by.login if self.g_pull.merged_by else ""),
"merged":
self.g_pull.merged,
"state":
self.g_pull.state,
"milestone":
(self.g_pull.milestone.title if self.g_pull.milestone else ""),
"base":
self.g_pull.base.ref,
"head":
self.g_pull.head.ref,
"locked":
self.g_pull._rawData['locked'],
"title":
self.g_pull.title,
"body":
self.g_pull.body,
"files": [f.filename for f in self.g_pull.get_files()],
"approved-reviews-by":
[r.user.login for r in reviews if r.state == "APPROVED"],
"dismissed-reviews-by":
[r.user for r in reviews if r.state == "DISMISSED"],
"changes-requested-reviews-by":
[r.user for r in reviews if r.state == "CHANGES_REQUESTED"],
"commented-reviews-by":
[r.user for r in reviews if r.state == "COMMENTED"],
"status-success":
[s.context for s in statuses if s.state == "success"],
# NOTE(jd) The Check API set conclusion to None for pending.
# NOTE(sileht): "pending" statuses are not really trackable, we
# voluntary drop this event because CIs just sent they status every
# minutes until the CI pass (at least Travis and Circle CI does
# that). This was causing a big load on Mergify for nothing useful
# tracked, and on big projects it can reach the rate limit very
# quickly.
# "status-pending": [s.context for s in statuses
# if s.state in ("pending", None)],
"status-failure":
[s.context for s in statuses if s.state == "failure"],
}
def _compute_approvals(self, branch_rule, collaborators):
"""Compute approvals.
:param branch_rule: The rule for the considered branch.
:param collaborators: The list of collaborators.
:return: A tuple (users_with_review_ok, users_with_review_ko)
"""
users_info = {}
reviews_ok = set()
reviews_ko = set()
for review in self.g_pull.get_reviews():
if review.user.id not in collaborators:
continue
users_info[review.user.login] = review.user.raw_data
if review.state == 'APPROVED':
reviews_ok.add(review.user.login)
if review.user.login in reviews_ko:
reviews_ko.remove(review.user.login)
elif review.state in ["DISMISSED", "CHANGES_REQUESTED"]:
if review.user.login in reviews_ok:
reviews_ok.remove(review.user.login)
if review.user.login in reviews_ko:
reviews_ko.remove(review.user.login)
if review.state == "CHANGES_REQUESTED":
reviews_ko.add(review.user.login)
elif review.state == 'COMMENTED':
pass
else:
self.log.error("review state unhandled", state=review.state)
return ([users_info[u]
for u in reviews_ok], [users_info[u] for u in reviews_ko])
@staticmethod
def _find_required_context(contexts, generic_check):
for c in contexts:
if generic_check.context.startswith(c):
return c
def _get_combined_status(self):
headers, data = self.g_pull.head.repo._requester.requestJsonAndCheck(
"GET",
self.g_pull.base.repo.url + "/commits/" + self.g_pull.head.sha +
"/status",
)
return github.CommitCombinedStatus.CommitCombinedStatus(
self.g_pull.head.repo._requester, headers, data, completed=True)
def _get_checks(self):
generic_checks = set()
try:
# NOTE(sileht): conclusion can be one of success, failure, neutral,
# cancelled, timed_out, or action_required, and None for "pending"
generic_checks |= set([
GenericCheck(c.name, c.conclusion)
for c in check_api.get_checks(self.g_pull)
])
except github.GithubException as e:
if (e.status != 403 or e.data["message"] !=
"Resource not accessible by integration"):
raise
# NOTE(sileht): state can be one of error, failure, pending,
# or success.
generic_checks |= set([
GenericCheck(s.context, s.state)
for s in self._get_combined_status().statuses
])
return generic_checks
def _compute_required_statuses(self, branch_rule):
# return True is CIs succeed, False is their fail, None
# is we don't known yet.
# FIXME(sileht): I don't use a Enum yet to not
protection = branch_rule["protection"]
if not protection["required_status_checks"]:
return StatusState.SUCCESS
# NOTE(sileht): Due to the difference of both API we use only success
# bellow.
contexts = set(protection["required_status_checks"]["contexts"])
seen_contexts = set()
for check in self._get_checks():
required_context = self._find_required_context(contexts, check)
if required_context:
seen_contexts.add(required_context)
if check.state in ["pending", None]:
return StatusState.PENDING
elif check.state != "success":
return StatusState.FAILURE
if contexts - seen_contexts:
return StatusState.PENDING
else:
return StatusState.SUCCESS
def _disabled_by_rules(self, branch_rule):
labels = [l.name for l in self.g_pull.labels]
enabling_label = branch_rule["enabling_label"]
if enabling_label is not None and enabling_label not in labels:
return "Disabled — enabling label missing"
if branch_rule["disabling_label"] in labels:
return "Disabled — disabling label present"
g_pull_files = [f.filename for f in self.g_pull.get_files()]
for w in branch_rule["disabling_files"]:
filtered = fnmatch.filter(g_pull_files, w)
if filtered:
return ("Disabled — %s is modified" % filtered[0])
return None
def _compute_status(self, branch_rule, collaborators):
disabled = self._disabled_by_rules(branch_rule)
mergify_state = MergifyState.NOT_READY
github_state = "pending"
github_desc = None
if disabled:
github_state = "failure"
github_desc = disabled
elif self._reviews_ko:
github_desc = "Change requests need to be dismissed"
elif (self._reviews_required
and len(self._reviews_ok) < self._reviews_required):
github_desc = ("%d/%d approvals required" %
(len(self._reviews_ok), self._reviews_required))
elif self.g_pull.mergeable_state in ["clean", "unstable", "has_hooks"]:
mergify_state = MergifyState.READY
github_state = "success"
github_desc = "Will be merged soon"
elif self.g_pull.mergeable_state == "blocked":
if self._required_statuses == StatusState.SUCCESS:
# FIXME(sileht) We are blocked but reviews are OK and CI passes
# So It's a Github bug or Github block the PR about something
# we don't yet support.
# We don't fully support require_code_owner_reviews, try so do
# some guessing.
protection = branch_rule["protection"]
if (protection["required_pull_request_reviews"]
and protection["required_pull_request_reviews"]
["require_code_owner_reviews"]
and (self.g_pull._rawData['requested_teams']
or self.g_pull._rawData['requested_reviewers'])):
github_desc = "Waiting for code owner review"
else:
# NOTE(sileht): assume it's the Github bug and the PR is
# ready, if it's not the merge button will just fail.
self.log.warning("mergeable_state is unexpected, "
"trying to merge the pull request")
mergify_state = MergifyState.READY
github_state = "success"
github_desc = "Will be merged soon"
elif self._required_statuses == StatusState.PENDING:
# Maybe clean soon, or maybe this is the previous run selected
# PR that we just rebase, or maybe not. But we set the
# mergify_state to ALMOST_READY to ensure we do not rebase
# multiple self.g_pull request in //
mergify_state = MergifyState.ALMOST_READY
github_desc = "Waiting for status checks success"
else:
github_desc = "Waiting for status checks success"
elif self.g_pull.mergeable_state == "behind":
# Not up2date, but ready to merge, is branch updatable
if not self.base_is_modifiable():
github_state = "failure"
github_desc = ("Pull request can't be updated with latest "
"base branch changes, owner doesn't allow "
"modification")
elif self._required_statuses == StatusState.SUCCESS:
mergify_state = MergifyState.NEED_BRANCH_UPDATE
github_desc = ("Pull request will be updated with latest base "
"branch changes soon")
else:
github_desc = "Waiting for status checks success"
elif self.g_pull.mergeable_state == "dirty":
github_desc = "Merge conflict need to be solved"
elif self.g_pull.mergeable_state == "unknown":
# Should not really occur, but who known
github_desc = "Pull request state reported unknown by Github"
else: # pragma: no cover
raise RuntimeError("%s: Unexpected mergify_state" % self)
if github_desc is None: # pragma: no cover
# Seatbelt
raise RuntimeError("%s: github_desc have not been set" % self)
return (mergify_state, github_state, github_desc)
UNUSABLE_STATES = ["unknown", None]
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.2),
stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_never)
def _ensure_mergable_state(self, force=False):
if self.g_pull.merged:
return
if (not force
and self.g_pull.mergeable_state not in self.UNUSABLE_STATES):
return
# Github is currently processing this PR, we wait the completion
self.log.info("refreshing")
# NOTE(sileht): Well github doesn't always update etag/last_modified
# when mergeable_state change, so we get a fresh pull request instead
# of using update()
self.g_pull = self.g_pull.base.repo.get_pull(self.g_pull.number)
if (self.g_pull.merged
or self.g_pull.mergeable_state not in self.UNUSABLE_STATES):
return
raise tenacity.TryAgain
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.2),
stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_never)
def _wait_for_sha_change(self, old_sha):
if (self.g_pull.merged or self.g_pull.head.sha != old_sha):
return
# Github is currently processing this PR, we wait the completion
self.log.info("refreshing")
# NOTE(sileht): Well github doesn't always update etag/last_modified
# when mergeable_state change, so we get a fresh pull request instead
# of using update()
self.g_pull = self.g_pull.base.repo.get_pull(self.g_pull.number)
if (self.g_pull.merged or self.g_pull.head.sha != old_sha):
return
raise tenacity.TryAgain
def wait_for_sha_change(self):
old_sha = self.g_pull.head.sha
self._wait_for_sha_change(old_sha)
self._ensure_mergable_state()
def __lt__(self, other):
return ((self.mergify_state, self.g_pull.updated_at) >
(other.mergify_state, other.g_pull.updated_at))
def __eq__(self, other):
return ((self.mergify_state,
self.g_pull.updated_at) == (other.mergify_state,
other.g_pull.updated_at))
def base_is_modifiable(self):
return (self.g_pull.raw_data["maintainer_can_modify"]
or self.g_pull.head.repo.id == self.g_pull.base.repo.id)
def _merge_failed(self, e):
# Don't log some common and valid error
if e.data["message"].startswith("Base branch was modified"):
return False
self.log.error("merge failed",
status=e.status,
error=e.data["message"],
exc_info=True)
return False
def merge(self, merge_method, rebase_fallback):
try:
self.g_pull.merge(sha=self.g_pull.head.sha,
merge_method=merge_method)
except github.GithubException as e: # pragma: no cover
if (self.g_pull.is_merged()
and e.data["message"] == "Pull Request is not mergeable"):
# Not a big deal, we will receive soon the pull_request close
# event
self.log.info("merged in the meantime")
return True
if (e.data["message"] != "This branch can't be rebased"
or merge_method != "rebase"
or (rebase_fallback == "none" or rebase_fallback is None)):
return self._merge_failed(e)
# If rebase fail retry with merge
try:
self.g_pull.merge(sha=self.g_pull.head.sha,
merge_method=rebase_fallback)
except github.GithubException as e:
return self._merge_failed(e)
return True
def post_check_status(self, state, msg, context=None):
redis = utils.get_redis_for_cache()
context = "pr" if context is None else context
msg_key = "%s/%s/%d/%s" % (self.installation_id,
self.g_pull.base.repo.full_name,
self.g_pull.number, context)
if len(msg) >= 140:
description = msg[0:137] + "..."
redis.hset("status", msg_key, msg.encode('utf8'))
target_url = "%s/check_status_msg/%s" % (config.BASE_URL, msg_key)
else:
description = msg
target_url = None
self.log.info("set status", state=state, description=description)
# NOTE(sileht): We can't use commit.create_status() because
# if use the head repo instead of the base repo
try:
self.g_pull._requester.requestJsonAndCheck(
"POST",
self.g_pull.base.repo.url + "/statuses/" +
self.g_pull.head.sha,
input={
'state': state,
'description': description,
'target_url': target_url,
'context': "%s/%s" % (config.CONTEXT, context)
},
headers={
'Accept': 'application/vnd.github.machine-man-preview+json'
})
except github.GithubException as e: # pragma: no cover
self.log.error("set status failed",
error=e.data["message"],
exc_info=True)
def set_and_post_error(self, github_description):
self._mergify_state = MergifyState.NOT_READY
self._github_state = "failure"
self._github_description = github_description
self.post_check_status(self.github_state, self.github_description)
def is_behind(self):
branch = self.g_pull.base.repo.get_branch(self.g_pull.base.ref)
for commit in self.g_pull.get_commits():
for parent in commit.parents:
if parent.sha == branch.commit.sha:
return False
return True
def __str__(self):
return ("%(login)s/%(repo)s/pull/%(number)d@%(branch)s "
"s:%(pr_state)s/%(statuses)s "
"r:%(approvals)s/%(required_approvals)s "
"-> %(mergify_state)s (%(github_state)s/%(github_desc)s)" % {
"login":
self.g_pull.base.user.login,
"repo":
self.g_pull.base.repo.name,
"number":
self.g_pull.number,
"branch":
self.g_pull.base.ref,
"pr_state": ("merged" if self.g_pull.merged else
(self.g_pull.mergeable_state or "none")),
"statuses":
str(self._required_statuses),
"approvals": ("notset" if self._reviews_ok is None else
len(self._reviews_ok)),
"required_approvals":
("notset" if self._reviews_required is None else
self._reviews_required),
"mergify_state":
str(self._mergify_state),
"github_state": ("notset" if self._github_state is None
else self._github_state),
"github_desc":
("notset" if self._github_description is None else
self._github_description),
})
def test_cephfs_share(self):
"""Test that CephFS shares can be accessed on two instances.
1. Spawn two servers
2. mount it on both
3. write a file on one
4. read it on the other
5. profit
"""
keyring = model.run_on_leader(
'ceph-mon', 'cat /etc/ceph/ceph.client.admin.keyring')['Stdout']
conf = model.run_on_leader('ceph-mon',
'cat /etc/ceph/ceph.conf')['Stdout']
# Spawn Servers
for attempt in Retrying(stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1,
min=2,
max=10)):
with attempt:
instance_1 = guest.launch_instance(
glance_setup.LTS_IMAGE_NAME,
vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX),
userdata=self.INSTANCE_USERDATA.format(
_indent(conf, 8), _indent(keyring, 8)))
for attempt in Retrying(stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1,
min=2,
max=10)):
with attempt:
instance_2 = guest.launch_instance(
glance_setup.LTS_IMAGE_NAME,
vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX),
userdata=self.INSTANCE_USERDATA.format(
_indent(conf, 8), _indent(keyring, 8)))
# Write a file on instance_1
def verify_setup(stdin, stdout, stderr):
status = stdout.channel.recv_exit_status()
self.assertEqual(status, 0)
fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0]
fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0]
username = guest.boot_tests['bionic']['username']
password = guest.boot_tests['bionic'].get('password')
privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME)
for attempt in Retrying(stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1,
min=2,
max=10)):
with attempt:
openstack_utils.ssh_command(
username,
fip_1,
'instance-1', 'sudo mount -a && '
'echo "test" | sudo tee /mnt/cephfs/test',
password=password,
privkey=privkey,
verify=verify_setup)
def verify(stdin, stdout, stderr):
status = stdout.channel.recv_exit_status()
self.assertEqual(status, 0)
out = ""
for line in iter(stdout.readline, ""):
out += line
self.assertEqual(out, "test\n")
openstack_utils.ssh_command(username,
fip_2,
'instance-2', 'sudo mount -a && '
'sudo cat /mnt/cephfs/test',
password=password,
privkey=privkey,
verify=verify)
raise Exception(msg.format(timeout))
statuses = get_workflow_statuses(workflow_ids, cromwell_url, cromwell_user, cromwell_password, caas_key)
all_succeeded = True
for i, status in enumerate(statuses):
if status in _failed_statuses:
raise Exception('Stopping because workflow {0} {1}'.format(workflow_ids[i], status))
elif status != 'Succeeded':
all_succeeded = False
if all_succeeded:
print('All workflows succeeded!')
break
else:
time.sleep(poll_interval_seconds)
@retry(reraise=True, wait=wait_exponential(multiplier=1, max=10), stop=stop_after_delay(20))
def start_workflow(
wdl_file, inputs_file, url, options_file=None, inputs_file2=None, zip_file=None, user=None,
password=None, caas_key=None, collection_name=None, label=None, validate_labels=True):
"""
Use HTTP POST to start workflow in Cromwell and retry with exponentially increasing wait times between requests
if there are any failures. View statistics about the retries with `start_workflow.retry.statistics`.
.. note::
The requests library could accept both Bytes and String objects as parameters of files, so there is no
strict restrictions on the type of inputs of this function.
:param _io.BytesIO wdl_file: wdl file.
:param _io.BytesIO inputs_file: inputs file.
:param str url: cromwell url.
return False
async def delete_dir_contents(path):
logger.info(f"Deleting directory contents of {path}...")
try:
for root, dirs, files in os.walk(path):
for f in files:
os.unlink(os.path.join(root, f))
for d in dirs:
shutil.rmtree(os.path.join(root, d))
except Exception as e:
logger.exception(f"Could not remove contents in {path}. Reason: {e}")
@retry(stop=stop_after_attempt(10), wait=wait_exponential(min=1, max=10))
async def deploy_compose(compose):
try:
if not isinstance(compose, dict):
compose = parse_yaml(compose)
# Dump the compose to a temporary compose file and launch that. This is so we can amend the compose and update the
# the stack without launching a new one
dump_yaml(config.TMP_COMPOSE, compose)
compose = config.TMP_COMPOSE
proc = await asyncio.create_subprocess_exec("docker", "stack", "deploy", "--compose-file", compose, "walkoff",
stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)
await log_proc_output(proc)
if proc.returncode:
class Context(object):
client: http.Client
pull: dict
subscription: subscription.Subscription
sources: List = dataclasses.field(default_factory=list)
_write_permission_cache: cachetools.LRUCache = dataclasses.field(
default_factory=lambda: cachetools.LRUCache(4096))
log: logging.LoggerAdapter = dataclasses.field(init=False)
def __post_init__(self):
self._ensure_complete()
self.log = daiquiri.getLogger(
self.__class__.__qualname__,
gh_owner=self.pull["base"]["user"]["login"]
if "base" in self.pull else "<unknown-yet>",
gh_repo=(self.pull["base"]["repo"]["name"]
if "base" in self.pull else "<unknown-yet>"),
gh_private=(self.pull["base"]["repo"]["private"]
if "base" in self.pull else "<unknown-yet>"),
gh_branch=self.pull["base"]["ref"]
if "base" in self.pull else "<unknown-yet>",
gh_pull=self.pull["number"],
gh_pull_sha=self.pull["base"]["sha"]
if "base" in self.pull else "<unknown-yet>",
gh_pull_url=self.pull.get("html_url", "<unknown-yet>"),
gh_pull_state=("merged" if self.pull.get("merged") else
(self.pull.get("mergeable_state", "unknown")
or "none")),
)
@property
def base_url(self):
"""The URL prefix to make GitHub request."""
return f"/repos/{self.pull['base']['user']['login']}/{self.pull['base']['repo']['name']}"
@property
def pull_request(self):
return PullRequest(self)
@cachetools.cachedmethod(
cache=operator.attrgetter("_write_permission_cache"),
key=functools.partial(cachetools.keys.hashkey,
"has_write_permissions"),
)
def has_write_permissions(self, login):
return self.client.item(
f"{self.base_url}/collaborators/{login}/permission"
)["permission"] in [
"admin",
"write",
]
def _get_valid_users(self):
bots = list(
set([
r["user"]["login"] for r in self.reviews
if r["user"] and r["user"]["type"] == "Bot"
]))
collabs = set([
r["user"]["login"] for r in self.reviews
if r["user"] and r["user"]["type"] != "Bot"
])
valid_collabs = [
login for login in collabs if self.has_write_permissions(login)
]
return bots + valid_collabs
@functools.cached_property
def consolidated_reviews(self):
# Ignore reviews that are not from someone with admin/write permissions
# And only keep the last review for each user.
comments = dict()
approvals = dict()
valid_users = self._get_valid_users()
for review in self.reviews:
if not review["user"] or review["user"]["login"] not in valid_users:
continue
# Only keep latest review of an user
if review["state"] == "COMMENTED":
comments[review["user"]["login"]] = review
else:
approvals[review["user"]["login"]] = review
return list(comments.values()), list(approvals.values())
def _get_consolidated_data(self, name):
if name == "assignee":
return [a["login"] for a in self.pull["assignees"]]
elif name == "label":
return [label["name"] for label in self.pull["labels"]]
elif name == "review-requested":
return [u["login"] for u in self.pull["requested_reviewers"]] + [
"@" + t["slug"] for t in self.pull["requested_teams"]
]
elif name == "draft":
return self.pull["draft"]
elif name == "author":
return self.pull["user"]["login"]
elif name == "merged-by":
return self.pull["merged_by"]["login"] if self.pull[
"merged_by"] else ""
elif name == "merged":
return self.pull["merged"]
elif name == "closed":
return self.pull["state"] == "closed"
elif name == "milestone":
return self.pull["milestone"]["title"] if self.pull[
"milestone"] else ""
elif name == "number":
return self.pull["number"]
elif name == "conflict":
return self.pull["mergeable_state"] == "dirty"
elif name == "base":
return self.pull["base"]["ref"]
elif name == "head":
return self.pull["head"]["ref"]
elif name == "locked":
return self.pull["locked"]
elif name == "title":
return self.pull["title"]
elif name == "body":
return self.pull["body"]
elif name == "files":
return [f["filename"] for f in self.files]
elif name == "approved-reviews-by":
_, approvals = self.consolidated_reviews
return [
r["user"]["login"] for r in approvals
if r["state"] == "APPROVED"
]
elif name == "dismissed-reviews-by":
_, approvals = self.consolidated_reviews
return [
r["user"]["login"] for r in approvals
if r["state"] == "DISMISSED"
]
elif name == "changes-requested-reviews-by":
_, approvals = self.consolidated_reviews
return [
r["user"]["login"] for r in approvals
if r["state"] == "CHANGES_REQUESTED"
]
elif name == "commented-reviews-by":
comments, _ = self.consolidated_reviews
return [
r["user"]["login"] for r in comments
if r["state"] == "COMMENTED"
]
# NOTE(jd) The Check API set conclusion to None for pending.
# NOTE(sileht): "pending" statuses are not really trackable, we
# voluntary drop this event because CIs just sent they status every
# minutes until the CI pass (at least Travis and Circle CI does
# that). This was causing a big load on Mergify for nothing useful
# tracked, and on big projects it can reach the rate limit very
# quickly.
# NOTE(sileht): Not handled for now: cancelled, timed_out, or action_required
elif name == "status-success":
return [
ctxt for ctxt, state in self.checks.items()
if state == "success"
]
elif name == "status-failure":
return [
ctxt for ctxt, state in self.checks.items()
if state == "failure"
]
elif name == "status-neutral":
return [
ctxt for ctxt, state in self.checks.items()
if state == "neutral"
]
else:
raise PullRequestAttributeError(name)
def update_pull_check_runs(self, check):
self.pull_check_runs = [
c for c in self.pull_check_runs if c["name"] != check["name"]
]
self.pull_check_runs.append(check)
@functools.cached_property
def pull_check_runs(self):
return check_api.get_checks_for_ref(self, self.pull["head"]["sha"])
@property
def pull_engine_check_runs(self):
return [
c for c in self.pull_check_runs
if c["app"]["id"] == config.INTEGRATION_ID
]
@functools.cached_property
def checks(self):
# NOTE(sileht): conclusion can be one of success, failure, neutral,
# cancelled, timed_out, or action_required, and None for "pending"
checks = dict(
(c["name"], c["conclusion"]) for c in self.pull_check_runs)
# NOTE(sileht): state can be one of error, failure, pending,
# or success.
checks.update((s["context"], s["state"]) for s in self.client.items(
f"{self.base_url}/commits/{self.pull['head']['sha']}/status",
list_items="statuses",
))
return checks
def _resolve_login(self, name):
if not name:
return []
elif not isinstance(name, str):
return [name]
elif name[0] != "@":
return [name]
if "/" in name:
organization, _, team_slug = name.partition("/")
if not team_slug or "/" in team_slug:
# Not a team slug
return [name]
organization = organization[1:]
else:
organization = self.pull["base"]["repo"]["owner"]["login"]
team_slug = name[1:]
try:
return [
member["login"] for member in self.client.items(
f"/orgs/{organization}/teams/{team_slug}/members")
]
except http.HTTPClientSideError as e:
self.log.warning(
"fail to get the organization, team or members",
team=name,
status=e.status_code,
detail=e.message,
)
return [name]
def resolve_teams(self, values):
if not values:
return []
if not isinstance(values, (list, tuple)):
values = [values]
values = list(
itertools.chain.from_iterable((map(self._resolve_login, values))))
return values
UNUSABLE_STATES = ["unknown", None]
# NOTE(sileht): quickly retry, if we don't get the status on time
# the exception is recatch in worker.py, so worker will retry it later
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=0.2),
stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_if_exception_type(
exceptions.MergeableStateUnknown),
reraise=True,
)
def _ensure_complete(self):
if not (self._is_data_complete()
and self._is_background_github_processing_completed()):
self.pull = self.client.item(
f"{self.base_url}/pulls/{self.pull['number']}")
if not self._is_data_complete():
self.log.error(
"/pulls/%s has returned an incomplete payload...",
self.pull["number"],
data=self.pull,
)
if self._is_background_github_processing_completed():
return
raise exceptions.MergeableStateUnknown(self)
def _is_data_complete(self):
# NOTE(sileht): If pull request come from /pulls listing or check-runs sometimes,
# they are incomplete, This ensure we have the complete view
fields_to_control = (
"state",
"mergeable_state",
"merged_by",
"merged",
"merged_at",
)
for field in fields_to_control:
if field not in self.pull:
return False
return True
def _is_background_github_processing_completed(self):
return (self.pull["state"] == "closed"
or self.pull["mergeable_state"] not in self.UNUSABLE_STATES)
def update(self):
# TODO(sileht): Remove me,
# Don't use it, because consolidated data are not updated after that.
# Only used by merge action for posting an update report after rebase.
self.pull = self.client.item(
f"{self.base_url}/pulls/{self.pull['number']}")
try:
del self.__dict__["pull_check_runs"]
except KeyError:
pass
@functools.cached_property
def is_behind(self):
branch_name_escaped = parse.quote(self.pull["base"]["ref"], safe="")
branch = self.client.item(
f"{self.base_url}/branches/{branch_name_escaped}")
for commit in self.commits:
for parent in commit["parents"]:
if parent["sha"] == branch["commit"]["sha"]:
return False
return True
def __str__(self):
return "%(login)s/%(repo)s/pull/%(number)d@%(branch)s " "s:%(pr_state)s" % {
"login":
self.pull["base"]["user"]["login"],
"repo":
self.pull["base"]["repo"]["name"],
"number":
self.pull["number"],
"branch":
self.pull["base"]["ref"],
"pr_state": ("merged" if self.pull["merged"] else
(self.pull["mergeable_state"] or "none")),
}
@functools.cached_property
def reviews(self):
return list(
self.client.items(
f"{self.base_url}/pulls/{self.pull['number']}/reviews"))
@functools.cached_property
def commits(self):
return list(
self.client.items(
f"{self.base_url}/pulls/{self.pull['number']}/commits"))
@functools.cached_property
def files(self):
return list(
self.client.items(
f"{self.base_url}/pulls/{self.pull['number']}/files"))
@property
def pull_from_fork(self):
return self.pull["head"]["repo"]["id"] != self.pull["base"]["repo"][
"id"]
def github_workflow_changed(self):
for f in self.files:
if f["filename"].startswith(".github/workflows"):
return True
return False
from bs4 import BeautifulSoup
from tqdm import tqdm
USER_AGENT = ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, '
'like Gecko) Chrome/55.0.2883.87 Safari/537.36')
BASE_LINK_EDGE = ('https://globalnewselection.s3.amazonaws.com/fm-playlist/'
'results/CFNYFM_pl_long.js?callback=plCallback')
BASE_LINK_INDIE = ('http://indie.streamon.fm/eventrange/{}-{}.json')
def to_epoch(dt):
return (dt - datetime(1970, 1, 1)).total_seconds()
@tenacity.retry(reraise=True,
wait=tenacity.wait_exponential(),
stop=tenacity.stop_after_attempt(5))
def get_indie_data(start, end):
return requests.get(BASE_LINK_INDIE.format(start, end),
headers={
'User-Agent': USER_AGENT
}).json()
@tenacity.retry(reraise=True,
wait=tenacity.wait_exponential(),
stop=tenacity.stop_after_attempt(5))
def get_edge_data():
soup = BeautifulSoup(
requests.get('https://edge.ca/music/').content, 'html.parser')
date = to_epoch(
class S3Storage(storage.StorageDriver):
WRITE_FULL = True
_consistency_wait = tenacity.wait_exponential(multiplier=0.1)
def __init__(self, conf):
super(S3Storage, self).__init__(conf)
self.s3, self._region_name, self._bucket_prefix = (
s3.get_connection(conf))
self._bucket_name = '%s-aggregates' % self._bucket_prefix
if conf.s3_check_consistency_timeout > 0:
self._consistency_stop = tenacity.stop_after_delay(
conf.s3_check_consistency_timeout)
else:
self._consistency_stop = None
def __str__(self):
return "%s: %s" % (self.__class__.__name__, self._bucket_name)
def upgrade(self):
super(S3Storage, self).upgrade()
try:
s3.create_bucket(self.s3, self._bucket_name, self._region_name)
except botocore.exceptions.ClientError as e:
if e.response['Error'].get('Code') != "BucketAlreadyExists":
raise
@staticmethod
def _object_name(split_key, aggregation, version=3):
name = '%s_%s_%s' % (
aggregation,
utils.timespan_total_seconds(split_key.sampling),
split_key,
)
return name + '_v%s' % version if version else name
@staticmethod
def _prefix(metric):
return str(metric.id) + '/'
def _put_object_safe(self, Bucket, Key, Body):
put = self.s3.put_object(Bucket=Bucket, Key=Key, Body=Body)
if self._consistency_stop:
def _head():
return self.s3.head_object(Bucket=Bucket,
Key=Key,
IfMatch=put['ETag'])
tenacity.Retrying(retry=tenacity.retry_if_result(
lambda r: r['ETag'] != put['ETag']),
wait=self._consistency_wait,
stop=self._consistency_stop)(_head)
def _store_metric_splits_unbatched(self, metric, key, aggregation, data,
offset, version):
self._put_object_safe(
Bucket=self._bucket_name,
Key=self._prefix(metric) +
self._object_name(key, aggregation.method, version),
Body=data)
def _delete_metric_splits_unbatched(self,
metric,
key,
aggregation,
version=3):
self.s3.delete_object(
Bucket=self._bucket_name,
Key=self._prefix(metric) +
self._object_name(key, aggregation.method, version))
def _delete_metric(self, metric):
bucket = self._bucket_name
response = {}
while response.get('IsTruncated', True):
if 'NextContinuationToken' in response:
kwargs = {
'ContinuationToken': response['NextContinuationToken']
}
else:
kwargs = {}
try:
response = self.s3.list_objects_v2(Bucket=bucket,
Prefix=self._prefix(metric),
**kwargs)
except botocore.exceptions.ClientError as e:
if e.response['Error'].get('Code') == "NoSuchKey":
# Maybe it never has been created (no measure)
return
raise
s3.bulk_delete(self.s3, bucket,
[c['Key'] for c in response.get('Contents', ())])
def _get_splits_unbatched(self, metric, key, aggregation, version=3):
try:
response = self.s3.get_object(
Bucket=self._bucket_name,
Key=self._prefix(metric) +
self._object_name(key, aggregation.method, version))
except botocore.exceptions.ClientError as e:
if e.response['Error'].get('Code') == 'NoSuchKey':
return
raise
return response['Body'].read()
def _metric_exists_p(self, metric, version):
unaggkey = self._build_unaggregated_timeserie_path(metric, version)
try:
self.s3.head_object(Bucket=self._bucket_name, Key=unaggkey)
except botocore.exceptions.ClientError as e:
if e.response['Error'].get('Code') == "404":
return False
raise
return True
def _list_split_keys(self, metric, aggregations, version=3):
bucket = self._bucket_name
keys = {}
for aggregation in aggregations:
keys[aggregation] = set()
response = {}
while response.get('IsTruncated', True):
if 'NextContinuationToken' in response:
kwargs = {
'ContinuationToken': response['NextContinuationToken']
}
else:
kwargs = {}
response = self.s3.list_objects_v2(
Bucket=bucket,
Prefix=self._prefix(metric) + '%s_%s' % (
aggregation.method,
utils.timespan_total_seconds(aggregation.granularity),
),
**kwargs)
# If response is empty then check that the metric exists
contents = response.get('Contents', ())
if not contents and not self._metric_exists_p(metric, version):
raise storage.MetricDoesNotExist(metric)
for f in contents:
try:
if (self._version_check(f['Key'], version)):
meta = f['Key'].split('_')
keys[aggregation].add(
carbonara.SplitKey(
utils.to_timestamp(meta[2]),
sampling=aggregation.granularity))
except (ValueError, IndexError):
# Might be "none", or any other file. Be resilient.
continue
return keys
@staticmethod
def _build_unaggregated_timeserie_path(metric, version):
return S3Storage._prefix(metric) + 'none' + ("_v%s" % version
if version else "")
def _get_or_create_unaggregated_timeseries_unbatched(
self, metric, version=3):
key = self._build_unaggregated_timeserie_path(metric, version)
try:
response = self.s3.get_object(Bucket=self._bucket_name, Key=key)
except botocore.exceptions.ClientError as e:
if e.response['Error'].get('Code') == "NoSuchKey":
# Create the metric with empty data
self._put_object_safe(Bucket=self._bucket_name,
Key=key,
Body="")
else:
raise
else:
return response['Body'].read() or None
def _store_unaggregated_timeseries_unbatched(self,
metric,
data,
version=3):
self._put_object_safe(Bucket=self._bucket_name,
Key=self._build_unaggregated_timeserie_path(
metric, version),
Body=data)
def set_link_bridge_stp(device, stp, namespace=None):
return _run_iproute_link('set', device, namespace=namespace,
kind='bridge', br_stp_state=stp)
@privileged.link_cmd.entrypoint
def set_link_bridge_master(device, bridge, namespace=None):
bridge_idx = get_link_id(bridge, namespace) if bridge else 0
return _run_iproute_link('set', device, namespace=namespace,
master=bridge_idx)
@tenacity.retry(
retry=tenacity.retry_if_exception_type(
netlink_exceptions.NetlinkDumpInterrupted),
wait=tenacity.wait_exponential(multiplier=0.02, max=1),
stop=tenacity.stop_after_delay(8),
reraise=True)
@privileged.link_cmd.entrypoint
def get_link_attributes(device, namespace):
link = _run_iproute_link("get", device, namespace)[0]
return {
'mtu': link.get_attr('IFLA_MTU'),
'qlen': link.get_attr('IFLA_TXQLEN'),
'state': link.get_attr('IFLA_OPERSTATE'),
'qdisc': link.get_attr('IFLA_QDISC'),
'brd': link.get_attr('IFLA_BROADCAST'),
'link/ether': link.get_attr('IFLA_ADDRESS'),
'alias': link.get_attr('IFLA_IFALIAS'),
'allmulticast': bool(link['flags'] & ifinfmsg.IFF_ALLMULTI),
'link_kind': link.get_nested('IFLA_LINKINFO', 'IFLA_INFO_KIND')
class MergifyPull(object):
# NOTE(sileht): Use from_cache/from_event not the constructor directly
g = attr.ib()
g_pull = attr.ib()
installation_id = attr.ib()
_consolidated_data = attr.ib(init=False, default=None)
@classmethod
def from_raw(cls, installation_id, installation_token, pull_raw):
g = github.Github(installation_token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
pull = github.PullRequest.PullRequest(g._Github__requester, {},
pull_raw,
completed=True)
return cls(g, pull, installation_id)
@classmethod
def from_number(cls, installation_id, installation_token, owner, reponame,
pull_number):
g = github.Github(installation_token,
base_url="https://api.%s" % config.GITHUB_DOMAIN)
repo = g.get_repo(owner + "/" + reponame)
pull = repo.get_pull(pull_number)
return cls(g, pull, installation_id)
def __attrs_post_init__(self):
self._ensure_mergable_state()
def _valid_perm(self, user):
if user.type == "Bot":
return True
return self.g_pull.base.repo.get_collaborator_permission(
user.login) in [
"admin",
"write",
]
def _get_reviews(self):
# Ignore reviews that are not from someone with admin/write permissions
# And only keep the last review for each user.
reviews = list(self.g_pull.get_reviews())
valid_users = list(
map(
lambda u: u.login,
filter(self._valid_perm, set([r.user for r in reviews])),
))
comments = dict()
approvals = dict()
for review in reviews:
if review.user.login not in valid_users:
continue
# Only keep latest review of an user
if review.state == "COMMENTED":
comments[review.user.login] = review
else:
approvals[review.user.login] = review
return list(comments.values()), list(approvals.values())
def to_dict(self):
if self._consolidated_data is None:
self._consolidated_data = self._get_consolidated_data()
return self._consolidated_data
def _get_consolidated_data(self):
comments, approvals = self._get_reviews()
statuses = self._get_checks()
# FIXME(jd) pygithub does 2 HTTP requests whereas 1 is enough!
(
review_requested_users,
review_requested_teams,
) = self.g_pull.get_review_requests()
return {
# Only use internally attributes
"_approvals":
approvals,
# Can be used by rules too
"assignee": [a.login for a in self.g_pull.assignees],
# NOTE(sileht): We put an empty label to allow people to match
# no label set
"label": [l.name for l in self.g_pull.labels],
"review-requested":
([u.login for u in review_requested_users] +
["@" + t.slug for t in review_requested_teams]),
"author":
self.g_pull.user.login,
"merged-by":
(self.g_pull.merged_by.login if self.g_pull.merged_by else ""),
"merged":
self.g_pull.merged,
"closed":
self.g_pull.state == "closed",
"milestone":
(self.g_pull.milestone.title if self.g_pull.milestone else ""),
"conflict":
self.g_pull.mergeable_state == "dirty",
"base":
self.g_pull.base.ref,
"head":
self.g_pull.head.ref,
"locked":
self.g_pull._rawData["locked"],
"title":
self.g_pull.title,
"body":
self.g_pull.body,
"files": [f.filename for f in self.g_pull.get_files()],
"approved-reviews-by":
[r.user.login for r in approvals if r.state == "APPROVED"],
"dismissed-reviews-by":
[r.user.login for r in approvals if r.state == "DISMISSED"],
"changes-requested-reviews-by": [
r.user.login for r in approvals
if r.state == "CHANGES_REQUESTED"
],
"commented-reviews-by":
[r.user.login for r in comments if r.state == "COMMENTED"],
"status-success":
[s.context for s in statuses if s.state == "success"],
# NOTE(jd) The Check API set conclusion to None for pending.
# NOTE(sileht): "pending" statuses are not really trackable, we
# voluntary drop this event because CIs just sent they status every
# minutes until the CI pass (at least Travis and Circle CI does
# that). This was causing a big load on Mergify for nothing useful
# tracked, and on big projects it can reach the rate limit very
# quickly.
# "status-pending": [s.context for s in statuses
# if s.state in ("pending", None)],
"status-failure":
[s.context for s in statuses if s.state == "failure"],
"status-neutral":
[s.context for s in statuses if s.state == "neutral"],
# NOTE(sileht): Not handled for now
# cancelled, timed_out, or action_required
}
def _get_statuses(self):
already_seen = set()
statuses = []
for status in github.PaginatedList.PaginatedList(
github.CommitStatus.CommitStatus,
self.g_pull._requester,
self.g_pull.base.repo.url + "/commits/" +
self.g_pull.head.sha + "/statuses",
None,
):
if status.context not in already_seen:
already_seen.add(status.context)
statuses.append(status)
return statuses
def _get_checks(self):
generic_checks = set()
try:
# NOTE(sileht): conclusion can be one of success, failure, neutral,
# cancelled, timed_out, or action_required, and None for "pending"
generic_checks |= set([
GenericCheck(c.name, c.conclusion)
for c in check_api.get_checks(self.g_pull)
])
except github.GithubException as e:
if (e.status != 403 or e.data["message"] !=
"Resource not accessible by integration"):
raise
# NOTE(sileht): state can be one of error, failure, pending,
# or success.
generic_checks |= set(
[GenericCheck(s.context, s.state) for s in self._get_statuses()])
return generic_checks
def _resolve_login(self, name):
if not name:
return []
elif not isinstance(name, str):
return [name]
elif name[0] != "@":
return [name]
if "/" in name:
organization, _, team_slug = name.partition("/")
if not team_slug or "/" in team_slug:
# Not a team slug
return [name]
organization = organization[1:]
else:
organization = self.g_pull.base.repo.owner.login
team_slug = name[1:]
try:
g_organization = self.g.get_organization(organization)
for team in g_organization.get_teams():
if team.slug == team_slug:
return [m.login for m in team.get_members()]
except github.GithubException as e:
if e.status >= 500:
raise
LOG.warning(
"fail to get the organization, team or members",
team=name,
status=e.status,
detail=e.data["message"],
pull_request=self,
)
return [name]
def resolve_teams(self, values):
if not values:
return []
if not isinstance(values, (list, tuple)):
values = [values]
return list(
itertools.chain.from_iterable((map(self._resolve_login, values))))
UNUSABLE_STATES = ["unknown", None]
# NOTE(sileht): quickly retry, if we don't get the status on time
# the exception is recatch in worker.py, so celery will retry it later
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=0.2),
stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_if_exception_type(
exceptions.MergeableStateUnknown),
reraise=True,
)
def _ensure_mergable_state(self, force=False):
if self.g_pull.state == "closed":
return
if not force and self.g_pull.mergeable_state not in self.UNUSABLE_STATES:
return
# Github is currently processing this PR, we wait the completion
LOG.info("refreshing", pull_request=self)
# NOTE(sileht): Well github doesn't always update etag/last_modified
# when mergeable_state change, so we get a fresh pull request instead
# of using update()
self.g_pull = self.g_pull.base.repo.get_pull(self.g_pull.number)
if (self.g_pull.state == "closed"
or self.g_pull.mergeable_state not in self.UNUSABLE_STATES):
return
raise exceptions.MergeableStateUnknown(self)
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=0.2),
stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_never,
)
def _wait_for_sha_change(self, old_sha):
if self.g_pull.state == "closed" or self.g_pull.head.sha != old_sha:
return
# Github is currently processing this PR, we wait the completion
LOG.info("refreshing", pull_request=self)
# NOTE(sileht): Well github doesn't always update etag/last_modified
# when mergeable_state change, so we get a fresh pull request instead
# of using update()
self.g_pull = self.g_pull.base.repo.get_pull(self.g_pull.number)
if self.g_pull.state == "closed" or self.g_pull.head.sha != old_sha:
return
raise tenacity.TryAgain
def wait_for_sha_change(self):
old_sha = self.g_pull.head.sha
self._wait_for_sha_change(old_sha)
self._ensure_mergable_state()
def base_is_modifiable(self):
return (self.g_pull.raw_data["maintainer_can_modify"]
or self.g_pull.head.repo.id == self.g_pull.base.repo.id)
def is_behind(self):
branch = self.g_pull.base.repo.get_branch(
parse.quote(self.g_pull.base.ref, safe=""))
for commit in self.g_pull.get_commits():
for parent in commit.parents:
if parent.sha == branch.commit.sha:
return False
return True
def get_merge_commit_message(self):
if not self.g_pull.body:
return
found = False
message_lines = []
for line in self.g_pull.body.split("\n"):
if MARKDOWN_COMMIT_MESSAGE_RE.match(line):
found = True
elif found and MARKDOWN_TITLE_RE.match(line):
break
elif found:
message_lines.append(line)
if found and message_lines:
return {
"commit_title": message_lines[0],
"commit_message": "\n".join(message_lines[1:]).strip(),
}
def __str__(self):
return "%(login)s/%(repo)s/pull/%(number)d@%(branch)s " "s:%(pr_state)s" % {
"login":
self.g_pull.base.user.login,
"repo":
self.g_pull.base.repo.name,
"number":
self.g_pull.number,
"branch":
self.g_pull.base.ref,
"pr_state": ("merged" if self.g_pull.merged else
(self.g_pull.mergeable_state or "none")),
}
])
def _do_update_branch(git, method, base_branch, head_branch):
if method == "merge":
git("merge", "--quiet", "upstream/%s" % base_branch, "-m",
"Merge branch '%s' into '%s'" % (base_branch, head_branch))
git("push", "--quiet", "origin", head_branch)
elif method == "rebase":
git("rebase", "upstream/%s" % base_branch)
git("push", "--quiet", "origin", head_branch, "-f")
else:
raise RuntimeError("Invalid branch update method")
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.2),
stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_if_exception_type(BranchUpdateNeedRetry))
def _do_update(pull, token, method="merge"):
# NOTE(sileht):
# $ curl https://api.github.com/repos/sileht/repotest/pulls/2 | jq .commits
# 2
# $ git clone https://[email protected]/sileht-tester/repotest \
# --depth=$((2 + 1)) -b sileht/testpr
# $ cd repotest
# $ git remote add upstream https://[email protected]/sileht/repotest.git
# $ git log | grep Date | tail -1
# Date: Fri Mar 30 21:30:26 2018 (10 days ago)
# $ git fetch upstream master --shallow-since="Fri Mar 30 21:30:26 2018"
# $ git rebase upstream/master
# $ git push origin sileht/testpr:sileht/testpr
class Bootloader:
""" A class to hold the logic for each of the possible commands. This follows the dispatch pattern we us in app_base
for calling actions in apps. The pattern as applied to the CLI follows close to this example:
https://chase-seibert.github.io/blog/2014/03/21/python-multilevel-argparse.html#
"""
def __init__(self, session=None, docker_client=None):
self.session: aiohttp.ClientSession = session
self.docker_client: aiodocker.Docker = docker_client
with open(".dockerignore") as f:
self.dockerignore = [line.strip() for line in f.readlines()]
@staticmethod
async def run():
""" Landing pad to launch primary command and do whatever async init the bootloader needs. """
# TODO: fill in the helps, and further develop cli with the end user in mind
commands = {"up", "down", "refresh"}
parser = argparse.ArgumentParser()
parser.add_argument("command", choices=commands)
parser.add_argument("args", nargs=argparse.REMAINDER)
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
# Parse out the command
args = parser.parse_args(sys.argv[1:2])
async with aiohttp.ClientSession() as session, connect_to_aiodocker() as docker_client:
bootloader = Bootloader(session, docker_client)
if hasattr(bootloader, args.command):
await getattr(bootloader, args.command)()
else:
logger.error("Invalid command.")
# TODO: Pipe this through the logger. print_help() accepts a file kwarg that we can use to do this
parser.print_help()
@retry(stop=stop_after_attempt(10), wait=wait_exponential(min=1, max=10))
async def wait_for_registry(self):
try:
async with self.session.get(f"http://{DOCKER_HOST_IP}:5000") as resp:
if resp.status == 200:
return True
else:
raise ConnectionError
except Exception as e:
logger.info("Registry not available yet, waiting to try again...")
raise e
@retry(stop=stop_after_attempt(10), wait=wait_exponential(min=1, max=10))
async def wait_for_minio(self):
try:
async with self.session.get(f"http://{config.MINIO}/minio/health/ready") as resp:
if resp.status == 200:
return True
else:
raise ConnectionError
except Exception as e:
logger.info("Minio not available yet, waiting to try again...")
raise e
async def up(self):
data = {"name": "postgres-data"}
# Create Postgres Volume
await self.docker_client.volumes.create(data)
# Create Walkoff encryption key
wek = await create_encryption_key(self.docker_client, "walkoff_encryption_key")
# Create internal user key
wik = await create_encryption_key(self.docker_client, "walkoff_internal_key")
# Create Postgres user password
wpk = await create_encryption_key(self.docker_client, "walkoff_postgres_key")
# Create Minio secret key
wmak = await create_encryption_key(self.docker_client, "walkoff_minio_access_key", b"walkoff")
wmsk = await create_encryption_key(self.docker_client, "walkoff_minio_secret_key")
# Set up a subcommand parser
parser = argparse.ArgumentParser(description="Bring the WALKOFF stack up and initialize it")
parser.add_argument("-b", "--build", action="store_true",
help="Builds and pushes all WALKOFF components to local registry.")
parser.add_argument("-d", "--debug", action="store_true",
help="Set log level to debug.")
parser.add_argument("-k", "--keys", action="store_true",
help="Prints all keys to STDOUT (dangerous).")
# Parse out the command
args = parser.parse_args(sys.argv[2:])
if args.debug:
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
logger.info("Creating persistent directories for registry, postgres, portainer...")
os.makedirs(static.REGISTRY_DATA_PATH, exist_ok=True)
os.makedirs(static.POSTGRES_DATA_PATH, exist_ok=True)
os.makedirs(static.PORTAINER_DATA_PATH, exist_ok=True)
os.makedirs(static.MINIO_DATA_PATH, exist_ok=True)
# Bring up the base compose with the registry
logger.info("Deploying base services (registry, postgres, portainer, redis)...")
base_compose = parse_yaml(config.BASE_COMPOSE)
await deploy_compose(base_compose)
await self.wait_for_registry()
# Merge the base, walkoff, and app composes
app_composes = generate_app_composes()
walkoff_compose = parse_yaml(config.WALKOFF_COMPOSE)
merged_compose = merge_composes(walkoff_compose, app_composes)
dump_yaml(config.TMP_COMPOSE, merged_compose)
if args.build:
walkoff_app_sdk = walkoff_compose["services"]["app_sdk"]
await build_image(self.docker_client, walkoff_app_sdk["image"],
walkoff_app_sdk["build"]["dockerfile"],
walkoff_app_sdk["build"]["context"],
self.dockerignore)
await push_image(self.docker_client, walkoff_app_sdk["image"])
builders = []
pushers = []
for service_name, service in walkoff_compose["services"].items():
if "build" in service:
build_func = build_image(self.docker_client, service["image"],
service["build"]["dockerfile"],
service["build"]["context"],
self.dockerignore)
push_func = push_image(self.docker_client, service["image"])
if args.debug:
await build_func
await push_func
else:
builders.append(build_func)
pushers.append(push_func)
if not args.debug:
logger.info("Building Docker images asynchronously, this could take some time...")
await asyncio.gather(*builders)
logger.info("Build process complete.")
logger.info("Pushing Docker images asynchronously, this could take some time...")
await asyncio.gather(*pushers)
logger.info("Push process complete.")
await self.wait_for_minio()
# await self.push_to_minio()
logger.info("Deploying Walkoff stack...")
return_code = await deploy_compose(merged_compose)
if args.keys:
if await are_you_sure("You specified -k/--keys, which will print all newly created keys to stdout."):
print(f"walkoff_encryption_key:\t\t{wek.decode()}")
print(f"walkoff_internal_key:\t\t{wik.decode()}")
print(f"walkoff_postgres_key:\t\t{wpk.decode()}")
print(f"walkoff_minio_access_key:\t{wmak.decode()}")
print(f"walkoff_minio_secret_key:\t{wmsk.decode()}\n\n")
logger.info("Walkoff stack deployed, it may take a little time to converge. \n"
"Use 'docker stack services walkoff' to check on Walkoff services. \n"
"Web interface should be available at 'https://127.0.0.1:8080' once walkoff_resource_nginx is up.")
return return_code
async def down(self):
# Set up a subcommand parser
parser = argparse.ArgumentParser(description="Remove the WALKOFF stack and optionally related artifacts.")
parser.add_argument("-k", "--key", action="store_true",
help="Removes the walkoff_encryption_key secret.")
parser.add_argument("-r", "--registry", action="store_true",
help="Clears the registry bind mount directory.")
parser.add_argument("-v", "--volume", action="store_true", help="Clears the postgresql volume")
parser.add_argument("-d", "--debug", action="store_true",
help="Set log level to debug.")
# Parse out the command
args = parser.parse_args(sys.argv[2:])
if args.debug:
logger.setLevel("DEBUG")
docker_logger.setLevel("DEBUG")
proc = await rm_stack("walkoff")
# if not args.skipnetwork:
# logger.info("Waiting for containers to exit and network to be removed...")
# await exponential_wait(check_for_network, [self.docker_client], "Network walkoff_default still exists")
if args.key:
if await are_you_sure("Deleting encryption key will render database unreadable, so it will be cleared. "
"This will delete all workflows, execution results, globals, users, roles, etc. "):
await delete_encryption_key(self.docker_client, "walkoff_encryption_key")
await delete_encryption_key(self.docker_client, "walkoff_internal_key")
await delete_encryption_key(self.docker_client, "walkoff_postgres_key")
await delete_dir_contents(static.POSTGRES_DATA_PATH)
if args.registry:
await delete_dir_contents(static.REGISTRY_DATA_PATH)
await delete_dir_contents(static.MINIO_DATA_PATH)
await delete_encryption_key(self.docker_client, "walkoff_minio_access_key")
await delete_encryption_key(self.docker_client, "walkoff_minio_secret_key")
if args.volume:
await remove_volume("walkoff_postgres-data", wait=True)
logger.info("Walkoff stack removed, it may take a little time to stop all services. "
"It is OK if the walkoff_default network is not fully removed.")
return proc.returncode
async def refresh(self):
parser = argparse.ArgumentParser(description="Rebuild a specific service and force it to update.")
parser.add_argument("-s", "--service",
help="Name of the service to rebuild and update. "
"You can specify a prefix ('walkoff_app' or 'walkoff_core') "
"to rebuild all in that category.")
args = parser.parse_args(sys.argv[2:])
compose = parse_yaml(config.TMP_COMPOSE)['services']
service_yaml = compose.get(args.service)
if service_yaml:
if await are_you_sure("Forcing a service to update will disrupt any work it is currently doing. "
"It is not yet guaranteed that a service will pick back up where it left off. "):
if "build" in service_yaml:
await build_image(self.docker_client, service_yaml["image"],
service_yaml["build"]["dockerfile"],
service_yaml["build"]["context"],
self.dockerignore)
await push_image(self.docker_client, service_yaml["image"])
service_name = f"walkoff_{args.service}"
await force_service_update(self.docker_client, service_name, service_yaml["image"])
else:
services = list(compose.keys())
services.sort()
logger.exception(f"No such service, valid services: {services}.")
"Callithrix jacchus - Common marmoset",
),
(
["melanogaster", "fruit fly"],
None,
"http://purl.obolibrary.org/obo/NCBITaxon_7227",
"Drosophila melanogaster - Fruit fly",
),
]
@lru_cache(maxsize=None)
@tenacity.retry(
reraise=True,
stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_exponential(exp_base=1.25, multiplier=1.25),
)
def parse_purlobourl(
url: str,
lookup: Optional[Tuple[str, ...]] = None) -> Optional[Dict[str, str]]:
"""Parse an Ontobee URL to return properties of a Class node
:param url: Ontobee URL
:param lookup: list of XML nodes to lookup
:return: dictionary containing found nodes
"""
req = requests.get(url, allow_redirects=True)
req.raise_for_status()
doc = parseString(req.text)
for elfound in doc.getElementsByTagName("Class"):
# OVERRIDE on_exception
# If stream error, wait for 5 minutes
def on_exception(self, exception):
logging.critical("Stream error")
print(str(datetime.datetime.now()) + ": STREAM ERROR - check log")
logging.critical(str(exception))
# Function for threads to continually check the queue and clean tweets
def cleanTweets(self):
while True:
cleanAndStore(self.q.get())
self.q.task_done()
# Wrapper function for the stream so that when timeouts occur
# tenacity will just retry the connection
@retry(wait=wait_exponential(multiplier=1, min=5, max=60))
def tenacityStream(stream):
try:
stream.filter(languages = ["en"], track = keywords, is_async = True, stall_warnings = True)
except Exception as e:
logging.info("Retrying stream...")
# Main Function
def main():
# Log stream start
logging.info("Stream started")
# Set up a stream listener
btcListener = BitcoinListener()
# Set up stream
authentication = OAuthHandler(twitterAuth["consumer_key"], twitterAuth["consumer_secret"])
authentication.set_access_token(twitterAuth["access_token"], twitterAuth["access_secret"])
from django.apps import apps
# Models need to be imported like this in order to avoid cyclic import issues with celery
def get_model(model_name):
return apps.get_model(app_label='app', model_name=model_name)
logging.basicConfig(stream=sys.stderr, level=logging.ERROR)
logger = logging.getLogger(__name__)
TENACITY_ARGUMENTS = {
'reraise': True,
'wait': tenacity.wait_exponential(multiplier=60, max=600),
'before_sleep': tenacity.before_sleep_log(logger,
logging.ERROR,
exc_info=True)
}
TENACITY_ARGUMENTS_FAST = {
'reraise': True,
'stop': tenacity.stop_after_attempt(5),
'wait': tenacity.wait_exponential(multiplier=30),
'before_sleep': tenacity.before_sleep_log(logger,
logging.ERROR,
exc_info=True)
}
def batch_qs(qs, batch_size=50000):
class ConfirmationSender:
def __init__(
self,
*,
transfer_event_queue: Queue,
home_bridge_contract: Contract,
private_key: bytes,
gas_price: web3types.Wei,
max_reorg_depth: int,
pending_transaction_queue: Queue,
sanity_check_transfer: Callable,
):
self.private_key = private_key
self.address = PrivateKey(
self.private_key).public_key.to_canonical_address()
self.address_hex = PrivateKey(
self.private_key).public_key.to_checksum_address()
if not is_bridge_validator(home_bridge_contract, self.address):
logger.warning(
f"The address {to_checksum_address(self.address)} is not a bridge validator to confirm "
f"transfers on the home bridge contract!")
self.transfer_event_queue = transfer_event_queue
self.home_bridge_contract = home_bridge_contract
self.gas_price = gas_price
self.max_reorg_depth = max_reorg_depth
self.w3 = self.home_bridge_contract.web3
self.pending_transaction_queue = pending_transaction_queue
self.sanity_check_transfer = sanity_check_transfer
self.chain_id = int(self.w3.eth.chainId)
self.services = [
Service("send-confirmation-transactions",
self.send_confirmation_transactions)
]
self.is_parity = self.w3.clientVersion.startswith(
"Parity") or self.w3.clientVersion.startswith("OpenEthereum")
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=1, min=5, max=120),
before_sleep=tenacity.before_sleep_log(logger, logging.WARN),
retry=tenacity.retry_if_exception(lambda exc: isinstance(
exc, Exception) and not isinstance(exc, NonceTooLowException)),
)
def _rpc_send_raw_transaction(self, raw_transaction):
try:
return self.w3.eth.sendRawTransaction(raw_transaction)
except ValueError as exc:
if is_nonce_too_low_exception(exc):
raise NonceTooLowException("nonce too low") from exc
else:
raise exc
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=1, min=5, max=120),
before_sleep=tenacity.before_sleep_log(logger, logging.WARN),
)
def get_next_nonce(self):
if self.is_parity:
return int(
self.w3.manager.request_blocking("parity_nextNonce",
[self.address_hex]),
16,
)
else:
return self.w3.eth.getTransactionCount(self.address, "pending")
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=1, min=5, max=120),
before_sleep=tenacity.before_sleep_log(logger, logging.WARN),
retry=tenacity.retry_if_exception(
lambda exc: isinstance(exc, NonceTooLowException)),
)
def send_confirmation_from_transfer_event(self, transfer_event):
nonce = self.get_next_nonce()
transaction = self.prepare_confirmation_transaction(
transfer_event=transfer_event, nonce=nonce, chain_id=self.chain_id)
assert transaction is not None
self.send_confirmation_transaction(transaction)
def send_confirmation_transactions(self):
while True:
transfer_event = self.transfer_event_queue.get()
try:
self.sanity_check_transfer(transfer_event)
except Exception as exc:
raise SystemExit(
f"Internal error: sanity check failed for {transfer_event}: {exc}"
) from exc
self.send_confirmation_from_transfer_event(transfer_event)
run = send_confirmation_transactions
def prepare_confirmation_transaction(self, transfer_event,
nonce: web3types.Nonce,
chain_id: int):
transfer_hash = compute_transfer_hash(transfer_event)
transaction_hash = transfer_event.transactionHash
amount = transfer_event.args.value
recipient = transfer_event.args["from"]
logger.info(
"confirmTransfer(transferHash=%s transactionHash=%s amount=%s recipient=%s) with nonce=%s, chain_id=%s",
transfer_hash.hex(),
transaction_hash.hex(),
amount,
recipient,
nonce,
chain_id,
)
# hard code gas limit to avoid executing the transaction (which would fail as the sender
# address is not defined before signing the transaction, but the contract asserts that
# it's a validator)
transaction = self.home_bridge_contract.functions.confirmTransfer(
transferHash=transfer_hash,
transactionHash=transaction_hash,
amount=amount,
recipient=recipient,
).buildTransaction({
"gasPrice": self.gas_price,
"nonce": nonce,
"gas": CONFIRMATION_TRANSACTION_GAS_LIMIT, # type: ignore
"chainId": chain_id,
})
signed_transaction = self.w3.eth.account.sign_transaction(
transaction, self.private_key)
return signed_transaction
def send_confirmation_transaction(self, transaction):
tx_hash = self._rpc_send_raw_transaction(transaction.rawTransaction)
self.pending_transaction_queue.put(transaction)
logger.info(f"Sent confirmation transaction {tx_hash.hex()}")
return tx_hash
def check_headers_available(headers):
product = '854239'
country_iso = '490'
res = fetch_tariff_excel_response(product, country_iso, headers)
df = pd.read_excel(BytesIO(res.content))
try:
assert len(df) > 0
return True
except:
return False
@retry(stop=stop_after_attempt(10),
wait=wait_fixed(10) + wait_exponential(multiplier=1, max=20))
def reload_available_headers():
headers = get_new_headers()
headers_is_available = check_headers_available(headers)
if headers_is_available:
return headers
else:
raise
def get_decent_headers():
try:
with open(CACHED_HEADERS, 'r') as f:
headers = json.load(f)
except (json.JSONDecodeError, FileNotFoundError) as e:
headers = reload_available_headers()
for rule in current_rules:
if '-i %s' % vif in rule and '--among-src' in rule:
ebtables(['-D', chain] + rule.split())
def _delete_mac_spoofing_protection(vifs, current_rules):
# delete the jump rule and then delete the whole chain
jumps = [vif for vif in vifs
if _mac_vif_jump_present(vif, current_rules)]
for vif in jumps:
ebtables(['-D', 'FORWARD', '-i', vif, '-j',
_mac_chain_name(vif)])
for vif in vifs:
chain = _mac_chain_name(vif)
if chain_exists(chain, current_rules):
ebtables(['-X', chain])
# Used to scope ebtables commands in testing
NAMESPACE = None
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=0.01),
retry=tenacity.retry_if_exception(lambda e: e.returncode == 255),
reraise=True
)
def ebtables(comm):
execute = ip_lib.IPWrapper(NAMESPACE).netns.execute
return execute(['ebtables', '--concurrent'] + comm, run_as_root=True)
class DesignateTests(BaseDesignateTest):
"""Designate charm restart and pause tests."""
TEST_DOMAIN = 'amuletexample.com.'
TEST_NS1_RECORD = 'ns1.{}'.format(TEST_DOMAIN)
TEST_NS2_RECORD = 'ns2.{}'.format(TEST_DOMAIN)
TEST_WWW_RECORD = "www.{}".format(TEST_DOMAIN)
TEST_RECORD = {TEST_WWW_RECORD: '10.0.0.23'}
def test_900_restart_on_config_change(self):
"""Checking restart happens on config change.
Change disk format and assert then change propagates to the correct
file and that services are restarted as a result
"""
# Expected default and alternate values
set_default = {'debug': 'False'}
set_alternate = {'debug': 'True'}
# Services which are expected to restart upon config change,
# and corresponding config files affected by the change
conf_file = '/etc/designate/designate.conf'
# Make config change, check for service restarts
self.restart_on_changed(conf_file, set_default, set_alternate,
{'DEFAULT': {
'debug': ['False']
}}, {'DEFAULT': {
'debug': ['True']
}}, self.designate_svcs)
def test_910_pause_and_resume(self):
"""Run pause and resume tests.
Pause service and check services are stopped then resume and check
they are started
"""
with self.pause_resume(self.designate_svcs, pgrep_full=False):
logging.info("Testing pause resume")
def _get_server_id(self, server_name=None, server_id=None):
for srv in self.server_list():
if isinstance(srv, dict):
if srv['id'] == server_id or srv['name'] == server_name:
return srv['id']
elif srv.name == server_name or srv.id == server_id:
return srv.id
return None
def _wait_on_server_gone(self, server_id):
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1,
min=5,
max=10),
reraise=True)
def wait():
logging.debug('Waiting for server %s to disappear', server_id)
if self._get_server_id(server_id=server_id):
raise Exception("Server Exists")
self.server_delete(server_id)
return wait()
def test_400_server_creation(self):
"""Simple api calls to create a server."""
# Designate does not allow the last server to be deleted so ensure
# that ns1 is always present
if self.post_xenial_queens:
logging.info('Skipping server creation tests for Queens and above')
return
if not self._get_server_id(server_name=self.TEST_NS1_RECORD):
server = servers.Server(name=self.TEST_NS1_RECORD)
new_server = self.server_create(server)
self.assertIsNotNone(new_server)
logging.debug('Checking if server exists before trying to create it')
old_server_id = self._get_server_id(server_name=self.TEST_NS2_RECORD)
if old_server_id:
logging.debug('Deleting old server')
self._wait_on_server_gone(old_server_id)
logging.debug('Creating new server')
server = servers.Server(name=self.TEST_NS2_RECORD)
new_server = self.server_create(server)
self.assertIsNotNone(new_server, "Failed to Create Server")
self._wait_on_server_gone(self._get_server_id(self.TEST_NS2_RECORD))
def _get_domain_id(self, domain_name=None, domain_id=None):
for dom in self.domain_list():
if isinstance(dom, dict):
if dom['id'] == domain_id or dom['name'] == domain_name:
return dom['id']
elif dom.id == domain_id or dom.name == domain_name:
return dom.id
return None
def _wait_on_domain_gone(self, domain_id):
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1,
min=5,
max=10),
reraise=True)
def wait():
logging.debug('Waiting for domain %s to disappear', domain_id)
if self._get_domain_id(domain_id=domain_id):
raise Exception("Domain Exists")
self.domain_delete(domain_id)
wait()
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=5,
max=10),
reraise=True)
def _wait_to_resolve_test_record(self):
dns_ip = zaza_juju.get_relation_from_unit(
'designate/0', 'designate-bind/0',
'dns-backend').get('private-address')
logging.info('Waiting for dns record to propagate @ {}'.format(dns_ip))
lookup_cmd = [
'dig', '+short', '@{}'.format(dns_ip), self.TEST_WWW_RECORD
]
cmd_out = subprocess.check_output(lookup_cmd,
universal_newlines=True).rstrip()
if not self.TEST_RECORD[self.TEST_WWW_RECORD] == cmd_out:
raise Exception("Record Doesn't Exist")
def test_400_domain_creation(self):
"""Simple api calls to create domain."""
logging.debug('Checking if domain exists before trying to create it')
old_dom_id = self._get_domain_id(domain_name=self.TEST_DOMAIN)
if old_dom_id:
logging.debug('Deleting old domain')
self._wait_on_domain_gone(old_dom_id)
logging.debug('Creating new domain')
domain = domains.Domain(name=self.TEST_DOMAIN,
email="*****@*****.**")
if self.post_xenial_queens:
new_domain = self.domain_create(name=domain.name,
email=domain.email)
else:
new_domain = self.domain_create(domain)
self.assertIsNotNone(new_domain)
logging.debug('Creating new test record')
_record = records.Record(name=self.TEST_WWW_RECORD,
type="A",
data=self.TEST_RECORD[self.TEST_WWW_RECORD])
if self.post_xenial_queens:
domain_id = new_domain['id']
self.designate.recordsets.create(domain_id, _record.name,
_record.type, [_record.data])
else:
domain_id = new_domain.id
self.designate.records.create(domain_id, _record)
self._wait_to_resolve_test_record()
logging.debug('Tidy up delete test record')
self._wait_on_domain_gone(domain_id)
logging.debug('OK')
except AttributeError:
fname = f.__name__
@six.wraps(f)
def _return_none_on_failure(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception as e:
LOG.critical("Unexpected error while calling %s: %s",
fname, e, exc_info=True)
return _return_none_on_failure
# Retry with exponential backoff for up to 1 minute
wait_exponential = tenacity.wait_exponential(multiplier=0.5, max=60)
retry_on_exception = tenacity.Retrying(wait=wait_exponential)
class _retry_on_exception_and_log(tenacity.retry_if_exception_type):
def __init__(self, msg):
super(_retry_on_exception_and_log, self).__init__()
self.msg = msg
def __call__(self, attempt):
if attempt.failed:
LOG.error(self.msg, exc_info=attempt.exception())
return super(_retry_on_exception_and_log, self).__call__(attempt)
def fairlay_outcome_printer(fairlay_df, bool_vec, title=None):
res = compute_fairlay_outcomes(fairlay_df, bool_vec)
total_outcome, total_ev, empirical_ci, n_matches = res
if title is not None:
print(title + "\n" + "-" * len(title))
print("Maps included: {}".format(n_matches))
print("Total EV: {:.2f} ({:.2f}, {:.2f})".format(total_ev, empirical_ci[0],
empirical_ci[1]))
print("Total outcome: {}".format(total_outcome))
print("")
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=2, max=30),
stop=tenacity.stop_after_attempt(10))
def _fetch_fairlay_data(url):
req = requests.get(url)
assert req.status_code == 200
return req
def fetch_markets(base_url=FAIRLAY_BASE_URL, qry_params=None):
"""Fetch 2 data from fairlay.com.
Currently only supporting category 32, i.e. esports. See
https://fairlay.com/api/#/data/?id=market-categories. Also, only Dota 2
markets, i.e. those with 'dota' in the competition name, are returned.
"""
MAX_RECORDS = 100000
auth = loading.load_auth_from_conf_options(CONF, group)
session = loading.load_session_from_conf_options(CONF, group, auth=auth)
return session
def _get_ironic_session():
global _IRONIC_SESSION
if not _IRONIC_SESSION:
_IRONIC_SESSION = get_session(IRONIC_GROUP)
return _IRONIC_SESSION
@tenacity.retry(
retry=tenacity.retry_if_exception_type(openstack.exceptions.NotSupported),
wait=tenacity.wait_exponential(max=30))
def get_client():
"""Get an ironic client connection."""
session = _get_ironic_session()
try:
return openstack.connection.Connection(
session=session, oslo_conf=CONF).baremetal
except openstack.exceptions.NotSupported as exc:
LOG.error('Ironic API might not be running, failed to establish a '
'connection with ironic, reason: %s. Retrying ...', exc)
raise
except Exception as exc:
LOG.error('Failed to establish a connection with ironic, reason: %s',
exc)
raise
db_name = sys.argv[2]
logging.info('Cluster: %s', cluster_url)
logging.info('Database: %s', db_name)
logging.info('Creating connection string.')
kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(
cluster_url)
logging.info('Creating client.')
client = KustoClient(kcsb)
@retry(
stop=stop_after_attempt(5),
wait=wait_exponential(multiplier=1, min=4, max=32),
retry=retry_if_exception(lambda x: not isinstance(x, KustoServiceError)),
after=after_log(logging, logging.DEBUG))
def execute_command(command: str) -> None:
client.execute_mgmt(db_name, command)
def create_table_set(base_name: str) -> None:
commands = init_script.replace('{TABLE_BASE}', base_name).split('\n\n')
for command in commands:
execute_command(command)
logging.info('Creating Usage Tables.')
create_table_set('Usage')
def boot_worker() -> None:
# Clone autoland
def clone_autoland() -> None:
logger.info(f"Cloning autoland in {REPO_DIR}...")
repository.clone(REPO_DIR, "https://hg.mozilla.org/integration/autoland")
def extract_past_failures_label() -> None:
try:
utils.extract_file(
os.path.join("data", test_scheduling.PAST_FAILURES_LABEL_DB)
)
logger.info("Label-level past failures DB extracted.")
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info(
"Label-level past failures DB not extracted, but missing models are allowed."
)
def extract_failing_together_label() -> None:
try:
utils.extract_file(
os.path.join("data", test_scheduling.FAILING_TOGETHER_LABEL_DB)
)
logger.info("Failing together label DB extracted.")
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info(
"Failing together label DB not extracted, but missing models are allowed."
)
def extract_failing_together_config_group() -> None:
try:
utils.extract_file(
os.path.join("data", test_scheduling.FAILING_TOGETHER_CONFIG_GROUP_DB)
)
logger.info("Failing together config/group DB extracted.")
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info(
"Failing together config/group DB not extracted, but missing models are allowed."
)
def extract_past_failures_group() -> None:
try:
utils.extract_file(
os.path.join("data", test_scheduling.PAST_FAILURES_GROUP_DB)
)
logger.info("Group-level past failures DB extracted.")
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info(
"Group-level past failures DB not extracted, but missing models are allowed."
)
def extract_touched_together() -> None:
try:
utils.extract_file(
os.path.join("data", test_scheduling.TOUCHED_TOGETHER_DB)
)
logger.info("Touched together DB extracted.")
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info(
"Touched together DB not extracted, but missing models are allowed."
)
def extract_commits() -> bool:
try:
utils.extract_file(f"{repository.COMMITS_DB}.zst")
logger.info("Commits DB extracted.")
return True
except FileNotFoundError:
logger.info("Commits DB not extracted, but missing models are allowed.")
assert ALLOW_MISSING_MODELS
return False
def extract_commit_experiences() -> None:
try:
utils.extract_file(os.path.join("data", repository.COMMIT_EXPERIENCES_DB))
logger.info("Commit experiences DB extracted.")
except FileNotFoundError:
logger.info(
"Commit experiences DB not extracted, but missing models are allowed."
)
assert ALLOW_MISSING_MODELS
@tenacity.retry(
stop=tenacity.stop_after_attempt(7),
wait=tenacity.wait_exponential(multiplier=1, min=1, max=8),
)
def retrieve_schedulable_tasks() -> None:
r = requests.get(
"https://hg.mozilla.org/integration/autoland/json-pushes?version=2&tipsonly=1"
)
r.raise_for_status()
revs = [
push_obj["changesets"][0]
for push_id, push_obj in r.json()["pushes"].items()
]
logger.info(f"Retrieving known tasks from {revs}")
# Store in a file the list of tasks in the latest autoland pushes.
# We use more than one to protect ourselves from broken decision tasks.
known_tasks = set()
for rev in revs:
r = requests.get(
f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/target-tasks.json"
)
if r.ok:
known_tasks.update(r.json())
logger.info(f"Retrieved {len(known_tasks)} tasks")
assert len(known_tasks) > 0
with open("known_tasks", "w") as f:
f.write("\n".join(known_tasks))
with concurrent.futures.ThreadPoolExecutor() as executor:
clone_autoland_future = executor.submit(clone_autoland)
retrieve_schedulable_tasks_future = executor.submit(retrieve_schedulable_tasks)
commits_db_extracted = extract_commits()
extract_commit_experiences()
extract_touched_together()
extract_past_failures_label()
extract_past_failures_group()
extract_failing_together_label()
extract_failing_together_config_group()
if commits_db_extracted:
# Update the commits DB.
logger.info("Browsing all commits...")
for commit in repository.get_commits():
pass
logger.info("All commits browsed.")
# Wait repository to be cloned, as it's required to call repository.download_commits.
logger.info("Waiting autoland to be cloned...")
clone_autoland_future.result()
rev_start = "children({})".format(commit["node"])
logger.info("Updating commits DB...")
commits = repository.download_commits(
REPO_DIR, rev_start, use_single_process=True
)
logger.info("Commits DB updated.")
logger.info("Updating touched together DB...")
if len(commits) > 0:
# Update the touched together DB.
update_touched_together_gen = test_scheduling.update_touched_together()
next(update_touched_together_gen)
update_touched_together_gen.send(commits[-1]["node"])
try:
update_touched_together_gen.send(None)
except StopIteration:
pass
logger.info("Touched together DB updated.")
# Wait list of schedulable tasks to be downloaded and written to disk.
retrieve_schedulable_tasks_future.result()
logger.info("Worker boot done")
def loop_client_recv(self):
# The built-in client loop_forever seems busted (it doesn't retry
# under all exceptions, so just do it ourselves...); arg...
def on_connect(client, userdata, flags, rc):
if rc == mqtt.MQTT_ERR_SUCCESS:
self.log.info("MQTT connected to %s:%s over %s",
self.config['firehose_host'],
self.config['firehose_port'],
self.config['firehose_transport'])
client.subscribe('#')
else:
self.log.error(
"MQTT not connected to %s:%s over %s, rc=%s",
self.config['firehose_host'],
self.config['firehose_port'],
self.config['firehose_transport'], rc)
def on_message(client, userdata, msg):
if not msg.topic or not msg.payload:
return
self.log.info(("Dispatching message on topic=%s"
" with payload=%s"), msg.topic, msg.payload)
try:
payload = msg.payload
if isinstance(payload, six.binary_type):
payload = payload.decode("utf8")
details = {'event': json.loads(payload)}
except (UnicodeError, ValueError):
self.log.exception(
"Received corrupted/invalid payload: %s", msg.payload)
else:
self.work_queue.put(details)
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=1, max=30),
before=tenacity.before_log(self.log, logging.INFO))
def loop_forever_until_dead():
if self.dying:
return
client = mqtt.Client(transport=self.config['firehose_transport'])
client.on_connect = on_connect
client.on_message = on_message
try:
client.connect(self.config['firehose_host'],
port=int(self.config['firehose_port']))
max_timeout = 1
while not self.dying:
rc = mqtt.MQTT_ERR_SUCCESS
start = time.time()
elapsed = 0
while rc == mqtt.MQTT_ERR_SUCCESS and (elapsed < max_timeout):
rc = client.loop(timeout=max(0, max_timeout - elapsed))
elapsed = time.time() - start
if not self.dying:
time.sleep(0.1)
except Exception:
self.log.exception("Failed mqtt client usage, retrying")
raise
loop_forever_until_dead()
class Backend(ovs_idl.Backend):
lookup_table = {}
ovsdb_connection = None
def __init__(self, connection):
self.ovsdb_connection = connection
super(Backend, self).__init__(connection)
def start_connection(self, connection):
try:
self.ovsdb_connection.start()
except Exception as e:
connection_exception = OvsdbConnectionUnavailable(
db_schema=self.schema, error=e)
LOG.exception(connection_exception)
raise connection_exception
@property
def idl(self):
return self.ovsdb_connection.idl
@property
def tables(self):
return self.idl.tables
_tables = tables
@n_utils.classproperty
def connection_string(cls):
raise NotImplementedError()
@n_utils.classproperty
def schema_helper(cls):
# SchemaHelper.get_idl_schema() sets schema_json to None which is
# called in Idl.__init__(), so if we've done that return new helper
try:
if cls._schema_helper.schema_json:
return cls._schema_helper
except AttributeError:
pass
ovsdb_monitor._check_and_set_ssl_files(cls.schema)
cls._schema_helper = idlutils.get_schema_helper(cls.connection_string,
cls.schema)
return cls._schema_helper
@classmethod
def schema_has_table(cls, table_name):
return table_name in cls.schema_helper.schema_json['tables']
def is_table_present(self, table_name):
return table_name in self._tables
def is_col_present(self, table_name, col_name):
return self.is_table_present(table_name) and (
col_name in self._tables[table_name].columns)
def create_transaction(self, check_error=False, log_errors=True):
return idl_trans.Transaction(
self, self.ovsdb_connection, self.ovsdb_connection.timeout,
check_error, log_errors)
# Check for a column match in the table. If not found do a retry with
# a stop delay of 10 secs. This function would be useful if the caller
# wants to verify for the presence of a particular row in the table
# with the column match before doing any transaction.
# Eg. We can check if Logical_Switch row is present before adding a
# logical switch port to it.
@tenacity.retry(retry=tenacity.retry_if_exception_type(RuntimeError),
wait=tenacity.wait_exponential(),
stop=tenacity.stop_after_delay(10),
reraise=True)
def check_for_row_by_value_and_retry(self, table, column, match):
try:
idlutils.row_by_value(self.idl, table, column, match)
except idlutils.RowNotFound:
msg = (_("%(match)s does not exist in %(column)s of %(table)s")
% {'match': match, 'column': column, 'table': table})
raise RuntimeError(msg)
def request(self,
method,
additional_headers=None,
retry=True,
timeout=None,
auth=None,
use_gzip_encoding=None,
params=None,
max_attempts=None,
**kwargs):
"""
Make an HTTP request by calling self._request with backoff retry.
:param method: request method
:type method: str
:param additional_headers: additional headers to include in the request
:type additional_headers: dict[str, str]
:param retry: boolean indicating whether to retry if the request fails
:type retry: boolean
:param timeout: timeout in seconds, overrides default_timeout_secs
:type timeout: float
:param timeout: timeout in seconds
:type timeout: float
:param auth: auth scheme for the request
:type auth: requests.auth.AuthBase
:param use_gzip_encoding: boolean indicating whether to pass gzip
encoding in the request headers or not
:type use_gzip_encoding: boolean | None
:param params: additional params to include in the request
:type params: str | dict[str, T] | None
:param max_attempts: maximum number of attempts to try for any request
:type max_attempts: int
:param kwargs: additional arguments to pass to requests.request
:type kwargs: dict[str, T]
:return: HTTP response
:rtype: requests.Response
"""
request = self._request
if retry:
if max_attempts is None:
max_attempts = self.default_max_attempts
# We retry only when it makes sense: either due to a network
# partition (e.g. connection errors) or if the request failed
# due to a server error such as 500s, timeouts, and so on.
request = tenacity.retry(
stop=tenacity.stop_after_attempt(max_attempts),
wait=tenacity.wait_exponential(),
retry=tenacity.retry_if_exception_type((
requests.exceptions.Timeout,
requests.exceptions.ConnectionError,
MesosServiceUnavailableException,
MesosInternalServerErrorException,
)),
reraise=True,
)(request)
try:
return request(
method=method,
additional_headers=additional_headers,
timeout=timeout,
auth=auth,
use_gzip_encoding=use_gzip_encoding,
params=params,
**kwargs
)
# If the request itself failed, an exception subclassed from
# RequestException will be raised. Catch this and reraise as
# MesosException since we want the caller to be able to catch
# and handle this.
except requests.exceptions.RequestException as err:
raise MesosException('Request failed', err)
class PodManager(LoggingMixin):
"""
Helper class for creating, monitoring, and otherwise interacting with Kubernetes pods
for use with the KubernetesPodOperator
"""
def __init__(
self,
kube_client: client.CoreV1Api = None,
in_cluster: bool = True,
cluster_context: Optional[str] = None,
):
"""
Creates the launcher.
:param kube_client: kubernetes client
:param in_cluster: whether we are in cluster
:param cluster_context: context of the cluster
"""
super().__init__()
self._client = kube_client or get_kube_client(
in_cluster=in_cluster, cluster_context=cluster_context)
self._watch = watch.Watch()
def run_pod_async(self, pod: V1Pod, **kwargs) -> V1Pod:
"""Runs POD asynchronously"""
sanitized_pod = self._client.api_client.sanitize_for_serialization(pod)
json_pod = json.dumps(sanitized_pod, indent=2)
self.log.debug('Pod Creation Request: \n%s', json_pod)
try:
resp = self._client.create_namespaced_pod(
body=sanitized_pod, namespace=pod.metadata.namespace, **kwargs)
self.log.debug('Pod Creation Response: %s', resp)
except Exception as e:
self.log.exception(
'Exception when attempting to create Namespaced Pod: %s',
str(json_pod).replace("\n", " "))
raise e
return resp
def delete_pod(self, pod: V1Pod) -> None:
"""Deletes POD"""
try:
self._client.delete_namespaced_pod(pod.metadata.name,
pod.metadata.namespace,
body=client.V1DeleteOptions())
except ApiException as e:
# If the pod is already deleted
if e.status != 404:
raise
@tenacity.retry(
stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_random_exponential(),
reraise=True,
retry=tenacity.retry_if_exception(should_retry_start_pod),
)
def create_pod(self, pod: V1Pod) -> V1Pod:
"""Launches the pod asynchronously."""
return self.run_pod_async(pod)
def await_pod_start(self, pod: V1Pod, startup_timeout: int = 120) -> None:
"""
Waits for the pod to reach phase other than ``Pending``
:param pod:
:param startup_timeout: Timeout (in seconds) for startup of the pod
(if pod is pending for too long, fails task)
:return:
"""
curr_time = datetime.now()
while True:
remote_pod = self.read_pod(pod)
if remote_pod.status.phase != PodPhase.PENDING:
break
self.log.warning("Pod not yet started: %s", pod.metadata.name)
delta = datetime.now() - curr_time
if delta.total_seconds() >= startup_timeout:
msg = (
f"Pod took longer than {startup_timeout} seconds to start. "
"Check the pod events in kubernetes to determine why.")
raise PodLaunchFailedException(msg)
time.sleep(1)
def follow_container_logs(self, pod: V1Pod,
container_name: str) -> PodLoggingStatus:
warnings.warn(
"Method `follow_container_logs` is deprecated. Use `fetch_container_logs` instead"
"with option `follow=True`.",
DeprecationWarning,
)
return self.fetch_container_logs(pod=pod,
container_name=container_name,
follow=True)
def fetch_container_logs(
self,
pod: V1Pod,
container_name: str,
*,
follow=False,
since_time: Optional[DateTime] = None) -> PodLoggingStatus:
"""
Follows the logs of container and streams to airflow logging.
Returns when container exits.
"""
def consume_logs(*,
since_time: Optional[DateTime] = None,
follow: bool = True) -> Optional[DateTime]:
"""
Tries to follow container logs until container completes.
For a long-running container, sometimes the log read may be interrupted
Such errors of this kind are suppressed.
Returns the last timestamp observed in logs.
"""
timestamp = None
try:
logs = self.read_pod_logs(
pod=pod,
container_name=container_name,
timestamps=True,
since_seconds=(math.ceil(
(pendulum.now() -
since_time).total_seconds()) if since_time else None),
follow=follow,
)
for line in logs:
timestamp, message = self.parse_log_line(
line.decode('utf-8'))
self.log.info(message)
except BaseHTTPError as e:
self.log.warning(
"Reading of logs interrupted with error %r; will retry. "
"Set log level to DEBUG for traceback.",
e,
)
self.log.debug(
"Traceback for interrupted logs read for pod %r",
pod.metadata.name,
exc_info=True,
)
return timestamp or since_time
# note: `read_pod_logs` follows the logs, so we shouldn't necessarily *need* to
# loop as we do here. But in a long-running process we might temporarily lose connectivity.
# So the looping logic is there to let us resume following the logs.
last_log_time = since_time
while True:
last_log_time = consume_logs(since_time=last_log_time,
follow=follow)
if not self.container_is_running(pod,
container_name=container_name):
return PodLoggingStatus(running=False,
last_log_time=last_log_time)
if not follow:
return PodLoggingStatus(running=True,
last_log_time=last_log_time)
else:
self.log.warning(
'Pod %s log read interrupted but container %s still running',
pod.metadata.name,
container_name,
)
time.sleep(1)
def await_container_completion(self, pod: V1Pod,
container_name: str) -> None:
while not self.container_is_running(pod=pod,
container_name=container_name):
time.sleep(1)
def await_pod_completion(self, pod: V1Pod) -> V1Pod:
"""
Monitors a pod and returns the final state
:param pod: pod spec that will be monitored
:return: Tuple[State, Optional[str]]
"""
while True:
remote_pod = self.read_pod(pod)
if remote_pod.status.phase in PodPhase.terminal_states:
break
self.log.info('Pod %s has phase %s', pod.metadata.name,
remote_pod.status.phase)
time.sleep(2)
return remote_pod
def parse_log_line(self, line: str) -> Tuple[Optional[DateTime], str]:
"""
Parse K8s log line and returns the final state
:param line: k8s log line
:return: timestamp and log message
:rtype: Tuple[str, str]
"""
split_at = line.find(' ')
if split_at == -1:
self.log.error(
f"Error parsing timestamp (no timestamp in message '${line}'). "
"Will continue execution but won't update timestamp")
return None, line
timestamp = line[:split_at]
message = line[split_at + 1:].rstrip()
try:
last_log_time = cast(DateTime, pendulum.parse(timestamp))
except ParserError:
self.log.error(
"Error parsing timestamp. Will continue execution but won't update timestamp"
)
return None, line
return last_log_time, message
def container_is_running(self, pod: V1Pod, container_name: str) -> bool:
"""Reads pod and checks if container is running"""
remote_pod = self.read_pod(pod)
return container_is_running(pod=remote_pod,
container_name=container_name)
@tenacity.retry(stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_exponential(),
reraise=True)
def read_pod_logs(
self,
pod: V1Pod,
container_name: str,
tail_lines: Optional[int] = None,
timestamps: bool = False,
since_seconds: Optional[int] = None,
follow=True,
) -> Iterable[bytes]:
"""Reads log from the POD"""
additional_kwargs = {}
if since_seconds:
additional_kwargs['since_seconds'] = since_seconds
if tail_lines:
additional_kwargs['tail_lines'] = tail_lines
try:
return self._client.read_namespaced_pod_log(
name=pod.metadata.name,
namespace=pod.metadata.namespace,
container=container_name,
follow=follow,
timestamps=timestamps,
_preload_content=False,
**additional_kwargs,
)
except BaseHTTPError:
self.log.exception(
'There was an error reading the kubernetes API.')
raise
@tenacity.retry(stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_exponential(),
reraise=True)
def read_pod_events(self, pod: V1Pod) -> "CoreV1EventList":
"""Reads events from the POD"""
try:
return self._client.list_namespaced_event(
namespace=pod.metadata.namespace,
field_selector=f"involvedObject.name={pod.metadata.name}")
except BaseHTTPError as e:
raise AirflowException(
f'There was an error reading the kubernetes API: {e}')
@tenacity.retry(stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_exponential(),
reraise=True)
def read_pod(self, pod: V1Pod) -> V1Pod:
"""Read POD information"""
try:
return self._client.read_namespaced_pod(pod.metadata.name,
pod.metadata.namespace)
except BaseHTTPError as e:
raise AirflowException(
f'There was an error reading the kubernetes API: {e}')
def extract_xcom(self, pod: V1Pod) -> str:
"""Retrieves XCom value and kills xcom sidecar container"""
with closing(
kubernetes_stream(
self._client.connect_get_namespaced_pod_exec,
pod.metadata.name,
pod.metadata.namespace,
container=PodDefaults.SIDECAR_CONTAINER_NAME,
command=['/bin/sh'],
stdin=True,
stdout=True,
stderr=True,
tty=False,
_preload_content=False,
)) as resp:
result = self._exec_pod_command(
resp, f'cat {PodDefaults.XCOM_MOUNT_PATH}/return.json')
self._exec_pod_command(resp, 'kill -s SIGINT 1')
if result is None:
raise AirflowException(
f'Failed to extract xcom from pod: {pod.metadata.name}')
return result
def _exec_pod_command(self, resp, command: str) -> Optional[str]:
if resp.is_open():
self.log.info('Running command... %s\n', command)
resp.write_stdin(command + '\n')
while resp.is_open():
resp.update(timeout=1)
if resp.peek_stdout():
return resp.read_stdout()
if resp.peek_stderr():
self.log.info("stderr from command: %s",
resp.read_stderr())
break
return None
class MergifyContext(object):
client = attr.ib()
pull = attr.ib()
_consolidated_data = attr.ib(init=False, default=None)
@property
def log(self):
return daiquiri.getLogger(
__name__,
gh_owner=self.pull["user"]["login"]
if "user" in self.pull else "<unknown-yet>",
gh_repo=(self.pull["base"]["repo"]["name"]
if "base" in self.pull else "<unknown-yet>"),
gh_private=(self.pull["base"]["repo"]["private"]
if "base" in self.pull else "<unknown-yet>"),
gh_branch=self.pull["base"]["ref"]
if "base" in self.pull else "<unknown-yet>",
gh_pull=self.pull["number"],
gh_pull_url=self.pull.get("html_url", "<unknown-yet>"),
gh_pull_state=("merged" if self.pull.get("merged") else
(self.pull.get("mergeable_state", "unknown")
or "none")),
)
def __attrs_post_init__(self):
self._ensure_complete()
def has_write_permissions(self, login):
# TODO(sileht): We should cache that, this is also used in command runner
return self.client.item(
f"collaborators/{login}/permission")["permission"] in [
"admin",
"write",
]
def _get_valid_users(self):
bots = list(
set([
r["user"]["login"] for r in self.reviews
if r["user"]["type"] == "Bot"
]))
collabs = set([
r["user"]["login"] for r in self.reviews
if r["user"]["type"] != "Bot"
])
valid_collabs = [
login for login in collabs if self.has_write_permissions(login)
]
return bots + valid_collabs
@functools_bp.cached_property
def consolidated_reviews(self):
# Ignore reviews that are not from someone with admin/write permissions
# And only keep the last review for each user.
comments = dict()
approvals = dict()
valid_users = self._get_valid_users()
for review in self.reviews:
if review["user"]["login"] not in valid_users:
continue
# Only keep latest review of an user
if review["state"] == "COMMENTED":
comments[review["user"]["login"]] = review
else:
approvals[review["user"]["login"]] = review
return list(comments.values()), list(approvals.values())
def to_dict(self):
if self._consolidated_data is None:
self._consolidated_data = self._get_consolidated_data()
return self._consolidated_data
def _get_consolidated_data(self):
comments, approvals = self.consolidated_reviews
return {
# Can be used by rules too
"assignee": [a["login"] for a in self.pull["assignees"]],
# NOTE(sileht): We put an empty label to allow people to match
# no label set
"label": [l["name"] for l in self.pull["labels"]],
"review-requested":
([u["login"] for u in self.pull["requested_reviewers"]] +
["@" + t["slug"] for t in self.pull["requested_teams"]]),
"author":
self.pull["user"]["login"],
"merged-by": (self.pull["merged_by"]["login"]
if self.pull["merged_by"] else ""),
"merged":
self.pull["merged"],
"closed":
self.pull["state"] == "closed",
"milestone": (self.pull["milestone"]["title"]
if self.pull["milestone"] else ""),
"conflict":
self.pull["mergeable_state"] == "dirty",
"base":
self.pull["base"]["ref"],
"head":
self.pull["head"]["ref"],
"locked":
self.pull["locked"],
"title":
self.pull["title"],
"body":
self.pull["body"],
"files": [f["filename"] for f in self.files],
"approved-reviews-by": [
r["user"]["login"] for r in approvals
if r["state"] == "APPROVED"
],
"dismissed-reviews-by": [
r["user"]["login"] for r in approvals
if r["state"] == "DISMISSED"
],
"changes-requested-reviews-by": [
r["user"]["login"] for r in approvals
if r["state"] == "CHANGES_REQUESTED"
],
"commented-reviews-by": [
r["user"]["login"] for r in comments
if r["state"] == "COMMENTED"
],
# NOTE(jd) The Check API set conclusion to None for pending.
# NOTE(sileht): "pending" statuses are not really trackable, we
# voluntary drop this event because CIs just sent they status every
# minutes until the CI pass (at least Travis and Circle CI does
# that). This was causing a big load on Mergify for nothing useful
# tracked, and on big projects it can reach the rate limit very
# quickly.
"status-success": [
name for name, state in self.checks.items()
if state == "success"
],
"status-failure": [
name for name, state in self.checks.items()
if state == "failure"
],
"status-neutral": [
name for name, state in self.checks.items()
if state == "neutral"
],
# NOTE(sileht): Not handled for now
# cancelled, timed_out, or action_required
}
@functools_bp.cached_property
def checks(self):
# NOTE(sileht): conclusion can be one of success, failure, neutral,
# cancelled, timed_out, or action_required, and None for "pending"
checks = dict(
(c["name"], c["conclusion"]) for c in check_api.get_checks(self))
# NOTE(sileht): state can be one of error, failure, pending,
# or success.
checks.update((s["context"], s["state"]) for s in self.client.items(
f"commits/{self.pull['head']['sha']}/status",
list_items="statuses"))
return checks
def _resolve_login(self, name):
if not name:
return []
elif not isinstance(name, str):
return [name]
elif name[0] != "@":
return [name]
if "/" in name:
organization, _, team_slug = name.partition("/")
if not team_slug or "/" in team_slug:
# Not a team slug
return [name]
organization = organization[1:]
else:
organization = self.pull["base"]["repo"]["owner"]["login"]
team_slug = name[1:]
try:
return [
member["login"] for member in self.client.items(
f"/orgs/{organization}/teams/{team_slug}/members")
]
except httpx.HTTPClientSideError as e:
self.log.warning(
"fail to get the organization, team or members",
team=name,
status=e.status_code,
detail=e.message,
)
return [name]
def resolve_teams(self, values):
if not values:
return []
if not isinstance(values, (list, tuple)):
values = [values]
values = list(
itertools.chain.from_iterable((map(self._resolve_login, values))))
return values
UNUSABLE_STATES = ["unknown", None]
# NOTE(sileht): quickly retry, if we don't get the status on time
# the exception is recatch in worker.py, so celery will retry it later
@tenacity.retry(
wait=tenacity.wait_exponential(multiplier=0.2),
stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_if_exception_type(
exceptions.MergeableStateUnknown),
reraise=True,
)
def _ensure_complete(self):
if not (self._is_data_complete()
and self._is_background_github_processing_completed()):
self.pull = self.client.item(f"pulls/{self.pull['number']}")
if not self._is_data_complete():
self.log.error(
"/pulls/%s has returned an incomplete payload...",
self.pull["number"],
data=self.pull,
)
if self._is_background_github_processing_completed():
return
raise exceptions.MergeableStateUnknown(self)
def _is_data_complete(self):
# NOTE(sileht): If pull request come from /pulls listing or check-runs sometimes,
# they are incomplete, This ensure we have the complete view
fields_to_control = (
"state",
"mergeable_state",
"merged_by",
"merged",
"merged_at",
)
for field in fields_to_control:
if field not in self.pull:
return False
return True
def _is_background_github_processing_completed(self):
return (self.pull["state"] == "closed"
or self.pull["mergeable_state"] not in self.UNUSABLE_STATES)
def update(self):
# TODO(sileht): Remove me,
# Don't use it, because consolidated data are not updated after that.
# Only used by merge action for posting an update report after rebase.
self.pull = self.client.item(f"pulls/{self.pull['number']}")
@functools_bp.cached_property
def is_behind(self):
branch_name_escaped = parse.quote(self.pull["base"]["ref"], safe="")
branch = self.client.item(f"branches/{branch_name_escaped}")
for commit in self.commits:
for parent in commit["parents"]:
if parent["sha"] == branch["commit"]["sha"]:
return False
return True
def get_merge_commit_message(self):
if not self.pull["body"]:
return
found = False
message_lines = []
for line in self.pull["body"].split("\n"):
if MARKDOWN_COMMIT_MESSAGE_RE.match(line):
found = True
elif found and MARKDOWN_TITLE_RE.match(line):
break
elif found:
message_lines.append(line)
if found and message_lines:
return {
"commit_title": message_lines[0],
"commit_message": "\n".join(message_lines[1:]).strip(),
}
def __str__(self):
return "%(login)s/%(repo)s/pull/%(number)d@%(branch)s " "s:%(pr_state)s" % {
"login":
self.pull["base"]["user"]["login"],
"repo":
self.pull["base"]["repo"]["name"],
"number":
self.pull["number"],
"branch":
self.pull["base"]["ref"],
"pr_state": ("merged" if self.pull["merged"] else
(self.pull["mergeable_state"] or "none")),
}
@functools_bp.cached_property
def reviews(self):
return list(self.client.items(f"pulls/{self.pull['number']}/reviews"))
@functools_bp.cached_property
def commits(self):
return list(self.client.items(f"pulls/{self.pull['number']}/commits"))
@functools_bp.cached_property
def files(self):
return list(self.client.items(f"pulls/{self.pull['number']}/files"))
@property
def pull_from_fork(self):
return self.pull["head"]["repo"]["id"] != self.pull["base"]["repo"][
"id"]
@property
def pull_base_is_modifiable(self):
return self.pull["maintainer_can_modify"] or not self.pull_from_fork
class PodLauncher(LoggingMixin):
"""Deprecated class for launching pods. please use
airflow.providers.cncf.kubernetes.utils.pod_launcher.PodLauncher instead
"""
def __init__(
self,
kube_client: client.CoreV1Api = None,
in_cluster: bool = True,
cluster_context: Optional[str] = None,
extract_xcom: bool = False,
):
"""
Deprecated class for launching pods. please use
airflow.providers.cncf.kubernetes.utils.pod_launcher.PodLauncher instead
Creates the launcher.
:param kube_client: kubernetes client
:param in_cluster: whether we are in cluster
:param cluster_context: context of the cluster
:param extract_xcom: whether we should extract xcom
"""
super().__init__()
self._client = kube_client or get_kube_client(
in_cluster=in_cluster, cluster_context=cluster_context)
self._watch = watch.Watch()
self.extract_xcom = extract_xcom
def run_pod_async(self, pod: V1Pod, **kwargs):
"""Runs POD asynchronously"""
pod_mutation_hook(pod)
sanitized_pod = self._client.api_client.sanitize_for_serialization(pod)
json_pod = json.dumps(sanitized_pod, indent=2)
self.log.debug('Pod Creation Request: \n%s', json_pod)
try:
resp = self._client.create_namespaced_pod(
body=sanitized_pod, namespace=pod.metadata.namespace, **kwargs)
self.log.debug('Pod Creation Response: %s', resp)
except Exception as e:
self.log.exception(
'Exception when attempting to create Namespaced Pod: %s',
json_pod)
raise e
return resp
def delete_pod(self, pod: V1Pod):
"""Deletes POD"""
try:
self._client.delete_namespaced_pod(pod.metadata.name,
pod.metadata.namespace,
body=client.V1DeleteOptions())
except ApiException as e:
# If the pod is already deleted
if e.status != 404:
raise
def start_pod(self, pod: V1Pod, startup_timeout: int = 120):
"""
Launches the pod synchronously and waits for completion.
:param pod:
:param startup_timeout: Timeout for startup of the pod (if pod is pending for too long, fails task)
:return:
"""
resp = self.run_pod_async(pod)
curr_time = dt.now()
if resp.status.start_time is None:
while self.pod_not_started(pod):
self.log.warning("Pod not yet started: %s", pod.metadata.name)
delta = dt.now() - curr_time
if delta.total_seconds() >= startup_timeout:
raise AirflowException("Pod took too long to start")
time.sleep(1)
def monitor_pod(self, pod: V1Pod,
get_logs: bool) -> Tuple[State, Optional[str]]:
"""
Monitors a pod and returns the final state
:param pod: pod spec that will be monitored
:type pod : V1Pod
:param get_logs: whether to read the logs locally
:return: Tuple[State, Optional[str]]
"""
if get_logs:
read_logs_since_sec = None
last_log_time = None
while True:
logs = self.read_pod_logs(pod,
timestamps=True,
since_seconds=read_logs_since_sec)
for line in logs:
timestamp, message = self.parse_log_line(
line.decode('utf-8'))
last_log_time = pendulum.parse(timestamp)
self.log.info(message)
time.sleep(1)
if not self.base_container_is_running(pod):
break
self.log.warning('Pod %s log read interrupted',
pod.metadata.name)
if last_log_time:
delta = pendulum.now() - last_log_time
# Prefer logs duplication rather than loss
read_logs_since_sec = math.ceil(delta.total_seconds())
result = None
if self.extract_xcom:
while self.base_container_is_running(pod):
self.log.info('Container %s has state %s', pod.metadata.name,
State.RUNNING)
time.sleep(2)
result = self._extract_xcom(pod)
self.log.info(result)
result = json.loads(result)
while self.pod_is_running(pod):
self.log.info('Pod %s has state %s', pod.metadata.name,
State.RUNNING)
time.sleep(2)
return self._task_status(self.read_pod(pod)), result
def parse_log_line(self, line: str) -> Tuple[str, str]:
"""
Parse K8s log line and returns the final state
:param line: k8s log line
:type line: str
:return: timestamp and log message
:rtype: Tuple[str, str]
"""
split_at = line.find(' ')
if split_at == -1:
raise Exception(
f'Log not in "{{timestamp}} {{log}}" format. Got: {line}')
timestamp = line[:split_at]
message = line[split_at + 1:].rstrip()
return timestamp, message
def _task_status(self, event):
self.log.info('Event: %s had an event of type %s', event.metadata.name,
event.status.phase)
status = self.process_status(event.metadata.name, event.status.phase)
return status
def pod_not_started(self, pod: V1Pod):
"""Tests if pod has not started"""
state = self._task_status(self.read_pod(pod))
return state == State.QUEUED
def pod_is_running(self, pod: V1Pod):
"""Tests if pod is running"""
state = self._task_status(self.read_pod(pod))
return state not in (State.SUCCESS, State.FAILED)
def base_container_is_running(self, pod: V1Pod):
"""Tests if base container is running"""
event = self.read_pod(pod)
status = next(
iter(
filter(lambda s: s.name == 'base',
event.status.container_statuses)), None)
if not status:
return False
return status.state.running is not None
@tenacity.retry(stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_exponential(),
reraise=True)
def read_pod_logs(
self,
pod: V1Pod,
tail_lines: Optional[int] = None,
timestamps: bool = False,
since_seconds: Optional[int] = None,
):
"""Reads log from the POD"""
additional_kwargs = {}
if since_seconds:
additional_kwargs['since_seconds'] = since_seconds
if tail_lines:
additional_kwargs['tail_lines'] = tail_lines
try:
return self._client.read_namespaced_pod_log(
name=pod.metadata.name,
namespace=pod.metadata.namespace,
container='base',
follow=True,
timestamps=timestamps,
_preload_content=False,
**additional_kwargs,
)
except BaseHTTPError as e:
raise AirflowException(
f'There was an error reading the kubernetes API: {e}')
@tenacity.retry(stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_exponential(),
reraise=True)
def read_pod_events(self, pod):
"""Reads events from the POD"""
try:
return self._client.list_namespaced_event(
namespace=pod.metadata.namespace,
field_selector=f"involvedObject.name={pod.metadata.name}")
except BaseHTTPError as e:
raise AirflowException(
f'There was an error reading the kubernetes API: {e}')
@tenacity.retry(stop=tenacity.stop_after_attempt(3),
wait=tenacity.wait_exponential(),
reraise=True)
def read_pod(self, pod: V1Pod):
"""Read POD information"""
try:
return self._client.read_namespaced_pod(pod.metadata.name,
pod.metadata.namespace)
except BaseHTTPError as e:
raise AirflowException(
f'There was an error reading the kubernetes API: {e}')
def _extract_xcom(self, pod: V1Pod):
resp = kubernetes_stream(
self._client.connect_get_namespaced_pod_exec,
pod.metadata.name,
pod.metadata.namespace,
container=PodDefaults.SIDECAR_CONTAINER_NAME,
command=['/bin/sh'],
stdin=True,
stdout=True,
stderr=True,
tty=False,
_preload_content=False,
)
try:
result = self._exec_pod_command(
resp, f'cat {PodDefaults.XCOM_MOUNT_PATH}/return.json')
self._exec_pod_command(resp, 'kill -s SIGINT 1')
finally:
resp.close()
if result is None:
raise AirflowException(
f'Failed to extract xcom from pod: {pod.metadata.name}')
return result
def _exec_pod_command(self, resp, command):
if resp.is_open():
self.log.info('Running command... %s\n', command)
resp.write_stdin(command + '\n')
while resp.is_open():
resp.update(timeout=1)
if resp.peek_stdout():
return resp.read_stdout()
if resp.peek_stderr():
self.log.info(resp.read_stderr())
break
return None
def process_status(self, job_id, status):
"""Process status information for the JOB"""
status = status.lower()
if status == PodStatus.PENDING:
return State.QUEUED
elif status == PodStatus.FAILED:
self.log.error('Event with job id %s Failed', job_id)
return State.FAILED
elif status == PodStatus.SUCCEEDED:
self.log.info('Event with job id %s Succeeded', job_id)
return State.SUCCESS
elif status == PodStatus.RUNNING:
return State.RUNNING
else:
self.log.error('Event: Invalid state %s on job %s', status, job_id)
return State.FAILED
