def validate(self, max_retries=0):
        """Performs basic **connection** validation of a sqlalchemy engine."""

        def _retry_on_exception(exc):
            LOG.warning("Engine connection (validate) failed due to '%s'", exc)
            if isinstance(exc, sa_exc.OperationalError) and \
               _is_db_connection_error(six.text_type(exc.args[0])):
                # We may be able to fix this by retrying...
                return True
            if isinstance(exc, (sa_exc.TimeoutError,
                                sa_exc.ResourceClosedError,
                                sa_exc.DisconnectionError)):
                # We may be able to fix this by retrying...
                return True
            # Other failures we likely can't fix by retrying...
            return False

        @tenacity.retry(
            stop=tenacity.stop_after_attempt(max(0, int(max_retries))),
            wait=tenacity.wait_exponential(),
            reraise=True,
            retry=tenacity.retry_if_exception(_retry_on_exception)
        )
        def _try_connect(engine):
            # See if we can make a connection happen.
            #
            # NOTE(harlowja): note that even though we are connecting
            # once it does not mean that we will be able to connect in
            # the future, so this is more of a sanity test and is not
            # complete connection insurance.
            with contextlib.closing(engine.connect()):
                pass

        _try_connect(self._engine)
Example #2
0
def retry_upon_exception(exc, delay, max_delay, max_attempts):
    return tenacity.retry(reraise=True,
                          retry=tenacity.retry_if_exception_type(exc),
                          wait=tenacity.wait_exponential(
                                multiplier=delay, max=max_delay),
                          stop=tenacity.stop_after_attempt(max_attempts),
                          before=_log_before_retry, after=_log_after_retry)
Example #3
0
    def join_group(self, group_id):
        if (not self._coordinator or not self._coordinator.is_started
                or not group_id):
            return

        @tenacity.retry(
            wait=tenacity.wait_exponential(
                multiplier=self.conf.coordination.retry_backoff,
                max=self.conf.coordination.max_retry_interval),
            retry=tenacity.retry_if_exception_type(
                ErrorJoiningPartitioningGroup))
        def _inner():
            try:
                join_req = self._coordinator.join_group(group_id)
                join_req.get()
                LOG.info(_LI('Joined partitioning group %s'), group_id)
            except tooz.coordination.MemberAlreadyExist:
                return
            except tooz.coordination.GroupNotCreated:
                create_grp_req = self._coordinator.create_group(group_id)
                try:
                    create_grp_req.get()
                except tooz.coordination.GroupAlreadyExist:
                    pass
                raise ErrorJoiningPartitioningGroup()
            except tooz.coordination.ToozError:
                LOG.exception(_LE('Error joining partitioning group %s,'
                                  ' re-trying'), group_id)
                raise ErrorJoiningPartitioningGroup()
            self._groups.add(group_id)

        return _inner()
Example #4
0
 def wrapped(*args, **kwargs):
     self = args[0]
     new_fn = tenacity.retry(
         reraise=True,
         retry=tenacity.retry_if_result(_ofport_result_pending),
         wait=tenacity.wait_exponential(multiplier=0.01, max=1),
         stop=tenacity.stop_after_delay(
             self.vsctl_timeout))(fn)
     return new_fn(*args, **kwargs)
def get_ovn_idls(driver, trigger):
    @tenacity.retry(
        wait=tenacity.wait_exponential(max=180),
        reraise=True)
    def get_ovn_idl_retry(cls, driver, trigger):
        LOG.info(_LI('Getting %(cls)s for %(trigger)s with retry'),
                 {'cls': cls.__name__, 'trigger': trigger.im_class.__name__})
        return cls(driver, trigger)

    nb_ovn_idl = get_ovn_idl_retry(OvsdbNbOvnIdl, driver, trigger)
    sb_ovn_idl = get_ovn_idl_retry(OvsdbSbOvnIdl, driver, trigger)
    return nb_ovn_idl, sb_ovn_idl
Example #6
0
def retry_upon_none_result(max_attempts, delay=0.5, max_delay=2, random=False):
    if random:
        wait_func = tenacity.wait_exponential(
            multiplier=delay, max=max_delay)
    else:
        wait_func = tenacity.wait_random_exponential(
            multiplier=delay, max=max_delay)
    return tenacity.retry(reraise=True,
                          retry=tenacity.retry_if_result(lambda x: x is None),
                          wait=wait_func,
                          stop=tenacity.stop_after_attempt(max_attempts),
                          before=_log_before_retry, after=_log_after_retry)
Example #7
0
def get_schema_helper(connection, schema_name, retry=True):
    try:
        return _get_schema_helper(connection, schema_name)
    except Exception:
        with excutils.save_and_reraise_exception(reraise=False) as ctx:
            if not retry:
                ctx.reraise = True
            # We may have failed due to set-manager not being called
            helpers.enable_connection_uri(connection)

            # There is a small window for a race, so retry up to a second
            @tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.01),
                            stop=tenacity.stop_after_delay(1),
                            reraise=True)
            def do_get_schema_helper():
                return _get_schema_helper(connection, schema_name)

            return do_get_schema_helper()
    def get_schema_helper(self):
        """Retrieve the schema helper object from OVSDB"""
        # The implementation of this function is same as the base class method
        # without the enable_connection_uri() called (since ovs-vsctl won't
        # exist on the controller node when using the reference architecture).
        try:
            helper = idlutils.get_schema_helper(self.connection,
                                                self.schema_name)
        except Exception:
            # There is a small window for a race, so retry up to a second
            @tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.01),
                            stop=tenacity.stop_after_delay(1),
                            reraise=True)
            def do_get_schema_helper():
                return idlutils.get_schema_helper(self.connection,
                                                  self.schema_name)
            helper = do_get_schema_helper()

        return helper
Example #9
0
def idl_factory():
    conn = cfg.CONF.OVS.ovsdb_connection
    schema_name = 'Open_vSwitch'
    try:
        helper = idlutils.get_schema_helper(conn, schema_name)
    except Exception:
        helpers.enable_connection_uri(conn)

        @tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.01),
                        stop=tenacity.stop_after_delay(1),
                        reraise=True)
        def do_get_schema_helper():
            return idlutils.get_schema_helper(conn, schema_name)

        helper = do_get_schema_helper()

    # TODO(twilson) We should still select only the tables/columns we use
    helper.register_all()
    return idl.Idl(conn, helper)
Example #10
0
    def get_schema_helper(self):
        """Retrieve the schema helper object from OVSDB"""
        try:
            helper = idlutils.get_schema_helper(self.connection,
                                                self.schema_name)
        except Exception:
            # We may have failed do to set-manager not being called
            helpers.enable_connection_uri(self.connection)

            # There is a small window for a race, so retry up to a second
            @tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.01),
                            stop=tenacity.stop_after_delay(1),
                            reraise=True)
            def do_get_schema_helper():
                return idlutils.get_schema_helper(self.connection,
                                                  self.schema_name)
            helper = do_get_schema_helper()

        return helper
Example #11
0
def retry_upon_exception_exclude_error_codes(
    exc, excluded_errors, delay, max_delay, max_attempts):
    """Retry with the configured exponential delay, unless the exception error
    code is in the given list
    """
    def retry_if_not_error_codes(e):
        # return True only for BadRequests without error codes or with error
        # codes not in the exclude list
        if isinstance(e, exc):
            error_code = _get_bad_request_error_code(e)
            if error_code and error_code not in excluded_errors:
                return True
        return False

    return tenacity.retry(reraise=True,
                          retry=tenacity.retry_if_exception(
                                retry_if_not_error_codes),
                          wait=tenacity.wait_exponential(
                                multiplier=delay, max=max_delay),
                          stop=tenacity.stop_after_attempt(max_attempts),
                          before=_log_before_retry, after=_log_after_retry)
Example #12
0
class GitRepo(object):
    """
        Class for work with repositories
    """

    def __init__(self, root_repo_dir, repo_name, branch, url, commit_id=None):
        """
        :param root_repo_dir: Directory where repositories will clone
        :param repo_name: Name of repository
        :param branch: Branch of repository
        :param commit_id: Commit ID
        """

        self.repo_name = repo_name
        self.branch_name = branch
        self.url = url
        self.commit_id = commit_id
        self.local_repo_dir = root_repo_dir / repo_name
        self.repo = None

        self.log = logging.getLogger(self.__class__.__name__)

    def prepare_repo(self):
        """
        Preparing repository for build
        Include cloning and updating repo to remote state

        :return: None
        """
        self.log.info('-' * 50)
        self.log.info("Getting repo " + self.repo_name)

        self.clone()
        self.repo = git.Repo(str(self.local_repo_dir))
        self.hard_reset()
        self.clean()
        self.checkout(branch_name="master", silent=True)

    @retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
    def clone(self):
        """
        Clone repo

        :return: None
        """

        # checking correctness of git repository
        # if dir is not repository, it will be removed
        if self.local_repo_dir.exists():
            try:
                git.Repo(str(self.local_repo_dir))
            except git.InvalidGitRepositoryError:
                self.log.info('Remove broken repo %s', self.local_repo_dir)
                remove_directory(self.local_repo_dir)

        if not self.local_repo_dir.exists():
            self.log.info("Clone repo " + self.repo_name)
            git.Git().clone(self.url, str(self.local_repo_dir))

    @retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
    def fetch(self, branch_name=None):
        """
        Fetch repo

        :return: None
        """

        refname = branch_name or self.branch_name
        self.log.info("Fetch repo %s to %s", self.repo_name, refname)
        self.repo.remotes.origin.fetch(refname)
        self.hard_reset('FETCH_HEAD')

    @retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
    def hard_reset(self, reset_to=None):
        """
        Hard reset repo

        :param reset_to: Commit ID or branch. If None - hard reset to HEAD
        :return: None
        """

        self.log.info("Hard reset repo " + self.repo_name)
        if reset_to:
            self.repo.git.reset('--hard', reset_to)
        else:
            self.repo.git.reset('--hard')

    @retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
    def checkout(self, branch_name=None, silent=False):
        """
        Checkout to certain state

        :param branch_name: Branch of repo.
               If None - checkout to commit ID from class variable commit_id

        :param silent: Flag for getting time of commit
               (set to True only if commit_id does not exist)
        :type silent: Boolean

        :return: None
        """

        checkout_dest = branch_name or self.commit_id
        self.log.info("Checkout repo %s to %s", self.repo_name, checkout_dest)
        try:
            self.repo.git.checkout(checkout_dest, force=True)
        except git.exc.GitCommandError:
            self.log.exception("Remote branch %s does not exist", checkout_dest)

        if str(self.commit_id).lower() == 'head':
            self.commit_id = str(self.repo.head.commit)

        if not silent:
            # error raises after checkout to master if we try
            # to get time of triggered commit_id before fetching repo
            # (commit does not exist in local repository yet)
            committed_date = self.repo.commit(checkout_dest).committed_date
            self.log.info("Committed date: %s", datetime.fromtimestamp(committed_date))

    @retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
    def clean(self):
        """
        Clean repo

        :return: None
        """

        self.log.info("Clean repo " + self.repo_name)
        self.repo.git.clean('-xdf')

    @retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=60))
    def pull(self):
        """
        Pull repo
        :return: None
        """

        self.log.info("Pull repo " + self.repo_name)
        self.repo.git.pull()

    def change_repo_state(self, branch_name=None, commit_time=None):
        """
        Change the repo state

        :param branch_name: name of branch to checkout
        :param commit_time: time of commit
        :return: None
        """

        self.branch_name = branch_name or self.branch_name
        self.fetch()

        if branch_name:
            # Checkout to branch
            self.checkout(branch_name=self.branch_name)

        if commit_time:
            self.revert_commit_by_time(commit_time)
        # Checkout to commit id
        self.checkout()

    def revert_commit_by_time(self, commit_time):
        """
        Sets commit by time.
        If commit date <= certain time,
        commit sets to class variable commit_id.

        :param commit_time: timestamp
        :return: None
        """

        self.commit_id = str(next(self.repo.iter_commits(
            until=commit_time, max_count=1)))
        self.log.info(f"Revert commit by time to: {datetime.fromtimestamp(commit_time)}")

    def get_time(self, commit_id=None):
        """
        Get datetime of commit

        :param commit_id: Commit ID
        :return: datetime
        """

        commit = commit_id if commit_id else self.commit_id
        return self.repo.commit(commit).committed_date

    def is_branch_exist(self, branch_name):
        """
        Check if branch exists in repo

        :param branch_name: branch name
        :return: True if branch exists else false
        """
        # Need to fetch all to get remote branches
        self.repo.remotes.origin.fetch()
        if self.repo.git.branch('--list', f'*/{branch_name}', '--all'):
            return True
        return False
Example #13
0
    for rule in current_rules:
        if '-i %s' % vif in rule and '--among-src' in rule:
            ebtables(['-D', chain] + rule.split())


def _delete_mac_spoofing_protection(vifs, current_rules, table, chain):
    # delete the jump rule and then delete the whole chain
    jumps = [vif for vif in vifs if _mac_vif_jump_present(vif, current_rules)]
    for vif in jumps:
        ebtables(['-D', chain, '-i', vif, '-j',
                  _mac_chain_name(vif)],
                 table=table)
    for vif in vifs:
        chain = _mac_chain_name(vif)
        if chain_exists(chain, current_rules):
            ebtables(['-X', chain], table=table)


# Used to scope ebtables commands in testing
NAMESPACE = None


@tenacity.retry(
    wait=tenacity.wait_exponential(multiplier=0.01),
    retry=tenacity.retry_if_exception(lambda e: e.returncode == 255),
    reraise=True)
def ebtables(comm, table='nat'):
    execute = ip_lib.IPWrapper(NAMESPACE).netns.execute
    return execute(['ebtables', '-t', table, '--concurrent'] + comm,
                   run_as_root=True)
Example #14
0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Code for configuring swift."""

import logging
import tenacity

import zaza.openstack.utilities.openstack as openstack


@tenacity.retry(wait=tenacity.wait_exponential(multiplier=10, max=300),
                reraise=True, stop=tenacity.stop_after_attempt(10),
                retry=tenacity.retry_if_exception_type(AssertionError))
def wait_for_region2():
    """Ensure two regions are present."""
    keystone_session = openstack.get_overcloud_keystone_session()
    keystone_client = (
        openstack.get_keystone_session_client(
            keystone_session,
            client_api_version='3'))
    swift_svc_id = keystone_client.services.find(name='swift').id
    regions = set([ep.region
                   for ep in keystone_client.endpoints.list(swift_svc_id)])
    logging.info('Checking there are 2 regions. Current count is {}'.format(
        len(regions)))
    assert len(set(regions)) == 2, "Incorrect number of regions"
Example #15
0
        table.add_row("kubectl", self.kubectl_version)
        table.add_row("gcloud", self.gcloud_version)
        table.add_row("cloud_sql_proxy", self.cloud_sql_proxy_version)
        table.add_row("mysql", self.mysql_version)
        table.add_row("sqlite3", self.sqlite3_version)
        table.add_row("psql", self.psql_version)
        console.print(table)


class FileIoException(Exception):
    """Raises when error happens in FileIo.io integration"""


@tenacity.retry(
    stop=tenacity.stop_after_attempt(5),
    wait=tenacity.wait_exponential(multiplier=1, max=10),
    retry=tenacity.retry_if_exception_type(FileIoException),
    before=tenacity.before_log(log, logging.DEBUG),
    after=tenacity.after_log(log, logging.DEBUG),
)
def _upload_text_to_fileio(content):
    """Upload text file to File.io service and return lnk"""
    resp = requests.post("https://file.io", data={"text": content})
    if not resp.ok:
        print(resp.json())
        raise FileIoException("Failed to send report to file.io service.")
    try:
        return resp.json()["link"]
    except ValueError as e:
        log.debug(e)
        raise FileIoException("Failed to send report to file.io service.")
Example #16
0
class NovaClientPlugin(microversion_mixin.MicroversionMixin,
                       client_plugin.ClientPlugin):

    deferred_server_statuses = [
        'BUILD', 'HARD_REBOOT', 'PASSWORD', 'REBOOT', 'RESCUE', 'RESIZE',
        'REVERT_RESIZE', 'SHUTOFF', 'SUSPENDED', 'VERIFY_RESIZE'
    ]

    exceptions_module = exceptions

    NOVA_API_VERSION = '2.1'

    max_microversion = cfg.CONF.max_nova_api_microversion

    service_types = [COMPUTE] = ['compute']

    def _get_service_name(self):
        return self.COMPUTE

    def _create(self, version=None):
        if not version:
            # TODO(prazumovsky): remove all unexpected calls from tests and
            # add default_version after that.
            version = self.NOVA_API_VERSION
        args = self._get_args(version)

        client = nc.Client(version, **args)
        return client

    def _get_args(self, version):
        endpoint_type = self._get_client_option(CLIENT_NAME, 'endpoint_type')

        return {
            'session':
            self.context.keystone_session,
            'endpoint_type':
            endpoint_type,
            'service_type':
            self.COMPUTE,
            'region_name':
            self._get_region_name(),
            'connect_retries':
            cfg.CONF.client_retry_limit,
            'http_log_debug':
            self._get_client_option(CLIENT_NAME, 'http_log_debug')
        }

    def get_max_microversion(self):
        if not self.max_microversion:
            client = self._create()
            self.max_microversion = client.versions.get_current().version
        return self.max_microversion

    def is_version_supported(self, version):
        api_ver = api_versions.get_api_version(version)
        max_api_ver = api_versions.get_api_version(self.get_max_microversion())
        return max_api_ver >= api_ver

    def is_not_found(self, ex):
        return isinstance(ex, (exceptions.NotFound, q_exceptions.NotFound))

    def is_over_limit(self, ex):
        return isinstance(ex, exceptions.OverLimit)

    def is_bad_request(self, ex):
        return isinstance(ex, exceptions.BadRequest)

    def is_conflict(self, ex):
        return isinstance(ex, exceptions.Conflict)

    def is_unprocessable_entity(self, ex):
        http_status = (getattr(ex, 'http_status', None)
                       or getattr(ex, 'code', None))
        return (isinstance(ex, exceptions.ClientException)
                and http_status == 422)

    @tenacity.retry(stop=tenacity.stop_after_attempt(
        max(cfg.CONF.client_retry_limit + 1, 0)),
                    retry=tenacity.retry_if_exception(
                        client_plugin.retry_if_connection_err),
                    reraise=True)
    def get_server(self, server):
        """Return fresh server object.

        Substitutes Nova's NotFound for Heat's EntityNotFound,
        to be returned to user as HTTP error.
        """
        try:
            return self.client().servers.get(server)
        except exceptions.NotFound:
            raise exception.EntityNotFound(entity='Server', name=server)

    def fetch_server(self, server_id):
        """Fetch fresh server object from Nova.

        Log warnings and return None for non-critical API errors.
        Use this method in various ``check_*_complete`` resource methods,
        where intermittent errors can be tolerated.
        """
        server = None
        try:
            server = self.client().servers.get(server_id)
        except exceptions.OverLimit as exc:
            LOG.warning(
                "Received an OverLimit response when "
                "fetching server (%(id)s) : %(exception)s", {
                    'id': server_id,
                    'exception': exc
                })
        except exceptions.ClientException as exc:
            if ((getattr(exc, 'http_status', getattr(exc, 'code', None))
                 in (500, 503))):
                LOG.warning(
                    "Received the following exception when "
                    "fetching server (%(id)s) : %(exception)s", {
                        'id': server_id,
                        'exception': exc
                    })
            else:
                raise
        return server

    def fetch_server_attr(self, server_id, attr):
        server = self.fetch_server(server_id)
        fetched_attr = getattr(server, attr, None)
        return fetched_attr

    def refresh_server(self, server):
        """Refresh server's attributes.

        Also log warnings for non-critical API errors.
        """
        try:
            server.get()
        except exceptions.OverLimit as exc:
            LOG.warning(
                "Server %(name)s (%(id)s) received an OverLimit "
                "response during server.get(): %(exception)s", {
                    'name': server.name,
                    'id': server.id,
                    'exception': exc
                })
        except exceptions.ClientException as exc:
            if ((getattr(exc, 'http_status', getattr(exc, 'code', None))
                 in (500, 503))):
                LOG.warning(
                    'Server "%(name)s" (%(id)s) received the '
                    'following exception during server.get(): '
                    '%(exception)s', {
                        'name': server.name,
                        'id': server.id,
                        'exception': exc
                    })
            else:
                raise

    def get_ip(self, server, net_type, ip_version):
        """Return the server's IP of the given type and version."""
        if net_type in server.addresses:
            for ip in server.addresses[net_type]:
                if ip['version'] == ip_version:
                    return ip['addr']

    def get_status(self, server):
        """Return the server's status.

        :param server: server object
        :returns: status as a string
        """
        # Some clouds append extra (STATUS) strings to the status, strip it
        return server.status.split('(')[0]

    def _check_active(self, server, res_name='Server'):
        """Check server status.

        Accepts both server IDs and server objects.
        Returns True if server is ACTIVE,
        raises errors when server has an ERROR or unknown to Heat status,
        returns False otherwise.

        :param res_name: name of the resource to use in the exception message

        """
        # not checking with is_uuid_like as most tests use strings e.g. '1234'
        if isinstance(server, str):
            server = self.fetch_server(server)
            if server is None:
                return False
            else:
                status = self.get_status(server)
        else:
            status = self.get_status(server)
            if status != 'ACTIVE':
                self.refresh_server(server)
                status = self.get_status(server)

        if status in self.deferred_server_statuses:
            return False
        elif status == 'ACTIVE':
            return True
        elif status == 'ERROR':
            fault = getattr(server, 'fault', {})
            raise exception.ResourceInError(
                resource_status=status,
                status_reason=_("Message: %(message)s, Code: %(code)s") % {
                    'message': fault.get('message', _('Unknown')),
                    'code': fault.get('code', _('Unknown'))
                })
        else:
            raise exception.ResourceUnknownStatus(
                resource_status=server.status,
                result=_('%s is not active') % res_name)

    def find_flavor_by_name_or_id(self, flavor):
        """Find the specified flavor by name or id.

        :param flavor: the name of the flavor to find
        :returns: the id of :flavor:
        """
        return self._find_flavor_id(self.context.tenant_id, flavor)

    @os_client.MEMOIZE_FINDER
    def _find_flavor_id(self, tenant_id, flavor):
        # tenant id in the signature is used for the memoization key,
        # that would differentiate similar resource names across tenants.
        return self.get_flavor(flavor).id

    def get_flavor(self, flavor_identifier):
        """Get the flavor object for the specified flavor name or id.

        :param flavor_identifier: the name or id of the flavor to find
        :returns: a flavor object with name or id :flavor:
        """
        try:
            flavor = self.client().flavors.get(flavor_identifier)
        except exceptions.NotFound:
            flavor = self.client().flavors.find(name=flavor_identifier)

        return flavor

    def get_host(self, host_name):
        """Get the host id specified by name.

        :param host_name: the name of host to find
        :returns: the list of match hosts
        :raises exception.EntityNotFound:
        """

        host_list = self.client().hosts.list()
        for host in host_list:
            if host.host_name == host_name and host.service == self.COMPUTE:
                return host

        raise exception.EntityNotFound(entity='Host', name=host_name)

    def get_keypair(self, key_name):
        """Get the public key specified by :key_name:

        :param key_name: the name of the key to look for
        :returns: the keypair (name, public_key) for :key_name:
        :raises exception.EntityNotFound:
        """
        try:
            return self.client().keypairs.get(key_name)
        except exceptions.NotFound:
            raise exception.EntityNotFound(entity='Key', name=key_name)

    def build_userdata(self,
                       metadata,
                       userdata=None,
                       instance_user=None,
                       user_data_format='HEAT_CFNTOOLS'):
        """Build multipart data blob for CloudInit and Ignition.

        Data blob includes user-supplied Metadata, user data, and the required
        Heat in-instance configuration.

        :param resource: the resource implementation
        :type resource: heat.engine.Resource
        :param userdata: user data string
        :type userdata: str or None
        :param instance_user: the user to create on the server
        :type instance_user: string
        :param user_data_format: Format of user data to return
        :type user_data_format: string
        :returns: multipart mime as a string
        """

        if user_data_format == 'RAW':
            return userdata

        is_cfntools = user_data_format == 'HEAT_CFNTOOLS'
        is_software_config = user_data_format == 'SOFTWARE_CONFIG'

        if (is_software_config
                and NovaClientPlugin.is_ignition_format(userdata)):
            return NovaClientPlugin.build_ignition_data(metadata, userdata)

        def make_subpart(content, filename, subtype=None):
            if subtype is None:
                subtype = os.path.splitext(filename)[0]
            if content is None:
                content = ''
            try:
                content.encode('us-ascii')
                charset = 'us-ascii'
            except UnicodeEncodeError:
                charset = 'utf-8'
            msg = (text.MIMEText(content, _subtype=subtype, _charset=charset)
                   if subtype else text.MIMEText(content, _charset=charset))

            msg.add_header('Content-Disposition',
                           'attachment',
                           filename=filename)
            return msg

        def read_cloudinit_file(fn):
            return pkgutil.get_data('heat',
                                    'cloudinit/%s' % fn).decode('utf-8')

        if instance_user:
            config_custom_user = '******' % instance_user
            # FIXME(shadower): compatibility workaround for cloud-init 0.6.3.
            # We can drop this once we stop supporting 0.6.3 (which ships
            # with Ubuntu 12.04 LTS).
            #
            # See bug https://bugs.launchpad.net/heat/+bug/1257410
            boothook_custom_user = r"""useradd -m %s
echo -e '%s\tALL=(ALL)\tNOPASSWD: ALL' >> /etc/sudoers
""" % (instance_user, instance_user)
        else:
            config_custom_user = ''
            boothook_custom_user = ''

        cloudinit_config = string.Template(
            read_cloudinit_file('config')).safe_substitute(
                add_custom_user=config_custom_user)
        cloudinit_boothook = string.Template(
            read_cloudinit_file('boothook.sh')).safe_substitute(
                add_custom_user=boothook_custom_user)

        attachments = [(cloudinit_config, 'cloud-config'),
                       (cloudinit_boothook, 'boothook.sh', 'cloud-boothook'),
                       (read_cloudinit_file('part_handler.py'),
                        'part-handler.py')]
        if is_cfntools:
            attachments.append((userdata, 'cfn-userdata', 'x-cfninitdata'))
        elif is_software_config:
            # attempt to parse userdata as a multipart message, and if it
            # is, add each part as an attachment
            userdata_parts = None
            try:
                userdata_parts = email.message_from_string(userdata)
            except Exception:
                pass
            if userdata_parts and userdata_parts.is_multipart():
                for part in userdata_parts.get_payload():
                    attachments.append(
                        (part.get_payload(), part.get_filename(),
                         part.get_content_subtype()))
            else:
                attachments.append((userdata, ''))

        if is_cfntools:
            attachments.append((read_cloudinit_file('loguserdata.py'),
                                'loguserdata.py', 'x-shellscript'))

        if metadata:
            attachments.append(
                (jsonutils.dumps(metadata), 'cfn-init-data', 'x-cfninitdata'))

        if is_cfntools:
            heat_client_plugin = self.context.clients.client_plugin('heat')
            cfn_md_url = heat_client_plugin.get_cfn_metadata_server_url()
            attachments.append(
                (cfn_md_url, 'cfn-metadata-server', 'x-cfninitdata'))

            # Create a boto config which the cfntools on the host use to know
            # where the cfn API is to be accessed
            cfn_url = urlparse.urlparse(cfn_md_url)
            is_secure = cfg.CONF.instance_connection_is_secure
            vcerts = cfg.CONF.instance_connection_https_validate_certificates
            boto_cfg = "\n".join([
                "[Boto]", "debug = 0",
                "is_secure = %s" % is_secure,
                "https_validate_certificates = %s" % vcerts,
                "cfn_region_name = heat",
                "cfn_region_endpoint = %s" % cfn_url.hostname
            ])
            attachments.append((boto_cfg, 'cfn-boto-cfg', 'x-cfninitdata'))

        subparts = [make_subpart(*args) for args in attachments]
        mime_blob = multipart.MIMEMultipart(_subparts=subparts)

        return mime_blob.as_string()

    @staticmethod
    def is_ignition_format(userdata):
        try:
            payload = jsonutils.loads(userdata)
            ig = payload.get("ignition")
            return True if ig and ig.get("version") else False
        except Exception:
            return False

    @staticmethod
    def build_ignition_data(metadata, userdata):
        if not metadata:
            return userdata

        payload = jsonutils.loads(userdata)
        encoded_metadata = urlparse.quote(jsonutils.dumps(metadata))
        path_list = [
            "/var/lib/heat-cfntools/cfn-init-data",
            "/var/lib/cloud/data/cfn-init-data"
        ]
        ignition_format_metadata = {
            "filesystem": "root",
            "group": {
                "name": "root"
            },
            "path": "",
            "user": {
                "name": "root"
            },
            "contents": {
                "source": "data:," + encoded_metadata,
                "verification": {}
            },
            "mode": 0o640
        }

        for path in path_list:
            storage = payload.setdefault('storage', {})
            try:
                files = storage.setdefault('files', [])
            except AttributeError:
                raise ValueError('Ignition "storage" section must be a map')
            else:
                try:
                    data = ignition_format_metadata.copy()
                    data["path"] = path
                    files.append(data)
                except AttributeError:
                    raise ValueError('Ignition "files" section must be a list')

        return jsonutils.dumps(payload)

    def check_delete_server_complete(self, server_id):
        """Wait for server to disappear from Nova."""
        try:
            server = self.fetch_server(server_id)
        except Exception as exc:
            self.ignore_not_found(exc)
            return True
        if not server:
            return False
        task_state_in_nova = getattr(server, 'OS-EXT-STS:task_state', None)
        # the status of server won't change until the delete task has done
        if task_state_in_nova == 'deleting':
            return False

        status = self.get_status(server)
        if status == 'DELETED':
            return True

        if status == 'SOFT_DELETED':
            self.client().servers.force_delete(server_id)
        elif status == 'ERROR':
            fault = getattr(server, 'fault', {})
            message = fault.get('message', 'Unknown')
            code = fault.get('code')
            errmsg = _("Server %(name)s delete failed: (%(code)s) "
                       "%(message)s") % dict(
                           name=server.name, code=code, message=message)
            raise exception.ResourceInError(resource_status=status,
                                            status_reason=errmsg)
        return False

    def rename(self, server, name):
        """Update the name for a server."""
        server.update(name)

    def resize(self, server_id, flavor_id):
        """Resize the server."""
        server = self.fetch_server(server_id)
        if server:
            server.resize(flavor_id)
            return True
        else:
            return False

    def check_resize(self, server_id, flavor):
        """Verify that a resizing server is properly resized.

        If that's the case, confirm the resize, if not raise an error.
        """
        server = self.fetch_server(server_id)
        # resize operation is asynchronous so the server resize may not start
        # when checking server status (the server may stay ACTIVE instead
        # of RESIZE).
        if not server or server.status in ('RESIZE', 'ACTIVE'):
            return False
        if server.status == 'VERIFY_RESIZE':
            return True
        else:
            raise exception.Error(
                _("Resizing to '%(flavor)s' failed, status '%(status)s'") %
                dict(flavor=flavor, status=server.status))

    def verify_resize(self, server_id):
        server = self.fetch_server(server_id)
        if not server:
            return False
        status = self.get_status(server)
        if status == 'VERIFY_RESIZE':
            server.confirm_resize()
            return True
        else:
            msg = _("Could not confirm resize of server %s") % server_id
            raise exception.ResourceUnknownStatus(result=msg,
                                                  resource_status=status)

    def check_verify_resize(self, server_id):
        server = self.fetch_server(server_id)
        if not server:
            return False
        status = self.get_status(server)
        if status == 'ACTIVE':
            return True
        if status == 'VERIFY_RESIZE':
            return False
        task_state_in_nova = getattr(server, 'OS-EXT-STS:task_state', None)
        # Wait till move out from any resize steps (including resize_finish).
        if task_state_in_nova is not None and 'resize' in task_state_in_nova:
            return False
        else:
            msg = _("Confirm resize for server %s failed") % server_id
            raise exception.ResourceUnknownStatus(result=msg,
                                                  resource_status=status)

    def rebuild(self,
                server_id,
                image_id,
                password=None,
                preserve_ephemeral=False,
                meta=None,
                files=None):
        """Rebuild the server and call check_rebuild to verify."""
        server = self.fetch_server(server_id)
        if server:
            server.rebuild(image_id,
                           password=password,
                           preserve_ephemeral=preserve_ephemeral,
                           meta=meta,
                           files=files)
            return True
        else:
            return False

    def check_rebuild(self, server_id):
        """Verify that a rebuilding server is rebuilt.

        Raise error if it ends up in an ERROR state.
        """
        server = self.fetch_server(server_id)
        if server is None or server.status == 'REBUILD':
            return False
        if server.status == 'ERROR':
            raise exception.Error(
                _("Rebuilding server failed, status '%s'") % server.status)
        else:
            return True

    def meta_serialize(self, metadata):
        """Serialize non-string metadata values before sending them to Nova."""
        if not isinstance(metadata, collections.Mapping):
            raise exception.StackValidationFailed(
                message=_("nova server metadata needs to be a Map."))

        return dict(
            (key,
             (value if isinstance(value, str) else jsonutils.dumps(value)))
            for (key, value) in metadata.items())

    def meta_update(self, server, metadata):
        """Delete/Add the metadata in nova as needed."""
        metadata = self.meta_serialize(metadata)
        current_md = server.metadata
        to_del = sorted(set(current_md) - set(metadata))
        client = self.client()
        if len(to_del) > 0:
            client.servers.delete_meta(server, to_del)

        client.servers.set_meta(server, metadata)

    def server_to_ipaddress(self, server):
        """Return the server's IP address, fetching it from Nova."""
        try:
            server = self.client().servers.get(server)
        except exceptions.NotFound as ex:
            LOG.warning('Instance (%(server)s) not found: %(ex)s', {
                'server': server,
                'ex': ex
            })
        else:
            for n in sorted(server.networks, reverse=True):
                if len(server.networks[n]) > 0:
                    return server.networks[n][0]

    @tenacity.retry(stop=tenacity.stop_after_attempt(
        max(cfg.CONF.client_retry_limit + 1, 0)),
                    retry=tenacity.retry_if_exception(
                        client_plugin.retry_if_connection_err),
                    reraise=True)
    def absolute_limits(self):
        """Return the absolute limits as a dictionary."""
        limits = self.client().limits.get()
        return dict([(limit.name, limit.value)
                     for limit in list(limits.absolute)])

    def get_console_urls(self, server):
        """Return dict-like structure of server's console urls.

        The actual console url is lazily resolved on access.
        """
        nc = self.client

        class ConsoleUrls(collections.Mapping):
            def __init__(self, server):
                self.console_method = server.get_console_url
                self.support_console_types = [
                    'novnc', 'xvpvnc', 'spice-html5', 'rdp-html5', 'serial',
                    'webmks'
                ]

            def __getitem__(self, key):
                try:
                    if key not in self.support_console_types:
                        raise exceptions.UnsupportedConsoleType(key)
                    if key == 'webmks':
                        data = nc().servers.get_console_url(server, key)
                    else:
                        data = self.console_method(key)
                    console_data = data.get('remote_console',
                                            data.get('console'))
                    url = console_data['url']
                except exceptions.UnsupportedConsoleType as ex:
                    url = ex.message
                except Exception as e:
                    url = _('Cannot get console url: %s') % str(e)

                return url

            def __len__(self):
                return len(self.support_console_types)

            def __iter__(self):
                return (key for key in self.support_console_types)

        return ConsoleUrls(server)

    def attach_volume(self, server_id, volume_id, device):
        try:
            va = self.client().volumes.create_server_volume(
                server_id=server_id, volume_id=volume_id, device=device)
        except Exception as ex:
            if self.is_client_exception(ex):
                raise exception.Error(
                    _("Failed to attach volume %(vol)s to server %(srv)s "
                      "- %(err)s") % {
                          'vol': volume_id,
                          'srv': server_id,
                          'err': ex
                      })
            else:
                raise
        return va.id

    def detach_volume(self, server_id, attach_id):
        # detach the volume using volume_attachment
        try:
            self.client().volumes.delete_server_volume(server_id, attach_id)
        except Exception as ex:
            if not (self.is_not_found(ex) or self.is_bad_request(ex)):
                raise exception.Error(
                    _("Could not detach attachment %(att)s "
                      "from server %(srv)s.") % {
                          'srv': server_id,
                          'att': attach_id
                      })

    def check_detach_volume_complete(self, server_id, attach_id):
        """Check that nova server lost attachment.

        This check is needed for immediate reattachment when updating:
        there might be some time between cinder marking volume as 'available'
        and nova removing attachment from its own objects, so we
        check that nova already knows that the volume is detached.
        """
        try:
            self.client().volumes.get_server_volume(server_id, attach_id)
        except Exception as ex:
            self.ignore_not_found(ex)
            LOG.info("Volume %(vol)s is detached from server %(srv)s", {
                'vol': attach_id,
                'srv': server_id
            })
            return True
        else:
            LOG.debug("Server %(srv)s still has attachment %(att)s.", {
                'att': attach_id,
                'srv': server_id
            })
            return False

    def associate_floatingip(self, server_id, floatingip_id):
        iface_list = self.fetch_server(server_id).interface_list()
        if len(iface_list) == 0:
            raise client_exception.InterfaceNotFound(id=server_id)
        if len(iface_list) > 1:
            LOG.warning(
                "Multiple interfaces found for server %s, "
                "using the first one.", server_id)

        port_id = iface_list[0].port_id
        fixed_ips = iface_list[0].fixed_ips
        fixed_address = next(ip['ip_address'] for ip in fixed_ips
                             if netutils.is_valid_ipv4(ip['ip_address']))
        request_body = {
            'floatingip': {
                'port_id': port_id,
                'fixed_ip_address': fixed_address
            }
        }

        self.clients.client('neutron').update_floatingip(
            floatingip_id, request_body)

    def dissociate_floatingip(self, floatingip_id):
        request_body = {
            'floatingip': {
                'port_id': None,
                'fixed_ip_address': None
            }
        }
        self.clients.client('neutron').update_floatingip(
            floatingip_id, request_body)

    def associate_floatingip_address(self, server_id, fip_address):
        fips = self.clients.client('neutron').list_floatingips(
            floating_ip_address=fip_address)['floatingips']
        if len(fips) == 0:
            args = {'ip_address': fip_address}
            raise client_exception.EntityMatchNotFound(entity='floatingip',
                                                       args=args)
        self.associate_floatingip(server_id, fips[0]['id'])

    def dissociate_floatingip_address(self, fip_address):
        fips = self.clients.client('neutron').list_floatingips(
            floating_ip_address=fip_address)['floatingips']
        if len(fips) == 0:
            args = {'ip_address': fip_address}
            raise client_exception.EntityMatchNotFound(entity='floatingip',
                                                       args=args)
        self.dissociate_floatingip(fips[0]['id'])

    def interface_detach(self, server_id, port_id):
        with self.ignore_not_found:
            server = self.fetch_server(server_id)
            if server:
                server.interface_detach(port_id)
                return True

    def interface_attach(self,
                         server_id,
                         port_id=None,
                         net_id=None,
                         fip=None,
                         security_groups=None):
        server = self.fetch_server(server_id)
        if server:
            attachment = server.interface_attach(port_id, net_id, fip)
            if not port_id and security_groups:
                props = {'security_groups': security_groups}
                self.clients.client('neutron').update_port(
                    attachment.port_id, {'port': props})
            return True
        else:
            return False

    @tenacity.retry(stop=tenacity.stop_after_attempt(
        cfg.CONF.max_interface_check_attempts),
                    wait=tenacity.wait_exponential(multiplier=0.5, max=12.0),
                    retry=tenacity.retry_if_result(
                        client_plugin.retry_if_result_is_false))
    def check_interface_detach(self, server_id, port_id):
        with self.ignore_not_found:
            server = self.fetch_server(server_id)
            if server:
                interfaces = server.interface_list()
                for iface in interfaces:
                    if iface.port_id == port_id:
                        return False
        return True

    @tenacity.retry(stop=tenacity.stop_after_attempt(
        cfg.CONF.max_interface_check_attempts),
                    wait=tenacity.wait_fixed(0.5),
                    retry=tenacity.retry_if_result(
                        client_plugin.retry_if_result_is_false))
    def check_interface_attach(self, server_id, port_id):
        if not port_id:
            return True

        server = self.fetch_server(server_id)
        if server:
            interfaces = server.interface_list()
            for iface in interfaces:
                if iface.port_id == port_id:
                    return True
        return False
Example #17
0
class MergifyPull(object):
    # NOTE(sileht): Use from_cache/from_event not the constructor directly
    g_pull = attr.ib()
    installation_id = attr.ib()
    _complete = attr.ib(init=False, default=False)
    _reviews_required = attr.ib(init=False, default=None)

    # Cached attributes
    _reviews_ok = attr.ib(init=False, default=None)
    _reviews_ko = attr.ib(init=False, default=None)
    _required_statuses = attr.ib(
        init=False,
        default=None,
        validator=attr.validators.optional(
            attr.validators.instance_of(StatusState)),
    )
    _mergify_state = attr.ib(
        init=False,
        default=None,
        validator=attr.validators.optional(
            attr.validators.instance_of(MergifyState)),
    )
    _github_state = attr.ib(init=False, default=None)
    _github_description = attr.ib(init=False, default=None)

    @classmethod
    def from_raw(cls, installation_id, installation_token, pull_raw):
        g = github.Github(installation_token)
        pull = github.PullRequest.PullRequest(g._Github__requester, {},
                                              pull_raw,
                                              completed=True)
        return cls(pull, installation_id)

    def __attrs_post_init__(self):
        self.log = daiquiri.getLogger(__name__, pull_request=self)
        self._ensure_mergable_state()

    def _ensure_complete(self):
        if not self._complete:  # pragma: no cover
            raise RuntimeError("%s: used an incomplete MergifyPull")

    @property
    def status(self):
        # TODO(sileht): Should be removed at some point. When the cache
        # will not have mergify_engine_status key anymore
        return {
            "mergify_state": self.mergify_state,
            "github_state": self.github_state,
            "github_description": self.github_description
        }

    @property
    def mergify_state(self):
        self._ensure_complete()
        return self._mergify_state

    @property
    def github_state(self):
        self._ensure_complete()
        return self._github_state

    @property
    def github_description(self):
        self._ensure_complete()
        return self._github_description

    def complete(self, cache, branch_rule, collaborators):
        need_to_be_saved = False

        protection = branch_rule["protection"]
        if protection["required_pull_request_reviews"]:
            self._reviews_required = protection[
                "required_pull_request_reviews"][
                    "required_approving_review_count"]
        else:
            self._reviews_required = 0

        if "mergify_engine_reviews_ok" in cache:
            self._reviews_ok = cache["mergify_engine_reviews_ok"]
            self._reviews_ko = cache["mergify_engine_reviews_ko"]
        else:
            need_to_be_saved = True
            self._reviews_ok, self._reviews_ko = self._compute_approvals(
                branch_rule, collaborators)

        if "mergify_engine_required_statuses" in cache:
            self._required_statuses = StatusState(
                cache["mergify_engine_required_statuses"])
        else:
            need_to_be_saved = True
            self._required_statuses = self._compute_required_statuses(
                branch_rule)

        if "mergify_engine_status" in cache:
            s = cache["mergify_engine_status"]
            self._mergify_state = MergifyState(s["mergify_state"])
            self._github_state = s["github_state"]
            self._github_description = s["github_description"]
        else:
            need_to_be_saved = True
            (self._mergify_state, self._github_state,
             self._github_description) = self._compute_status(
                 branch_rule, collaborators)

        self._complete = True
        return need_to_be_saved

    def refresh(self, branch_rule, collaborators):
        # NOTE(sileht): Redownload the PULL to ensure we don't have
        # any etag floating around
        self._ensure_mergable_state(force=True)
        return self.complete({}, branch_rule, collaborators)

    def jsonify(self):
        raw = copy.copy(self.g_pull.raw_data)
        raw["mergify_engine_reviews_ok"] = self._reviews_ok
        raw["mergify_engine_reviews_ko"] = self._reviews_ko
        raw["mergify_engine_required_statuses"] = self._required_statuses.value
        raw["mergify_engine_status"] = {
            "mergify_state": self._mergify_state.value,
            "github_description": self._github_description,
            "github_state": self._github_state
        }
        return raw

    def _get_reviews(self):
        # Ignore reviews that are not from someone with write permissions
        # And only keep the last review for each user.
        return list(
            dict((review.user.login, review)
                 for review in self.g_pull.get_reviews()
                 if (review._rawData['author_association'] in (
                     "COLLABORATOR", "MEMBER", "OWNER"))).values())

    def to_dict(self):
        reviews = self._get_reviews()
        statuses = self._get_checks()
        # FIXME(jd) pygithub does 2 HTTP requests whereas 1 is enough!
        review_requested_users, review_requested_teams = (
            self.g_pull.get_review_requests())
        return {
            "assignee": [a.login for a in self.g_pull.assignees],
            "label": [l.name for l in self.g_pull.labels],
            "review-requested":
            ([u.login for u in review_requested_users] +
             ["@" + t.slug for t in review_requested_teams]),
            "author":
            self.g_pull.user.login,
            "merged-by":
            (self.g_pull.merged_by.login if self.g_pull.merged_by else ""),
            "merged":
            self.g_pull.merged,
            "state":
            self.g_pull.state,
            "milestone":
            (self.g_pull.milestone.title if self.g_pull.milestone else ""),
            "base":
            self.g_pull.base.ref,
            "head":
            self.g_pull.head.ref,
            "locked":
            self.g_pull._rawData['locked'],
            "title":
            self.g_pull.title,
            "body":
            self.g_pull.body,
            "files": [f.filename for f in self.g_pull.get_files()],
            "approved-reviews-by":
            [r.user.login for r in reviews if r.state == "APPROVED"],
            "dismissed-reviews-by":
            [r.user for r in reviews if r.state == "DISMISSED"],
            "changes-requested-reviews-by":
            [r.user for r in reviews if r.state == "CHANGES_REQUESTED"],
            "commented-reviews-by":
            [r.user for r in reviews if r.state == "COMMENTED"],
            "status-success":
            [s.context for s in statuses if s.state == "success"],
            # NOTE(jd) The Check API set conclusion to None for pending.
            # NOTE(sileht): "pending" statuses are not really trackable, we
            # voluntary drop this event because CIs just sent they status every
            # minutes until the CI pass (at least Travis and Circle CI does
            # that). This was causing a big load on Mergify for nothing useful
            # tracked, and on big projects it can reach the rate limit very
            # quickly.
            # "status-pending": [s.context for s in statuses
            #                    if s.state in ("pending", None)],
            "status-failure":
            [s.context for s in statuses if s.state == "failure"],
        }

    def _compute_approvals(self, branch_rule, collaborators):
        """Compute approvals.

        :param branch_rule: The rule for the considered branch.
        :param collaborators: The list of collaborators.
        :return: A tuple (users_with_review_ok, users_with_review_ko)
        """
        users_info = {}
        reviews_ok = set()
        reviews_ko = set()
        for review in self.g_pull.get_reviews():
            if review.user.id not in collaborators:
                continue

            users_info[review.user.login] = review.user.raw_data
            if review.state == 'APPROVED':
                reviews_ok.add(review.user.login)
                if review.user.login in reviews_ko:
                    reviews_ko.remove(review.user.login)

            elif review.state in ["DISMISSED", "CHANGES_REQUESTED"]:
                if review.user.login in reviews_ok:
                    reviews_ok.remove(review.user.login)
                if review.user.login in reviews_ko:
                    reviews_ko.remove(review.user.login)
                if review.state == "CHANGES_REQUESTED":
                    reviews_ko.add(review.user.login)
            elif review.state == 'COMMENTED':
                pass
            else:
                self.log.error("review state unhandled", state=review.state)

        return ([users_info[u]
                 for u in reviews_ok], [users_info[u] for u in reviews_ko])

    @staticmethod
    def _find_required_context(contexts, generic_check):
        for c in contexts:
            if generic_check.context.startswith(c):
                return c

    def _get_combined_status(self):
        headers, data = self.g_pull.head.repo._requester.requestJsonAndCheck(
            "GET",
            self.g_pull.base.repo.url + "/commits/" + self.g_pull.head.sha +
            "/status",
        )
        return github.CommitCombinedStatus.CommitCombinedStatus(
            self.g_pull.head.repo._requester, headers, data, completed=True)

    def _get_checks(self):
        generic_checks = set()
        try:
            # NOTE(sileht): conclusion can be one of success, failure, neutral,
            # cancelled, timed_out, or action_required, and  None for "pending"
            generic_checks |= set([
                GenericCheck(c.name, c.conclusion)
                for c in check_api.get_checks(self.g_pull)
            ])
        except github.GithubException as e:
            if (e.status != 403 or e.data["message"] !=
                    "Resource not accessible by integration"):
                raise

        # NOTE(sileht): state can be one of error, failure, pending,
        # or success.
        generic_checks |= set([
            GenericCheck(s.context, s.state)
            for s in self._get_combined_status().statuses
        ])
        return generic_checks

    def _compute_required_statuses(self, branch_rule):
        # return True is CIs succeed, False is their fail, None
        # is we don't known yet.
        # FIXME(sileht): I don't use a Enum yet to not
        protection = branch_rule["protection"]
        if not protection["required_status_checks"]:
            return StatusState.SUCCESS

        # NOTE(sileht): Due to the difference of both API we use only success
        # bellow.
        contexts = set(protection["required_status_checks"]["contexts"])
        seen_contexts = set()

        for check in self._get_checks():
            required_context = self._find_required_context(contexts, check)
            if required_context:
                seen_contexts.add(required_context)
                if check.state in ["pending", None]:
                    return StatusState.PENDING
                elif check.state != "success":
                    return StatusState.FAILURE

        if contexts - seen_contexts:
            return StatusState.PENDING
        else:
            return StatusState.SUCCESS

    def _disabled_by_rules(self, branch_rule):
        labels = [l.name for l in self.g_pull.labels]

        enabling_label = branch_rule["enabling_label"]
        if enabling_label is not None and enabling_label not in labels:
            return "Disabled — enabling label missing"

        if branch_rule["disabling_label"] in labels:
            return "Disabled — disabling label present"

        g_pull_files = [f.filename for f in self.g_pull.get_files()]
        for w in branch_rule["disabling_files"]:
            filtered = fnmatch.filter(g_pull_files, w)
            if filtered:
                return ("Disabled — %s is modified" % filtered[0])
        return None

    def _compute_status(self, branch_rule, collaborators):
        disabled = self._disabled_by_rules(branch_rule)

        mergify_state = MergifyState.NOT_READY
        github_state = "pending"
        github_desc = None

        if disabled:
            github_state = "failure"
            github_desc = disabled

        elif self._reviews_ko:
            github_desc = "Change requests need to be dismissed"

        elif (self._reviews_required
              and len(self._reviews_ok) < self._reviews_required):
            github_desc = ("%d/%d approvals required" %
                           (len(self._reviews_ok), self._reviews_required))

        elif self.g_pull.mergeable_state in ["clean", "unstable", "has_hooks"]:
            mergify_state = MergifyState.READY
            github_state = "success"
            github_desc = "Will be merged soon"

        elif self.g_pull.mergeable_state == "blocked":
            if self._required_statuses == StatusState.SUCCESS:
                # FIXME(sileht) We are blocked but reviews are OK and CI passes
                # So It's a Github bug or Github block the PR about something
                # we don't yet support.

                # We don't fully support require_code_owner_reviews, try so do
                # some guessing.
                protection = branch_rule["protection"]
                if (protection["required_pull_request_reviews"]
                        and protection["required_pull_request_reviews"]
                    ["require_code_owner_reviews"]
                        and (self.g_pull._rawData['requested_teams']
                             or self.g_pull._rawData['requested_reviewers'])):
                    github_desc = "Waiting for code owner review"

                else:
                    # NOTE(sileht): assume it's the Github bug and the PR is
                    # ready, if it's not the merge button will just fail.
                    self.log.warning("mergeable_state is unexpected, "
                                     "trying to merge the pull request")
                    mergify_state = MergifyState.READY
                    github_state = "success"
                    github_desc = "Will be merged soon"

            elif self._required_statuses == StatusState.PENDING:
                # Maybe clean soon, or maybe this is the previous run selected
                # PR that we just rebase, or maybe not. But we set the
                # mergify_state to ALMOST_READY to ensure we do not rebase
                # multiple self.g_pull request in //
                mergify_state = MergifyState.ALMOST_READY
                github_desc = "Waiting for status checks success"
            else:
                github_desc = "Waiting for status checks success"

        elif self.g_pull.mergeable_state == "behind":
            # Not up2date, but ready to merge, is branch updatable
            if not self.base_is_modifiable():
                github_state = "failure"
                github_desc = ("Pull request can't be updated with latest "
                               "base branch changes, owner doesn't allow "
                               "modification")
            elif self._required_statuses == StatusState.SUCCESS:
                mergify_state = MergifyState.NEED_BRANCH_UPDATE
                github_desc = ("Pull request will be updated with latest base "
                               "branch changes soon")
            else:
                github_desc = "Waiting for status checks success"

        elif self.g_pull.mergeable_state == "dirty":
            github_desc = "Merge conflict need to be solved"

        elif self.g_pull.mergeable_state == "unknown":
            # Should not really occur, but who known
            github_desc = "Pull request state reported unknown by Github"
        else:  # pragma: no cover
            raise RuntimeError("%s: Unexpected mergify_state" % self)

        if github_desc is None:  # pragma: no cover
            # Seatbelt
            raise RuntimeError("%s: github_desc have not been set" % self)

        return (mergify_state, github_state, github_desc)

    UNUSABLE_STATES = ["unknown", None]

    @tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.2),
                    stop=tenacity.stop_after_attempt(5),
                    retry=tenacity.retry_never)
    def _ensure_mergable_state(self, force=False):
        if self.g_pull.merged:
            return
        if (not force
                and self.g_pull.mergeable_state not in self.UNUSABLE_STATES):
            return

        # Github is currently processing this PR, we wait the completion
        self.log.info("refreshing")

        # NOTE(sileht): Well github doesn't always update etag/last_modified
        # when mergeable_state change, so we get a fresh pull request instead
        # of using update()
        self.g_pull = self.g_pull.base.repo.get_pull(self.g_pull.number)
        if (self.g_pull.merged
                or self.g_pull.mergeable_state not in self.UNUSABLE_STATES):
            return
        raise tenacity.TryAgain

    @tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.2),
                    stop=tenacity.stop_after_attempt(5),
                    retry=tenacity.retry_never)
    def _wait_for_sha_change(self, old_sha):
        if (self.g_pull.merged or self.g_pull.head.sha != old_sha):
            return

        # Github is currently processing this PR, we wait the completion
        self.log.info("refreshing")

        # NOTE(sileht): Well github doesn't always update etag/last_modified
        # when mergeable_state change, so we get a fresh pull request instead
        # of using update()
        self.g_pull = self.g_pull.base.repo.get_pull(self.g_pull.number)
        if (self.g_pull.merged or self.g_pull.head.sha != old_sha):
            return
        raise tenacity.TryAgain

    def wait_for_sha_change(self):
        old_sha = self.g_pull.head.sha
        self._wait_for_sha_change(old_sha)
        self._ensure_mergable_state()

    def __lt__(self, other):
        return ((self.mergify_state, self.g_pull.updated_at) >
                (other.mergify_state, other.g_pull.updated_at))

    def __eq__(self, other):
        return ((self.mergify_state,
                 self.g_pull.updated_at) == (other.mergify_state,
                                             other.g_pull.updated_at))

    def base_is_modifiable(self):
        return (self.g_pull.raw_data["maintainer_can_modify"]
                or self.g_pull.head.repo.id == self.g_pull.base.repo.id)

    def _merge_failed(self, e):
        # Don't log some  common and valid error
        if e.data["message"].startswith("Base branch was modified"):
            return False

        self.log.error("merge failed",
                       status=e.status,
                       error=e.data["message"],
                       exc_info=True)
        return False

    def merge(self, merge_method, rebase_fallback):
        try:
            self.g_pull.merge(sha=self.g_pull.head.sha,
                              merge_method=merge_method)
        except github.GithubException as e:  # pragma: no cover
            if (self.g_pull.is_merged()
                    and e.data["message"] == "Pull Request is not mergeable"):
                # Not a big deal, we will receive soon the pull_request close
                # event
                self.log.info("merged in the meantime")
                return True

            if (e.data["message"] != "This branch can't be rebased"
                    or merge_method != "rebase"
                    or (rebase_fallback == "none" or rebase_fallback is None)):
                return self._merge_failed(e)

            # If rebase fail retry with merge
            try:
                self.g_pull.merge(sha=self.g_pull.head.sha,
                                  merge_method=rebase_fallback)
            except github.GithubException as e:
                return self._merge_failed(e)
        return True

    def post_check_status(self, state, msg, context=None):

        redis = utils.get_redis_for_cache()
        context = "pr" if context is None else context
        msg_key = "%s/%s/%d/%s" % (self.installation_id,
                                   self.g_pull.base.repo.full_name,
                                   self.g_pull.number, context)

        if len(msg) >= 140:
            description = msg[0:137] + "..."
            redis.hset("status", msg_key, msg.encode('utf8'))
            target_url = "%s/check_status_msg/%s" % (config.BASE_URL, msg_key)
        else:
            description = msg
            target_url = None

        self.log.info("set status", state=state, description=description)
        # NOTE(sileht): We can't use commit.create_status() because
        # if use the head repo instead of the base repo
        try:
            self.g_pull._requester.requestJsonAndCheck(
                "POST",
                self.g_pull.base.repo.url + "/statuses/" +
                self.g_pull.head.sha,
                input={
                    'state': state,
                    'description': description,
                    'target_url': target_url,
                    'context': "%s/%s" % (config.CONTEXT, context)
                },
                headers={
                    'Accept': 'application/vnd.github.machine-man-preview+json'
                })
        except github.GithubException as e:  # pragma: no cover
            self.log.error("set status failed",
                           error=e.data["message"],
                           exc_info=True)

    def set_and_post_error(self, github_description):
        self._mergify_state = MergifyState.NOT_READY
        self._github_state = "failure"
        self._github_description = github_description
        self.post_check_status(self.github_state, self.github_description)

    def is_behind(self):
        branch = self.g_pull.base.repo.get_branch(self.g_pull.base.ref)
        for commit in self.g_pull.get_commits():
            for parent in commit.parents:
                if parent.sha == branch.commit.sha:
                    return False
        return True

    def __str__(self):
        return ("%(login)s/%(repo)s/pull/%(number)d@%(branch)s "
                "s:%(pr_state)s/%(statuses)s "
                "r:%(approvals)s/%(required_approvals)s "
                "-> %(mergify_state)s (%(github_state)s/%(github_desc)s)" % {
                    "login":
                    self.g_pull.base.user.login,
                    "repo":
                    self.g_pull.base.repo.name,
                    "number":
                    self.g_pull.number,
                    "branch":
                    self.g_pull.base.ref,
                    "pr_state": ("merged" if self.g_pull.merged else
                                 (self.g_pull.mergeable_state or "none")),
                    "statuses":
                    str(self._required_statuses),
                    "approvals": ("notset" if self._reviews_ok is None else
                                  len(self._reviews_ok)),
                    "required_approvals":
                    ("notset" if self._reviews_required is None else
                     self._reviews_required),
                    "mergify_state":
                    str(self._mergify_state),
                    "github_state": ("notset" if self._github_state is None
                                     else self._github_state),
                    "github_desc":
                    ("notset" if self._github_description is None else
                     self._github_description),
                })
Example #18
0
    def test_cephfs_share(self):
        """Test that CephFS shares can be accessed on two instances.

        1. Spawn two servers
        2. mount it on both
        3. write a file on one
        4. read it on the other
        5. profit
        """
        keyring = model.run_on_leader(
            'ceph-mon', 'cat /etc/ceph/ceph.client.admin.keyring')['Stdout']
        conf = model.run_on_leader('ceph-mon',
                                   'cat /etc/ceph/ceph.conf')['Stdout']
        # Spawn Servers
        for attempt in Retrying(stop=stop_after_attempt(3),
                                wait=wait_exponential(multiplier=1,
                                                      min=2,
                                                      max=10)):
            with attempt:
                instance_1 = guest.launch_instance(
                    glance_setup.LTS_IMAGE_NAME,
                    vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX),
                    userdata=self.INSTANCE_USERDATA.format(
                        _indent(conf, 8), _indent(keyring, 8)))

        for attempt in Retrying(stop=stop_after_attempt(3),
                                wait=wait_exponential(multiplier=1,
                                                      min=2,
                                                      max=10)):
            with attempt:
                instance_2 = guest.launch_instance(
                    glance_setup.LTS_IMAGE_NAME,
                    vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX),
                    userdata=self.INSTANCE_USERDATA.format(
                        _indent(conf, 8), _indent(keyring, 8)))

        # Write a file on instance_1
        def verify_setup(stdin, stdout, stderr):
            status = stdout.channel.recv_exit_status()
            self.assertEqual(status, 0)

        fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0]
        fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0]
        username = guest.boot_tests['bionic']['username']
        password = guest.boot_tests['bionic'].get('password')
        privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME)

        for attempt in Retrying(stop=stop_after_attempt(3),
                                wait=wait_exponential(multiplier=1,
                                                      min=2,
                                                      max=10)):
            with attempt:
                openstack_utils.ssh_command(
                    username,
                    fip_1,
                    'instance-1', 'sudo mount -a && '
                    'echo "test" | sudo tee /mnt/cephfs/test',
                    password=password,
                    privkey=privkey,
                    verify=verify_setup)

        def verify(stdin, stdout, stderr):
            status = stdout.channel.recv_exit_status()
            self.assertEqual(status, 0)
            out = ""
            for line in iter(stdout.readline, ""):
                out += line
            self.assertEqual(out, "test\n")

        openstack_utils.ssh_command(username,
                                    fip_2,
                                    'instance-2', 'sudo mount -a && '
                                    'sudo cat /mnt/cephfs/test',
                                    password=password,
                                    privkey=privkey,
                                    verify=verify)
Example #19
0
            raise Exception(msg.format(timeout))
        statuses = get_workflow_statuses(workflow_ids, cromwell_url, cromwell_user, cromwell_password, caas_key)
        all_succeeded = True
        for i, status in enumerate(statuses):
            if status in _failed_statuses:
                raise Exception('Stopping because workflow {0} {1}'.format(workflow_ids[i], status))
            elif status != 'Succeeded':
                all_succeeded = False
        if all_succeeded:
            print('All workflows succeeded!')
            break
        else:
            time.sleep(poll_interval_seconds)


@retry(reraise=True, wait=wait_exponential(multiplier=1, max=10), stop=stop_after_delay(20))
def start_workflow(
        wdl_file, inputs_file, url, options_file=None, inputs_file2=None, zip_file=None, user=None,
        password=None, caas_key=None, collection_name=None, label=None, validate_labels=True):
    """
    Use HTTP POST to start workflow in Cromwell and retry with exponentially increasing wait times between requests
    if there are any failures. View statistics about the retries with `start_workflow.retry.statistics`.

    .. note::

        The requests library could accept both Bytes and String objects as parameters of files, so there is no
        strict restrictions on the type of inputs of this function.

    :param _io.BytesIO wdl_file: wdl file.
    :param _io.BytesIO inputs_file: inputs file.
    :param str url: cromwell url.
Example #20
0
        return False


async def delete_dir_contents(path):
    logger.info(f"Deleting directory contents of {path}...")
    try:
        for root, dirs, files in os.walk(path):
            for f in files:
                os.unlink(os.path.join(root, f))
            for d in dirs:
                shutil.rmtree(os.path.join(root, d))
    except Exception as e:
        logger.exception(f"Could not remove contents in {path}. Reason: {e}")


@retry(stop=stop_after_attempt(10), wait=wait_exponential(min=1, max=10))
async def deploy_compose(compose):
    try:
        if not isinstance(compose, dict):
            compose = parse_yaml(compose)

        # Dump the compose to a temporary compose file and launch that. This is so we can amend the compose and update the
        # the stack without launching a new one
        dump_yaml(config.TMP_COMPOSE, compose)
        compose = config.TMP_COMPOSE

        proc = await asyncio.create_subprocess_exec("docker", "stack", "deploy", "--compose-file", compose, "walkoff",
                                                    stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)
        await log_proc_output(proc)

        if proc.returncode:
Example #21
0
class Context(object):
    client: http.Client
    pull: dict
    subscription: subscription.Subscription
    sources: List = dataclasses.field(default_factory=list)
    _write_permission_cache: cachetools.LRUCache = dataclasses.field(
        default_factory=lambda: cachetools.LRUCache(4096))
    log: logging.LoggerAdapter = dataclasses.field(init=False)

    def __post_init__(self):
        self._ensure_complete()

        self.log = daiquiri.getLogger(
            self.__class__.__qualname__,
            gh_owner=self.pull["base"]["user"]["login"]
            if "base" in self.pull else "<unknown-yet>",
            gh_repo=(self.pull["base"]["repo"]["name"]
                     if "base" in self.pull else "<unknown-yet>"),
            gh_private=(self.pull["base"]["repo"]["private"]
                        if "base" in self.pull else "<unknown-yet>"),
            gh_branch=self.pull["base"]["ref"]
            if "base" in self.pull else "<unknown-yet>",
            gh_pull=self.pull["number"],
            gh_pull_sha=self.pull["base"]["sha"]
            if "base" in self.pull else "<unknown-yet>",
            gh_pull_url=self.pull.get("html_url", "<unknown-yet>"),
            gh_pull_state=("merged" if self.pull.get("merged") else
                           (self.pull.get("mergeable_state", "unknown")
                            or "none")),
        )

    @property
    def base_url(self):
        """The URL prefix to make GitHub request."""
        return f"/repos/{self.pull['base']['user']['login']}/{self.pull['base']['repo']['name']}"

    @property
    def pull_request(self):
        return PullRequest(self)

    @cachetools.cachedmethod(
        cache=operator.attrgetter("_write_permission_cache"),
        key=functools.partial(cachetools.keys.hashkey,
                              "has_write_permissions"),
    )
    def has_write_permissions(self, login):
        return self.client.item(
            f"{self.base_url}/collaborators/{login}/permission"
        )["permission"] in [
            "admin",
            "write",
        ]

    def _get_valid_users(self):
        bots = list(
            set([
                r["user"]["login"] for r in self.reviews
                if r["user"] and r["user"]["type"] == "Bot"
            ]))
        collabs = set([
            r["user"]["login"] for r in self.reviews
            if r["user"] and r["user"]["type"] != "Bot"
        ])
        valid_collabs = [
            login for login in collabs if self.has_write_permissions(login)
        ]
        return bots + valid_collabs

    @functools.cached_property
    def consolidated_reviews(self):
        # Ignore reviews that are not from someone with admin/write permissions
        # And only keep the last review for each user.
        comments = dict()
        approvals = dict()
        valid_users = self._get_valid_users()
        for review in self.reviews:
            if not review["user"] or review["user"]["login"] not in valid_users:
                continue
            # Only keep latest review of an user
            if review["state"] == "COMMENTED":
                comments[review["user"]["login"]] = review
            else:
                approvals[review["user"]["login"]] = review
        return list(comments.values()), list(approvals.values())

    def _get_consolidated_data(self, name):
        if name == "assignee":
            return [a["login"] for a in self.pull["assignees"]]

        elif name == "label":
            return [label["name"] for label in self.pull["labels"]]

        elif name == "review-requested":
            return [u["login"] for u in self.pull["requested_reviewers"]] + [
                "@" + t["slug"] for t in self.pull["requested_teams"]
            ]

        elif name == "draft":
            return self.pull["draft"]

        elif name == "author":
            return self.pull["user"]["login"]

        elif name == "merged-by":
            return self.pull["merged_by"]["login"] if self.pull[
                "merged_by"] else ""

        elif name == "merged":
            return self.pull["merged"]

        elif name == "closed":
            return self.pull["state"] == "closed"

        elif name == "milestone":
            return self.pull["milestone"]["title"] if self.pull[
                "milestone"] else ""

        elif name == "number":
            return self.pull["number"]

        elif name == "conflict":
            return self.pull["mergeable_state"] == "dirty"

        elif name == "base":
            return self.pull["base"]["ref"]

        elif name == "head":
            return self.pull["head"]["ref"]

        elif name == "locked":
            return self.pull["locked"]

        elif name == "title":
            return self.pull["title"]

        elif name == "body":
            return self.pull["body"]

        elif name == "files":
            return [f["filename"] for f in self.files]

        elif name == "approved-reviews-by":
            _, approvals = self.consolidated_reviews
            return [
                r["user"]["login"] for r in approvals
                if r["state"] == "APPROVED"
            ]
        elif name == "dismissed-reviews-by":
            _, approvals = self.consolidated_reviews
            return [
                r["user"]["login"] for r in approvals
                if r["state"] == "DISMISSED"
            ]
        elif name == "changes-requested-reviews-by":
            _, approvals = self.consolidated_reviews
            return [
                r["user"]["login"] for r in approvals
                if r["state"] == "CHANGES_REQUESTED"
            ]
        elif name == "commented-reviews-by":
            comments, _ = self.consolidated_reviews
            return [
                r["user"]["login"] for r in comments
                if r["state"] == "COMMENTED"
            ]

        # NOTE(jd) The Check API set conclusion to None for pending.
        # NOTE(sileht): "pending" statuses are not really trackable, we
        # voluntary drop this event because CIs just sent they status every
        # minutes until the CI pass (at least Travis and Circle CI does
        # that). This was causing a big load on Mergify for nothing useful
        # tracked, and on big projects it can reach the rate limit very
        # quickly.
        # NOTE(sileht): Not handled for now: cancelled, timed_out, or action_required
        elif name == "status-success":
            return [
                ctxt for ctxt, state in self.checks.items()
                if state == "success"
            ]
        elif name == "status-failure":
            return [
                ctxt for ctxt, state in self.checks.items()
                if state == "failure"
            ]
        elif name == "status-neutral":
            return [
                ctxt for ctxt, state in self.checks.items()
                if state == "neutral"
            ]

        else:
            raise PullRequestAttributeError(name)

    def update_pull_check_runs(self, check):
        self.pull_check_runs = [
            c for c in self.pull_check_runs if c["name"] != check["name"]
        ]
        self.pull_check_runs.append(check)

    @functools.cached_property
    def pull_check_runs(self):
        return check_api.get_checks_for_ref(self, self.pull["head"]["sha"])

    @property
    def pull_engine_check_runs(self):
        return [
            c for c in self.pull_check_runs
            if c["app"]["id"] == config.INTEGRATION_ID
        ]

    @functools.cached_property
    def checks(self):
        # NOTE(sileht): conclusion can be one of success, failure, neutral,
        # cancelled, timed_out, or action_required, and  None for "pending"
        checks = dict(
            (c["name"], c["conclusion"]) for c in self.pull_check_runs)
        # NOTE(sileht): state can be one of error, failure, pending,
        # or success.
        checks.update((s["context"], s["state"]) for s in self.client.items(
            f"{self.base_url}/commits/{self.pull['head']['sha']}/status",
            list_items="statuses",
        ))
        return checks

    def _resolve_login(self, name):
        if not name:
            return []
        elif not isinstance(name, str):
            return [name]
        elif name[0] != "@":
            return [name]

        if "/" in name:
            organization, _, team_slug = name.partition("/")
            if not team_slug or "/" in team_slug:
                # Not a team slug
                return [name]
            organization = organization[1:]
        else:
            organization = self.pull["base"]["repo"]["owner"]["login"]
            team_slug = name[1:]

        try:
            return [
                member["login"] for member in self.client.items(
                    f"/orgs/{organization}/teams/{team_slug}/members")
            ]
        except http.HTTPClientSideError as e:
            self.log.warning(
                "fail to get the organization, team or members",
                team=name,
                status=e.status_code,
                detail=e.message,
            )
        return [name]

    def resolve_teams(self, values):
        if not values:
            return []
        if not isinstance(values, (list, tuple)):
            values = [values]
        values = list(
            itertools.chain.from_iterable((map(self._resolve_login, values))))
        return values

    UNUSABLE_STATES = ["unknown", None]

    # NOTE(sileht): quickly retry, if we don't get the status on time
    # the exception is recatch in worker.py, so worker will retry it later
    @tenacity.retry(
        wait=tenacity.wait_exponential(multiplier=0.2),
        stop=tenacity.stop_after_attempt(5),
        retry=tenacity.retry_if_exception_type(
            exceptions.MergeableStateUnknown),
        reraise=True,
    )
    def _ensure_complete(self):
        if not (self._is_data_complete()
                and self._is_background_github_processing_completed()):
            self.pull = self.client.item(
                f"{self.base_url}/pulls/{self.pull['number']}")

        if not self._is_data_complete():
            self.log.error(
                "/pulls/%s has returned an incomplete payload...",
                self.pull["number"],
                data=self.pull,
            )

        if self._is_background_github_processing_completed():
            return

        raise exceptions.MergeableStateUnknown(self)

    def _is_data_complete(self):
        # NOTE(sileht): If pull request come from /pulls listing or check-runs sometimes,
        # they are incomplete, This ensure we have the complete view
        fields_to_control = (
            "state",
            "mergeable_state",
            "merged_by",
            "merged",
            "merged_at",
        )
        for field in fields_to_control:
            if field not in self.pull:
                return False
        return True

    def _is_background_github_processing_completed(self):
        return (self.pull["state"] == "closed"
                or self.pull["mergeable_state"] not in self.UNUSABLE_STATES)

    def update(self):
        # TODO(sileht): Remove me,
        # Don't use it, because consolidated data are not updated after that.
        # Only used by merge action for posting an update report after rebase.
        self.pull = self.client.item(
            f"{self.base_url}/pulls/{self.pull['number']}")
        try:
            del self.__dict__["pull_check_runs"]
        except KeyError:
            pass

    @functools.cached_property
    def is_behind(self):
        branch_name_escaped = parse.quote(self.pull["base"]["ref"], safe="")
        branch = self.client.item(
            f"{self.base_url}/branches/{branch_name_escaped}")
        for commit in self.commits:
            for parent in commit["parents"]:
                if parent["sha"] == branch["commit"]["sha"]:
                    return False
        return True

    def __str__(self):
        return "%(login)s/%(repo)s/pull/%(number)d@%(branch)s " "s:%(pr_state)s" % {
            "login":
            self.pull["base"]["user"]["login"],
            "repo":
            self.pull["base"]["repo"]["name"],
            "number":
            self.pull["number"],
            "branch":
            self.pull["base"]["ref"],
            "pr_state": ("merged" if self.pull["merged"] else
                         (self.pull["mergeable_state"] or "none")),
        }

    @functools.cached_property
    def reviews(self):
        return list(
            self.client.items(
                f"{self.base_url}/pulls/{self.pull['number']}/reviews"))

    @functools.cached_property
    def commits(self):
        return list(
            self.client.items(
                f"{self.base_url}/pulls/{self.pull['number']}/commits"))

    @functools.cached_property
    def files(self):
        return list(
            self.client.items(
                f"{self.base_url}/pulls/{self.pull['number']}/files"))

    @property
    def pull_from_fork(self):
        return self.pull["head"]["repo"]["id"] != self.pull["base"]["repo"][
            "id"]

    def github_workflow_changed(self):
        for f in self.files:
            if f["filename"].startswith(".github/workflows"):
                return True
        return False
Example #22
0
from bs4 import BeautifulSoup
from tqdm import tqdm

USER_AGENT = ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, '
              'like Gecko) Chrome/55.0.2883.87 Safari/537.36')
BASE_LINK_EDGE = ('https://globalnewselection.s3.amazonaws.com/fm-playlist/'
                  'results/CFNYFM_pl_long.js?callback=plCallback')
BASE_LINK_INDIE = ('http://indie.streamon.fm/eventrange/{}-{}.json')


def to_epoch(dt):
    return (dt - datetime(1970, 1, 1)).total_seconds()


@tenacity.retry(reraise=True,
                wait=tenacity.wait_exponential(),
                stop=tenacity.stop_after_attempt(5))
def get_indie_data(start, end):
    return requests.get(BASE_LINK_INDIE.format(start, end),
                        headers={
                            'User-Agent': USER_AGENT
                        }).json()


@tenacity.retry(reraise=True,
                wait=tenacity.wait_exponential(),
                stop=tenacity.stop_after_attempt(5))
def get_edge_data():
    soup = BeautifulSoup(
        requests.get('https://edge.ca/music/').content, 'html.parser')
    date = to_epoch(
Example #23
0
class S3Storage(storage.StorageDriver):

    WRITE_FULL = True

    _consistency_wait = tenacity.wait_exponential(multiplier=0.1)

    def __init__(self, conf):
        super(S3Storage, self).__init__(conf)
        self.s3, self._region_name, self._bucket_prefix = (
            s3.get_connection(conf))
        self._bucket_name = '%s-aggregates' % self._bucket_prefix
        if conf.s3_check_consistency_timeout > 0:
            self._consistency_stop = tenacity.stop_after_delay(
                conf.s3_check_consistency_timeout)
        else:
            self._consistency_stop = None

    def __str__(self):
        return "%s: %s" % (self.__class__.__name__, self._bucket_name)

    def upgrade(self):
        super(S3Storage, self).upgrade()
        try:
            s3.create_bucket(self.s3, self._bucket_name, self._region_name)
        except botocore.exceptions.ClientError as e:
            if e.response['Error'].get('Code') != "BucketAlreadyExists":
                raise

    @staticmethod
    def _object_name(split_key, aggregation, version=3):
        name = '%s_%s_%s' % (
            aggregation,
            utils.timespan_total_seconds(split_key.sampling),
            split_key,
        )
        return name + '_v%s' % version if version else name

    @staticmethod
    def _prefix(metric):
        return str(metric.id) + '/'

    def _put_object_safe(self, Bucket, Key, Body):
        put = self.s3.put_object(Bucket=Bucket, Key=Key, Body=Body)

        if self._consistency_stop:

            def _head():
                return self.s3.head_object(Bucket=Bucket,
                                           Key=Key,
                                           IfMatch=put['ETag'])

            tenacity.Retrying(retry=tenacity.retry_if_result(
                lambda r: r['ETag'] != put['ETag']),
                              wait=self._consistency_wait,
                              stop=self._consistency_stop)(_head)

    def _store_metric_splits_unbatched(self, metric, key, aggregation, data,
                                       offset, version):
        self._put_object_safe(
            Bucket=self._bucket_name,
            Key=self._prefix(metric) +
            self._object_name(key, aggregation.method, version),
            Body=data)

    def _delete_metric_splits_unbatched(self,
                                        metric,
                                        key,
                                        aggregation,
                                        version=3):
        self.s3.delete_object(
            Bucket=self._bucket_name,
            Key=self._prefix(metric) +
            self._object_name(key, aggregation.method, version))

    def _delete_metric(self, metric):
        bucket = self._bucket_name
        response = {}
        while response.get('IsTruncated', True):
            if 'NextContinuationToken' in response:
                kwargs = {
                    'ContinuationToken': response['NextContinuationToken']
                }
            else:
                kwargs = {}
            try:
                response = self.s3.list_objects_v2(Bucket=bucket,
                                                   Prefix=self._prefix(metric),
                                                   **kwargs)
            except botocore.exceptions.ClientError as e:
                if e.response['Error'].get('Code') == "NoSuchKey":
                    # Maybe it never has been created (no measure)
                    return
                raise
            s3.bulk_delete(self.s3, bucket,
                           [c['Key'] for c in response.get('Contents', ())])

    def _get_splits_unbatched(self, metric, key, aggregation, version=3):
        try:
            response = self.s3.get_object(
                Bucket=self._bucket_name,
                Key=self._prefix(metric) +
                self._object_name(key, aggregation.method, version))
        except botocore.exceptions.ClientError as e:
            if e.response['Error'].get('Code') == 'NoSuchKey':
                return
            raise
        return response['Body'].read()

    def _metric_exists_p(self, metric, version):
        unaggkey = self._build_unaggregated_timeserie_path(metric, version)
        try:
            self.s3.head_object(Bucket=self._bucket_name, Key=unaggkey)
        except botocore.exceptions.ClientError as e:
            if e.response['Error'].get('Code') == "404":
                return False
            raise
        return True

    def _list_split_keys(self, metric, aggregations, version=3):
        bucket = self._bucket_name
        keys = {}
        for aggregation in aggregations:
            keys[aggregation] = set()
            response = {}
            while response.get('IsTruncated', True):
                if 'NextContinuationToken' in response:
                    kwargs = {
                        'ContinuationToken': response['NextContinuationToken']
                    }
                else:
                    kwargs = {}
                response = self.s3.list_objects_v2(
                    Bucket=bucket,
                    Prefix=self._prefix(metric) + '%s_%s' % (
                        aggregation.method,
                        utils.timespan_total_seconds(aggregation.granularity),
                    ),
                    **kwargs)
                # If response is empty then check that the metric exists
                contents = response.get('Contents', ())
                if not contents and not self._metric_exists_p(metric, version):
                    raise storage.MetricDoesNotExist(metric)
                for f in contents:
                    try:
                        if (self._version_check(f['Key'], version)):
                            meta = f['Key'].split('_')
                            keys[aggregation].add(
                                carbonara.SplitKey(
                                    utils.to_timestamp(meta[2]),
                                    sampling=aggregation.granularity))
                    except (ValueError, IndexError):
                        # Might be "none", or any other file. Be resilient.
                        continue
        return keys

    @staticmethod
    def _build_unaggregated_timeserie_path(metric, version):
        return S3Storage._prefix(metric) + 'none' + ("_v%s" % version
                                                     if version else "")

    def _get_or_create_unaggregated_timeseries_unbatched(
            self, metric, version=3):
        key = self._build_unaggregated_timeserie_path(metric, version)
        try:
            response = self.s3.get_object(Bucket=self._bucket_name, Key=key)
        except botocore.exceptions.ClientError as e:
            if e.response['Error'].get('Code') == "NoSuchKey":
                # Create the metric with empty data
                self._put_object_safe(Bucket=self._bucket_name,
                                      Key=key,
                                      Body="")
            else:
                raise
        else:
            return response['Body'].read() or None

    def _store_unaggregated_timeseries_unbatched(self,
                                                 metric,
                                                 data,
                                                 version=3):
        self._put_object_safe(Bucket=self._bucket_name,
                              Key=self._build_unaggregated_timeserie_path(
                                  metric, version),
                              Body=data)
Example #24
0
def set_link_bridge_stp(device, stp, namespace=None):
    return _run_iproute_link('set', device, namespace=namespace,
                             kind='bridge', br_stp_state=stp)


@privileged.link_cmd.entrypoint
def set_link_bridge_master(device, bridge, namespace=None):
    bridge_idx = get_link_id(bridge, namespace) if bridge else 0
    return _run_iproute_link('set', device, namespace=namespace,
                             master=bridge_idx)


@tenacity.retry(
    retry=tenacity.retry_if_exception_type(
        netlink_exceptions.NetlinkDumpInterrupted),
    wait=tenacity.wait_exponential(multiplier=0.02, max=1),
    stop=tenacity.stop_after_delay(8),
    reraise=True)
@privileged.link_cmd.entrypoint
def get_link_attributes(device, namespace):
    link = _run_iproute_link("get", device, namespace)[0]
    return {
        'mtu': link.get_attr('IFLA_MTU'),
        'qlen': link.get_attr('IFLA_TXQLEN'),
        'state': link.get_attr('IFLA_OPERSTATE'),
        'qdisc': link.get_attr('IFLA_QDISC'),
        'brd': link.get_attr('IFLA_BROADCAST'),
        'link/ether': link.get_attr('IFLA_ADDRESS'),
        'alias': link.get_attr('IFLA_IFALIAS'),
        'allmulticast': bool(link['flags'] & ifinfmsg.IFF_ALLMULTI),
        'link_kind': link.get_nested('IFLA_LINKINFO', 'IFLA_INFO_KIND')
Example #25
0
class MergifyPull(object):
    # NOTE(sileht): Use from_cache/from_event not the constructor directly
    g = attr.ib()
    g_pull = attr.ib()
    installation_id = attr.ib()
    _consolidated_data = attr.ib(init=False, default=None)

    @classmethod
    def from_raw(cls, installation_id, installation_token, pull_raw):
        g = github.Github(installation_token,
                          base_url="https://api.%s" % config.GITHUB_DOMAIN)
        pull = github.PullRequest.PullRequest(g._Github__requester, {},
                                              pull_raw,
                                              completed=True)
        return cls(g, pull, installation_id)

    @classmethod
    def from_number(cls, installation_id, installation_token, owner, reponame,
                    pull_number):
        g = github.Github(installation_token,
                          base_url="https://api.%s" % config.GITHUB_DOMAIN)
        repo = g.get_repo(owner + "/" + reponame)
        pull = repo.get_pull(pull_number)
        return cls(g, pull, installation_id)

    def __attrs_post_init__(self):
        self._ensure_mergable_state()

    def _valid_perm(self, user):
        if user.type == "Bot":
            return True
        return self.g_pull.base.repo.get_collaborator_permission(
            user.login) in [
                "admin",
                "write",
            ]

    def _get_reviews(self):
        # Ignore reviews that are not from someone with admin/write permissions
        # And only keep the last review for each user.
        reviews = list(self.g_pull.get_reviews())
        valid_users = list(
            map(
                lambda u: u.login,
                filter(self._valid_perm, set([r.user for r in reviews])),
            ))
        comments = dict()
        approvals = dict()
        for review in reviews:
            if review.user.login not in valid_users:
                continue
            # Only keep latest review of an user
            if review.state == "COMMENTED":
                comments[review.user.login] = review
            else:
                approvals[review.user.login] = review
        return list(comments.values()), list(approvals.values())

    def to_dict(self):
        if self._consolidated_data is None:
            self._consolidated_data = self._get_consolidated_data()
        return self._consolidated_data

    def _get_consolidated_data(self):
        comments, approvals = self._get_reviews()
        statuses = self._get_checks()
        # FIXME(jd) pygithub does 2 HTTP requests whereas 1 is enough!
        (
            review_requested_users,
            review_requested_teams,
        ) = self.g_pull.get_review_requests()
        return {
            # Only use internally attributes
            "_approvals":
            approvals,
            # Can be used by rules too
            "assignee": [a.login for a in self.g_pull.assignees],
            # NOTE(sileht): We put an empty label to allow people to match
            # no label set
            "label": [l.name for l in self.g_pull.labels],
            "review-requested":
            ([u.login for u in review_requested_users] +
             ["@" + t.slug for t in review_requested_teams]),
            "author":
            self.g_pull.user.login,
            "merged-by":
            (self.g_pull.merged_by.login if self.g_pull.merged_by else ""),
            "merged":
            self.g_pull.merged,
            "closed":
            self.g_pull.state == "closed",
            "milestone":
            (self.g_pull.milestone.title if self.g_pull.milestone else ""),
            "conflict":
            self.g_pull.mergeable_state == "dirty",
            "base":
            self.g_pull.base.ref,
            "head":
            self.g_pull.head.ref,
            "locked":
            self.g_pull._rawData["locked"],
            "title":
            self.g_pull.title,
            "body":
            self.g_pull.body,
            "files": [f.filename for f in self.g_pull.get_files()],
            "approved-reviews-by":
            [r.user.login for r in approvals if r.state == "APPROVED"],
            "dismissed-reviews-by":
            [r.user.login for r in approvals if r.state == "DISMISSED"],
            "changes-requested-reviews-by": [
                r.user.login for r in approvals
                if r.state == "CHANGES_REQUESTED"
            ],
            "commented-reviews-by":
            [r.user.login for r in comments if r.state == "COMMENTED"],
            "status-success":
            [s.context for s in statuses if s.state == "success"],
            # NOTE(jd) The Check API set conclusion to None for pending.
            # NOTE(sileht): "pending" statuses are not really trackable, we
            # voluntary drop this event because CIs just sent they status every
            # minutes until the CI pass (at least Travis and Circle CI does
            # that). This was causing a big load on Mergify for nothing useful
            # tracked, and on big projects it can reach the rate limit very
            # quickly.
            # "status-pending": [s.context for s in statuses
            #                    if s.state in ("pending", None)],
            "status-failure":
            [s.context for s in statuses if s.state == "failure"],
            "status-neutral":
            [s.context for s in statuses if s.state == "neutral"],
            # NOTE(sileht): Not handled for now
            # cancelled, timed_out, or action_required
        }

    def _get_statuses(self):
        already_seen = set()
        statuses = []
        for status in github.PaginatedList.PaginatedList(
                github.CommitStatus.CommitStatus,
                self.g_pull._requester,
                self.g_pull.base.repo.url + "/commits/" +
                self.g_pull.head.sha + "/statuses",
                None,
        ):
            if status.context not in already_seen:
                already_seen.add(status.context)
                statuses.append(status)
        return statuses

    def _get_checks(self):
        generic_checks = set()
        try:
            # NOTE(sileht): conclusion can be one of success, failure, neutral,
            # cancelled, timed_out, or action_required, and  None for "pending"
            generic_checks |= set([
                GenericCheck(c.name, c.conclusion)
                for c in check_api.get_checks(self.g_pull)
            ])
        except github.GithubException as e:
            if (e.status != 403 or e.data["message"] !=
                    "Resource not accessible by integration"):
                raise

        # NOTE(sileht): state can be one of error, failure, pending,
        # or success.
        generic_checks |= set(
            [GenericCheck(s.context, s.state) for s in self._get_statuses()])
        return generic_checks

    def _resolve_login(self, name):
        if not name:
            return []
        elif not isinstance(name, str):
            return [name]
        elif name[0] != "@":
            return [name]

        if "/" in name:
            organization, _, team_slug = name.partition("/")
            if not team_slug or "/" in team_slug:
                # Not a team slug
                return [name]
            organization = organization[1:]
        else:
            organization = self.g_pull.base.repo.owner.login
            team_slug = name[1:]

        try:
            g_organization = self.g.get_organization(organization)
            for team in g_organization.get_teams():
                if team.slug == team_slug:
                    return [m.login for m in team.get_members()]
        except github.GithubException as e:
            if e.status >= 500:
                raise
            LOG.warning(
                "fail to get the organization, team or members",
                team=name,
                status=e.status,
                detail=e.data["message"],
                pull_request=self,
            )
        return [name]

    def resolve_teams(self, values):
        if not values:
            return []
        if not isinstance(values, (list, tuple)):
            values = [values]
        return list(
            itertools.chain.from_iterable((map(self._resolve_login, values))))

    UNUSABLE_STATES = ["unknown", None]

    # NOTE(sileht): quickly retry, if we don't get the status on time
    # the exception is recatch in worker.py, so celery will retry it later
    @tenacity.retry(
        wait=tenacity.wait_exponential(multiplier=0.2),
        stop=tenacity.stop_after_attempt(5),
        retry=tenacity.retry_if_exception_type(
            exceptions.MergeableStateUnknown),
        reraise=True,
    )
    def _ensure_mergable_state(self, force=False):
        if self.g_pull.state == "closed":
            return
        if not force and self.g_pull.mergeable_state not in self.UNUSABLE_STATES:
            return

        # Github is currently processing this PR, we wait the completion
        LOG.info("refreshing", pull_request=self)

        # NOTE(sileht): Well github doesn't always update etag/last_modified
        # when mergeable_state change, so we get a fresh pull request instead
        # of using update()
        self.g_pull = self.g_pull.base.repo.get_pull(self.g_pull.number)
        if (self.g_pull.state == "closed"
                or self.g_pull.mergeable_state not in self.UNUSABLE_STATES):
            return

        raise exceptions.MergeableStateUnknown(self)

    @tenacity.retry(
        wait=tenacity.wait_exponential(multiplier=0.2),
        stop=tenacity.stop_after_attempt(5),
        retry=tenacity.retry_never,
    )
    def _wait_for_sha_change(self, old_sha):
        if self.g_pull.state == "closed" or self.g_pull.head.sha != old_sha:
            return

        # Github is currently processing this PR, we wait the completion
        LOG.info("refreshing", pull_request=self)

        # NOTE(sileht): Well github doesn't always update etag/last_modified
        # when mergeable_state change, so we get a fresh pull request instead
        # of using update()
        self.g_pull = self.g_pull.base.repo.get_pull(self.g_pull.number)
        if self.g_pull.state == "closed" or self.g_pull.head.sha != old_sha:
            return
        raise tenacity.TryAgain

    def wait_for_sha_change(self):
        old_sha = self.g_pull.head.sha
        self._wait_for_sha_change(old_sha)
        self._ensure_mergable_state()

    def base_is_modifiable(self):
        return (self.g_pull.raw_data["maintainer_can_modify"]
                or self.g_pull.head.repo.id == self.g_pull.base.repo.id)

    def is_behind(self):
        branch = self.g_pull.base.repo.get_branch(
            parse.quote(self.g_pull.base.ref, safe=""))
        for commit in self.g_pull.get_commits():
            for parent in commit.parents:
                if parent.sha == branch.commit.sha:
                    return False
        return True

    def get_merge_commit_message(self):
        if not self.g_pull.body:
            return

        found = False
        message_lines = []

        for line in self.g_pull.body.split("\n"):
            if MARKDOWN_COMMIT_MESSAGE_RE.match(line):
                found = True
            elif found and MARKDOWN_TITLE_RE.match(line):
                break
            elif found:
                message_lines.append(line)

        if found and message_lines:
            return {
                "commit_title": message_lines[0],
                "commit_message": "\n".join(message_lines[1:]).strip(),
            }

    def __str__(self):
        return "%(login)s/%(repo)s/pull/%(number)d@%(branch)s " "s:%(pr_state)s" % {
            "login":
            self.g_pull.base.user.login,
            "repo":
            self.g_pull.base.repo.name,
            "number":
            self.g_pull.number,
            "branch":
            self.g_pull.base.ref,
            "pr_state": ("merged" if self.g_pull.merged else
                         (self.g_pull.mergeable_state or "none")),
        }
])


def _do_update_branch(git, method, base_branch, head_branch):
    if method == "merge":
        git("merge", "--quiet", "upstream/%s" % base_branch, "-m",
            "Merge branch '%s' into '%s'" % (base_branch, head_branch))
        git("push", "--quiet", "origin", head_branch)
    elif method == "rebase":
        git("rebase", "upstream/%s" % base_branch)
        git("push", "--quiet", "origin", head_branch, "-f")
    else:
        raise RuntimeError("Invalid branch update method")


@tenacity.retry(wait=tenacity.wait_exponential(multiplier=0.2),
                stop=tenacity.stop_after_attempt(5),
                retry=tenacity.retry_if_exception_type(BranchUpdateNeedRetry))
def _do_update(pull, token, method="merge"):
    # NOTE(sileht):
    # $ curl https://api.github.com/repos/sileht/repotest/pulls/2 | jq .commits
    # 2
    # $ git clone https://[email protected]/sileht-tester/repotest \
    #           --depth=$((2 + 1)) -b sileht/testpr
    # $ cd repotest
    # $ git remote add upstream https://[email protected]/sileht/repotest.git
    # $ git log | grep Date | tail -1
    # Date:   Fri Mar 30 21:30:26 2018 (10 days ago)
    # $ git fetch upstream master --shallow-since="Fri Mar 30 21:30:26 2018"
    # $ git rebase upstream/master
    # $ git push origin sileht/testpr:sileht/testpr
Example #27
0
class Bootloader:
    """ A class to hold the logic for each of the possible commands. This follows the dispatch pattern we us in app_base
        for calling actions in apps. The pattern as applied to the CLI follows close to this example:
        https://chase-seibert.github.io/blog/2014/03/21/python-multilevel-argparse.html#
    """

    def __init__(self, session=None, docker_client=None):
        self.session: aiohttp.ClientSession = session
        self.docker_client: aiodocker.Docker = docker_client
        with open(".dockerignore") as f:
            self.dockerignore = [line.strip() for line in f.readlines()]

    @staticmethod
    async def run():
        """ Landing pad to launch primary command and do whatever async init the bootloader needs. """
        # TODO: fill in the helps, and further develop cli with the end user in mind
        commands = {"up", "down", "refresh"}
        parser = argparse.ArgumentParser()
        parser.add_argument("command", choices=commands)
        parser.add_argument("args", nargs=argparse.REMAINDER)

        logger.setLevel("DEBUG")
        docker_logger.setLevel("DEBUG")

        # Parse out the command
        args = parser.parse_args(sys.argv[1:2])

        async with aiohttp.ClientSession() as session, connect_to_aiodocker() as docker_client:
            bootloader = Bootloader(session, docker_client)

            if hasattr(bootloader, args.command):
                await getattr(bootloader, args.command)()
            else:
                logger.error("Invalid command.")
                # TODO: Pipe this through the logger. print_help() accepts a file kwarg that we can use to do this
                parser.print_help()

    @retry(stop=stop_after_attempt(10), wait=wait_exponential(min=1, max=10))
    async def wait_for_registry(self):
        try:
            async with self.session.get(f"http://{DOCKER_HOST_IP}:5000") as resp:
                if resp.status == 200:
                    return True
                else:
                    raise ConnectionError
        except Exception as e:
            logger.info("Registry not available yet, waiting to try again...")
            raise e

    @retry(stop=stop_after_attempt(10), wait=wait_exponential(min=1, max=10))
    async def wait_for_minio(self):
        try:
            async with self.session.get(f"http://{config.MINIO}/minio/health/ready") as resp:
                if resp.status == 200:
                    return True
                else:
                    raise ConnectionError
        except Exception as e:
            logger.info("Minio not available yet, waiting to try again...")
            raise e

    async def up(self):
        data = {"name": "postgres-data"}
        # Create Postgres Volume
        await self.docker_client.volumes.create(data)

        # Create Walkoff encryption key
        wek = await create_encryption_key(self.docker_client, "walkoff_encryption_key")

        # Create internal user key
        wik = await create_encryption_key(self.docker_client, "walkoff_internal_key")

        # Create Postgres user password
        wpk = await create_encryption_key(self.docker_client, "walkoff_postgres_key")

        # Create Minio secret key
        wmak = await create_encryption_key(self.docker_client, "walkoff_minio_access_key", b"walkoff")
        wmsk = await create_encryption_key(self.docker_client, "walkoff_minio_secret_key")

        # Set up a subcommand parser
        parser = argparse.ArgumentParser(description="Bring the WALKOFF stack up and initialize it")
        parser.add_argument("-b", "--build", action="store_true",
                            help="Builds and pushes all WALKOFF components to local registry.")
        parser.add_argument("-d", "--debug", action="store_true",
                            help="Set log level to debug.")
        parser.add_argument("-k", "--keys", action="store_true",
                            help="Prints all keys to STDOUT (dangerous).")
        # Parse out the command
        args = parser.parse_args(sys.argv[2:])

        if args.debug:
            logger.setLevel("DEBUG")
            docker_logger.setLevel("DEBUG")

        logger.info("Creating persistent directories for registry, postgres, portainer...")
        os.makedirs(static.REGISTRY_DATA_PATH, exist_ok=True)
        os.makedirs(static.POSTGRES_DATA_PATH, exist_ok=True)
        os.makedirs(static.PORTAINER_DATA_PATH, exist_ok=True)
        os.makedirs(static.MINIO_DATA_PATH, exist_ok=True)

        # Bring up the base compose with the registry
        logger.info("Deploying base services (registry, postgres, portainer, redis)...")
        base_compose = parse_yaml(config.BASE_COMPOSE)

        await deploy_compose(base_compose)

        await self.wait_for_registry()

        # Merge the base, walkoff, and app composes
        app_composes = generate_app_composes()
        walkoff_compose = parse_yaml(config.WALKOFF_COMPOSE)
        merged_compose = merge_composes(walkoff_compose, app_composes)

        dump_yaml(config.TMP_COMPOSE, merged_compose)

        if args.build:
            walkoff_app_sdk = walkoff_compose["services"]["app_sdk"]
            await build_image(self.docker_client, walkoff_app_sdk["image"],
                              walkoff_app_sdk["build"]["dockerfile"],
                              walkoff_app_sdk["build"]["context"],
                              self.dockerignore)
            await push_image(self.docker_client, walkoff_app_sdk["image"])

            builders = []
            pushers = []

            for service_name, service in walkoff_compose["services"].items():
                if "build" in service:
                    build_func = build_image(self.docker_client, service["image"],
                                             service["build"]["dockerfile"],
                                             service["build"]["context"],
                                             self.dockerignore)
                    push_func = push_image(self.docker_client, service["image"])
                    if args.debug:
                        await build_func
                        await push_func
                    else:
                        builders.append(build_func)
                        pushers.append(push_func)

            if not args.debug:
                logger.info("Building Docker images asynchronously, this could take some time...")
                await asyncio.gather(*builders)
                logger.info("Build process complete.")
                logger.info("Pushing Docker images asynchronously, this could take some time...")
                await asyncio.gather(*pushers)
                logger.info("Push process complete.")

        await self.wait_for_minio()
        # await self.push_to_minio()

        logger.info("Deploying Walkoff stack...")

        return_code = await deploy_compose(merged_compose)

        if args.keys:
            if await are_you_sure("You specified -k/--keys, which will print all newly created keys to stdout."):
                print(f"walkoff_encryption_key:\t\t{wek.decode()}")
                print(f"walkoff_internal_key:\t\t{wik.decode()}")
                print(f"walkoff_postgres_key:\t\t{wpk.decode()}")
                print(f"walkoff_minio_access_key:\t{wmak.decode()}")
                print(f"walkoff_minio_secret_key:\t{wmsk.decode()}\n\n")

        logger.info("Walkoff stack deployed, it may take a little time to converge. \n"
                    "Use 'docker stack services walkoff' to check on Walkoff services. \n"
                    "Web interface should be available at 'https://127.0.0.1:8080' once walkoff_resource_nginx is up.")

        return return_code

    async def down(self):

        # Set up a subcommand parser
        parser = argparse.ArgumentParser(description="Remove the WALKOFF stack and optionally related artifacts.")
        parser.add_argument("-k", "--key", action="store_true",
                            help="Removes the walkoff_encryption_key secret.")
        parser.add_argument("-r", "--registry", action="store_true",
                            help="Clears the registry bind mount directory.")
        parser.add_argument("-v", "--volume", action="store_true", help="Clears the postgresql volume")
        parser.add_argument("-d", "--debug", action="store_true",
                            help="Set log level to debug.")

        # Parse out the command
        args = parser.parse_args(sys.argv[2:])

        if args.debug:
            logger.setLevel("DEBUG")
            docker_logger.setLevel("DEBUG")

        proc = await rm_stack("walkoff")

        # if not args.skipnetwork:
        #     logger.info("Waiting for containers to exit and network to be removed...")
        #     await exponential_wait(check_for_network, [self.docker_client], "Network walkoff_default still exists")

        if args.key:
            if await are_you_sure("Deleting encryption key will render database unreadable, so it will be cleared. "
                                  "This will delete all workflows, execution results, globals, users, roles, etc. "):
                await delete_encryption_key(self.docker_client, "walkoff_encryption_key")
                await delete_encryption_key(self.docker_client, "walkoff_internal_key")
                await delete_encryption_key(self.docker_client, "walkoff_postgres_key")
                await delete_dir_contents(static.POSTGRES_DATA_PATH)

        if args.registry:
            await delete_dir_contents(static.REGISTRY_DATA_PATH)
            await delete_dir_contents(static.MINIO_DATA_PATH)
            await delete_encryption_key(self.docker_client, "walkoff_minio_access_key")
            await delete_encryption_key(self.docker_client, "walkoff_minio_secret_key")

        if args.volume:
            await remove_volume("walkoff_postgres-data", wait=True)


        logger.info("Walkoff stack removed, it may take a little time to stop all services. "
                    "It is OK if the walkoff_default network is not fully removed.")

        return proc.returncode

    async def refresh(self):
        parser = argparse.ArgumentParser(description="Rebuild a specific service and force it to update.")
        parser.add_argument("-s", "--service",
                            help="Name of the service to rebuild and update. "
                                 "You can specify a prefix ('walkoff_app' or 'walkoff_core') "
                                 "to rebuild all in that category.")

        args = parser.parse_args(sys.argv[2:])

        compose = parse_yaml(config.TMP_COMPOSE)['services']
        service_yaml = compose.get(args.service)

        if service_yaml:
            if await are_you_sure("Forcing a service to update will disrupt any work it is currently doing. "
                                  "It is not yet guaranteed that a service will pick back up where it left off. "):
                if "build" in service_yaml:
                    await build_image(self.docker_client, service_yaml["image"],
                                      service_yaml["build"]["dockerfile"],
                                      service_yaml["build"]["context"],
                                      self.dockerignore)
                    await push_image(self.docker_client, service_yaml["image"])

                service_name = f"walkoff_{args.service}"
                await force_service_update(self.docker_client, service_name, service_yaml["image"])
        else:
            services = list(compose.keys())
            services.sort()
            logger.exception(f"No such service, valid services: {services}.")
Example #28
0
        "Callithrix jacchus - Common marmoset",
    ),
    (
        ["melanogaster", "fruit fly"],
        None,
        "http://purl.obolibrary.org/obo/NCBITaxon_7227",
        "Drosophila melanogaster - Fruit fly",
    ),
]


@lru_cache(maxsize=None)
@tenacity.retry(
    reraise=True,
    stop=tenacity.stop_after_attempt(3),
    wait=tenacity.wait_exponential(exp_base=1.25, multiplier=1.25),
)
def parse_purlobourl(
        url: str,
        lookup: Optional[Tuple[str, ...]] = None) -> Optional[Dict[str, str]]:
    """Parse an Ontobee URL to return properties of a Class node

    :param url: Ontobee URL
    :param lookup: list of XML nodes to lookup
    :return: dictionary containing found nodes
    """

    req = requests.get(url, allow_redirects=True)
    req.raise_for_status()
    doc = parseString(req.text)
    for elfound in doc.getElementsByTagName("Class"):
    # OVERRIDE on_exception
    # If stream error, wait for 5 minutes
    def on_exception(self, exception):
        logging.critical("Stream error")
        print(str(datetime.datetime.now()) + ": STREAM ERROR - check log")
        logging.critical(str(exception))
    
    # Function for threads to continually check the queue and clean tweets
    def cleanTweets(self):
        while True:
            cleanAndStore(self.q.get())
            self.q.task_done()

# Wrapper function for the stream so that when timeouts occur
# tenacity will just retry the connection
@retry(wait=wait_exponential(multiplier=1, min=5, max=60))
def tenacityStream(stream):
    try:
        stream.filter(languages = ["en"], track = keywords, is_async = True, stall_warnings = True)
    except Exception as e:
        logging.info("Retrying stream...")

# Main Function
def main():
    # Log stream start
    logging.info("Stream started")
    # Set up a stream listener
    btcListener = BitcoinListener()
    # Set up stream
    authentication = OAuthHandler(twitterAuth["consumer_key"], twitterAuth["consumer_secret"])
    authentication.set_access_token(twitterAuth["access_token"], twitterAuth["access_secret"])
Example #30
0
from django.apps import apps


# Models need to be imported like this in order to avoid cyclic import issues with celery
def get_model(model_name):
    return apps.get_model(app_label='app', model_name=model_name)


logging.basicConfig(stream=sys.stderr, level=logging.ERROR)

logger = logging.getLogger(__name__)

TENACITY_ARGUMENTS = {
    'reraise': True,
    'wait': tenacity.wait_exponential(multiplier=60, max=600),
    'before_sleep': tenacity.before_sleep_log(logger,
                                              logging.ERROR,
                                              exc_info=True)
}
TENACITY_ARGUMENTS_FAST = {
    'reraise': True,
    'stop': tenacity.stop_after_attempt(5),
    'wait': tenacity.wait_exponential(multiplier=30),
    'before_sleep': tenacity.before_sleep_log(logger,
                                              logging.ERROR,
                                              exc_info=True)
}


def batch_qs(qs, batch_size=50000):
class ConfirmationSender:
    def __init__(
        self,
        *,
        transfer_event_queue: Queue,
        home_bridge_contract: Contract,
        private_key: bytes,
        gas_price: web3types.Wei,
        max_reorg_depth: int,
        pending_transaction_queue: Queue,
        sanity_check_transfer: Callable,
    ):
        self.private_key = private_key
        self.address = PrivateKey(
            self.private_key).public_key.to_canonical_address()
        self.address_hex = PrivateKey(
            self.private_key).public_key.to_checksum_address()
        if not is_bridge_validator(home_bridge_contract, self.address):
            logger.warning(
                f"The address {to_checksum_address(self.address)} is not a bridge validator to confirm "
                f"transfers on the home bridge contract!")

        self.transfer_event_queue = transfer_event_queue
        self.home_bridge_contract = home_bridge_contract
        self.gas_price = gas_price
        self.max_reorg_depth = max_reorg_depth
        self.w3 = self.home_bridge_contract.web3
        self.pending_transaction_queue = pending_transaction_queue
        self.sanity_check_transfer = sanity_check_transfer
        self.chain_id = int(self.w3.eth.chainId)

        self.services = [
            Service("send-confirmation-transactions",
                    self.send_confirmation_transactions)
        ]

        self.is_parity = self.w3.clientVersion.startswith(
            "Parity") or self.w3.clientVersion.startswith("OpenEthereum")

    @tenacity.retry(
        wait=tenacity.wait_exponential(multiplier=1, min=5, max=120),
        before_sleep=tenacity.before_sleep_log(logger, logging.WARN),
        retry=tenacity.retry_if_exception(lambda exc: isinstance(
            exc, Exception) and not isinstance(exc, NonceTooLowException)),
    )
    def _rpc_send_raw_transaction(self, raw_transaction):
        try:
            return self.w3.eth.sendRawTransaction(raw_transaction)
        except ValueError as exc:
            if is_nonce_too_low_exception(exc):
                raise NonceTooLowException("nonce too low") from exc
            else:
                raise exc

    @tenacity.retry(
        wait=tenacity.wait_exponential(multiplier=1, min=5, max=120),
        before_sleep=tenacity.before_sleep_log(logger, logging.WARN),
    )
    def get_next_nonce(self):
        if self.is_parity:
            return int(
                self.w3.manager.request_blocking("parity_nextNonce",
                                                 [self.address_hex]),
                16,
            )
        else:
            return self.w3.eth.getTransactionCount(self.address, "pending")

    @tenacity.retry(
        wait=tenacity.wait_exponential(multiplier=1, min=5, max=120),
        before_sleep=tenacity.before_sleep_log(logger, logging.WARN),
        retry=tenacity.retry_if_exception(
            lambda exc: isinstance(exc, NonceTooLowException)),
    )
    def send_confirmation_from_transfer_event(self, transfer_event):
        nonce = self.get_next_nonce()
        transaction = self.prepare_confirmation_transaction(
            transfer_event=transfer_event, nonce=nonce, chain_id=self.chain_id)
        assert transaction is not None
        self.send_confirmation_transaction(transaction)

    def send_confirmation_transactions(self):
        while True:
            transfer_event = self.transfer_event_queue.get()
            try:
                self.sanity_check_transfer(transfer_event)
            except Exception as exc:
                raise SystemExit(
                    f"Internal error: sanity check failed for {transfer_event}: {exc}"
                ) from exc
            self.send_confirmation_from_transfer_event(transfer_event)

    run = send_confirmation_transactions

    def prepare_confirmation_transaction(self, transfer_event,
                                         nonce: web3types.Nonce,
                                         chain_id: int):
        transfer_hash = compute_transfer_hash(transfer_event)
        transaction_hash = transfer_event.transactionHash
        amount = transfer_event.args.value
        recipient = transfer_event.args["from"]

        logger.info(
            "confirmTransfer(transferHash=%s transactionHash=%s amount=%s recipient=%s) with nonce=%s, chain_id=%s",
            transfer_hash.hex(),
            transaction_hash.hex(),
            amount,
            recipient,
            nonce,
            chain_id,
        )
        # hard code gas limit to avoid executing the transaction (which would fail as the sender
        # address is not defined before signing the transaction, but the contract asserts that
        # it's a validator)
        transaction = self.home_bridge_contract.functions.confirmTransfer(
            transferHash=transfer_hash,
            transactionHash=transaction_hash,
            amount=amount,
            recipient=recipient,
        ).buildTransaction({
            "gasPrice": self.gas_price,
            "nonce": nonce,
            "gas": CONFIRMATION_TRANSACTION_GAS_LIMIT,  # type: ignore
            "chainId": chain_id,
        })
        signed_transaction = self.w3.eth.account.sign_transaction(
            transaction, self.private_key)

        return signed_transaction

    def send_confirmation_transaction(self, transaction):
        tx_hash = self._rpc_send_raw_transaction(transaction.rawTransaction)
        self.pending_transaction_queue.put(transaction)
        logger.info(f"Sent confirmation transaction {tx_hash.hex()}")
        return tx_hash
Example #32
0

def check_headers_available(headers):
    product = '854239'
    country_iso = '490'
    res = fetch_tariff_excel_response(product, country_iso, headers)
    df = pd.read_excel(BytesIO(res.content))
    try:
        assert len(df) > 0
        return True
    except:
        return False


@retry(stop=stop_after_attempt(10),
       wait=wait_fixed(10) + wait_exponential(multiplier=1, max=20))
def reload_available_headers():
    headers = get_new_headers()
    headers_is_available = check_headers_available(headers)
    if headers_is_available:
        return headers
    else:
        raise


def get_decent_headers():
    try:
        with open(CACHED_HEADERS, 'r') as f:
            headers = json.load(f)
    except (json.JSONDecodeError, FileNotFoundError) as e:
        headers = reload_available_headers()
Example #33
0
    for rule in current_rules:
        if '-i %s' % vif in rule and '--among-src' in rule:
            ebtables(['-D', chain] + rule.split())


def _delete_mac_spoofing_protection(vifs, current_rules):
    # delete the jump rule and then delete the whole chain
    jumps = [vif for vif in vifs
             if _mac_vif_jump_present(vif, current_rules)]
    for vif in jumps:
        ebtables(['-D', 'FORWARD', '-i', vif, '-j',
                  _mac_chain_name(vif)])
    for vif in vifs:
        chain = _mac_chain_name(vif)
        if chain_exists(chain, current_rules):
            ebtables(['-X', chain])


# Used to scope ebtables commands in testing
NAMESPACE = None


@tenacity.retry(
    wait=tenacity.wait_exponential(multiplier=0.01),
    retry=tenacity.retry_if_exception(lambda e: e.returncode == 255),
    reraise=True
)
def ebtables(comm):
    execute = ip_lib.IPWrapper(NAMESPACE).netns.execute
    return execute(['ebtables', '--concurrent'] + comm, run_as_root=True)
Example #34
0
class DesignateTests(BaseDesignateTest):
    """Designate charm restart and pause tests."""

    TEST_DOMAIN = 'amuletexample.com.'
    TEST_NS1_RECORD = 'ns1.{}'.format(TEST_DOMAIN)
    TEST_NS2_RECORD = 'ns2.{}'.format(TEST_DOMAIN)
    TEST_WWW_RECORD = "www.{}".format(TEST_DOMAIN)
    TEST_RECORD = {TEST_WWW_RECORD: '10.0.0.23'}

    def test_900_restart_on_config_change(self):
        """Checking restart happens on config change.

        Change disk format and assert then change propagates to the correct
        file and that services are restarted as a result
        """
        # Expected default and alternate values
        set_default = {'debug': 'False'}
        set_alternate = {'debug': 'True'}

        # Services which are expected to restart upon config change,
        # and corresponding config files affected by the change
        conf_file = '/etc/designate/designate.conf'

        # Make config change, check for service restarts
        self.restart_on_changed(conf_file, set_default, set_alternate,
                                {'DEFAULT': {
                                    'debug': ['False']
                                }}, {'DEFAULT': {
                                    'debug': ['True']
                                }}, self.designate_svcs)

    def test_910_pause_and_resume(self):
        """Run pause and resume tests.

        Pause service and check services are stopped then resume and check
        they are started
        """
        with self.pause_resume(self.designate_svcs, pgrep_full=False):
            logging.info("Testing pause resume")

    def _get_server_id(self, server_name=None, server_id=None):
        for srv in self.server_list():
            if isinstance(srv, dict):
                if srv['id'] == server_id or srv['name'] == server_name:
                    return srv['id']
            elif srv.name == server_name or srv.id == server_id:
                return srv.id
        return None

    def _wait_on_server_gone(self, server_id):
        @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1,
                                                       min=5,
                                                       max=10),
                        reraise=True)
        def wait():
            logging.debug('Waiting for server %s to disappear', server_id)
            if self._get_server_id(server_id=server_id):
                raise Exception("Server Exists")

        self.server_delete(server_id)
        return wait()

    def test_400_server_creation(self):
        """Simple api calls to create a server."""
        # Designate does not allow the last server to be deleted so ensure
        # that ns1 is always present
        if self.post_xenial_queens:
            logging.info('Skipping server creation tests for Queens and above')
            return

        if not self._get_server_id(server_name=self.TEST_NS1_RECORD):
            server = servers.Server(name=self.TEST_NS1_RECORD)
            new_server = self.server_create(server)
            self.assertIsNotNone(new_server)

        logging.debug('Checking if server exists before trying to create it')
        old_server_id = self._get_server_id(server_name=self.TEST_NS2_RECORD)
        if old_server_id:
            logging.debug('Deleting old server')
            self._wait_on_server_gone(old_server_id)

        logging.debug('Creating new server')
        server = servers.Server(name=self.TEST_NS2_RECORD)
        new_server = self.server_create(server)
        self.assertIsNotNone(new_server, "Failed to Create Server")
        self._wait_on_server_gone(self._get_server_id(self.TEST_NS2_RECORD))

    def _get_domain_id(self, domain_name=None, domain_id=None):
        for dom in self.domain_list():
            if isinstance(dom, dict):
                if dom['id'] == domain_id or dom['name'] == domain_name:
                    return dom['id']
            elif dom.id == domain_id or dom.name == domain_name:
                return dom.id
        return None

    def _wait_on_domain_gone(self, domain_id):
        @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1,
                                                       min=5,
                                                       max=10),
                        reraise=True)
        def wait():
            logging.debug('Waiting for domain %s to disappear', domain_id)
            if self._get_domain_id(domain_id=domain_id):
                raise Exception("Domain Exists")

        self.domain_delete(domain_id)
        wait()

    @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=5,
                                                   max=10),
                    reraise=True)
    def _wait_to_resolve_test_record(self):
        dns_ip = zaza_juju.get_relation_from_unit(
            'designate/0', 'designate-bind/0',
            'dns-backend').get('private-address')

        logging.info('Waiting for dns record to propagate @ {}'.format(dns_ip))
        lookup_cmd = [
            'dig', '+short', '@{}'.format(dns_ip), self.TEST_WWW_RECORD
        ]
        cmd_out = subprocess.check_output(lookup_cmd,
                                          universal_newlines=True).rstrip()
        if not self.TEST_RECORD[self.TEST_WWW_RECORD] == cmd_out:
            raise Exception("Record Doesn't Exist")

    def test_400_domain_creation(self):
        """Simple api calls to create domain."""
        logging.debug('Checking if domain exists before trying to create it')
        old_dom_id = self._get_domain_id(domain_name=self.TEST_DOMAIN)
        if old_dom_id:
            logging.debug('Deleting old domain')
            self._wait_on_domain_gone(old_dom_id)

        logging.debug('Creating new domain')
        domain = domains.Domain(name=self.TEST_DOMAIN,
                                email="*****@*****.**")

        if self.post_xenial_queens:
            new_domain = self.domain_create(name=domain.name,
                                            email=domain.email)
        else:
            new_domain = self.domain_create(domain)
        self.assertIsNotNone(new_domain)

        logging.debug('Creating new test record')
        _record = records.Record(name=self.TEST_WWW_RECORD,
                                 type="A",
                                 data=self.TEST_RECORD[self.TEST_WWW_RECORD])

        if self.post_xenial_queens:
            domain_id = new_domain['id']
            self.designate.recordsets.create(domain_id, _record.name,
                                             _record.type, [_record.data])
        else:
            domain_id = new_domain.id
            self.designate.records.create(domain_id, _record)

        self._wait_to_resolve_test_record()

        logging.debug('Tidy up delete test record')
        self._wait_on_domain_gone(domain_id)
        logging.debug('OK')
Example #35
0
    except AttributeError:
        fname = f.__name__

    @six.wraps(f)
    def _return_none_on_failure(*args, **kwargs):
        try:
            return f(*args, **kwargs)
        except Exception as e:
            LOG.critical("Unexpected error while calling %s: %s",
                         fname, e, exc_info=True)

    return _return_none_on_failure


# Retry with exponential backoff for up to 1 minute
wait_exponential = tenacity.wait_exponential(multiplier=0.5, max=60)

retry_on_exception = tenacity.Retrying(wait=wait_exponential)


class _retry_on_exception_and_log(tenacity.retry_if_exception_type):
    def __init__(self, msg):
        super(_retry_on_exception_and_log, self).__init__()
        self.msg = msg

    def __call__(self, attempt):
        if attempt.failed:
            LOG.error(self.msg, exc_info=attempt.exception())
        return super(_retry_on_exception_and_log, self).__call__(attempt)

Example #36
0

def fairlay_outcome_printer(fairlay_df, bool_vec, title=None):
    res = compute_fairlay_outcomes(fairlay_df, bool_vec)
    total_outcome, total_ev, empirical_ci, n_matches = res

    if title is not None:
        print(title + "\n" + "-" * len(title))
    print("Maps included: {}".format(n_matches))
    print("Total EV: {:.2f} ({:.2f}, {:.2f})".format(total_ev, empirical_ci[0],
                                                     empirical_ci[1]))
    print("Total outcome: {}".format(total_outcome))
    print("")


@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=2, max=30),
                stop=tenacity.stop_after_attempt(10))
def _fetch_fairlay_data(url):
    req = requests.get(url)
    assert req.status_code == 200
    return req


def fetch_markets(base_url=FAIRLAY_BASE_URL, qry_params=None):
    """Fetch 2 data from fairlay.com.

    Currently only supporting category 32, i.e. esports. See
    https://fairlay.com/api/#/data/?id=market-categories. Also, only Dota 2
    markets, i.e. those with 'dota' in the competition name, are returned.
    """
    MAX_RECORDS = 100000
Example #37
0
    auth = loading.load_auth_from_conf_options(CONF, group)
    session = loading.load_session_from_conf_options(CONF, group, auth=auth)
    return session


def _get_ironic_session():
    global _IRONIC_SESSION

    if not _IRONIC_SESSION:
        _IRONIC_SESSION = get_session(IRONIC_GROUP)
    return _IRONIC_SESSION


@tenacity.retry(
    retry=tenacity.retry_if_exception_type(openstack.exceptions.NotSupported),
    wait=tenacity.wait_exponential(max=30))
def get_client():
    """Get an ironic client connection."""
    session = _get_ironic_session()

    try:
        return openstack.connection.Connection(
            session=session, oslo_conf=CONF).baremetal
    except openstack.exceptions.NotSupported as exc:
        LOG.error('Ironic API might not be running, failed to establish a '
                  'connection with ironic, reason: %s. Retrying ...', exc)
        raise
    except Exception as exc:
        LOG.error('Failed to establish a connection with ironic, reason: %s',
                  exc)
        raise
Example #38
0
db_name = sys.argv[2]

logging.info('Cluster: %s', cluster_url)
logging.info('Database: %s', db_name)

logging.info('Creating connection string.')
kcsb = KustoConnectionStringBuilder.with_aad_managed_service_identity_authentication(
    cluster_url)

logging.info('Creating client.')
client = KustoClient(kcsb)


@retry(
    stop=stop_after_attempt(5),
    wait=wait_exponential(multiplier=1, min=4, max=32),
    retry=retry_if_exception(lambda x: not isinstance(x, KustoServiceError)),
    after=after_log(logging, logging.DEBUG))
def execute_command(command: str) -> None:
    client.execute_mgmt(db_name, command)


def create_table_set(base_name: str) -> None:
    commands = init_script.replace('{TABLE_BASE}', base_name).split('\n\n')
    for command in commands:
        execute_command(command)


logging.info('Creating Usage Tables.')
create_table_set('Usage')
Example #39
0
def boot_worker() -> None:
    # Clone autoland
    def clone_autoland() -> None:
        logger.info(f"Cloning autoland in {REPO_DIR}...")
        repository.clone(REPO_DIR, "https://hg.mozilla.org/integration/autoland")

    def extract_past_failures_label() -> None:
        try:
            utils.extract_file(
                os.path.join("data", test_scheduling.PAST_FAILURES_LABEL_DB)
            )
            logger.info("Label-level past failures DB extracted.")
        except FileNotFoundError:
            assert ALLOW_MISSING_MODELS
            logger.info(
                "Label-level past failures DB not extracted, but missing models are allowed."
            )

    def extract_failing_together_label() -> None:
        try:
            utils.extract_file(
                os.path.join("data", test_scheduling.FAILING_TOGETHER_LABEL_DB)
            )
            logger.info("Failing together label DB extracted.")
        except FileNotFoundError:
            assert ALLOW_MISSING_MODELS
            logger.info(
                "Failing together label DB not extracted, but missing models are allowed."
            )

    def extract_failing_together_config_group() -> None:
        try:
            utils.extract_file(
                os.path.join("data", test_scheduling.FAILING_TOGETHER_CONFIG_GROUP_DB)
            )
            logger.info("Failing together config/group DB extracted.")
        except FileNotFoundError:
            assert ALLOW_MISSING_MODELS
            logger.info(
                "Failing together config/group DB not extracted, but missing models are allowed."
            )

    def extract_past_failures_group() -> None:
        try:
            utils.extract_file(
                os.path.join("data", test_scheduling.PAST_FAILURES_GROUP_DB)
            )
            logger.info("Group-level past failures DB extracted.")
        except FileNotFoundError:
            assert ALLOW_MISSING_MODELS
            logger.info(
                "Group-level past failures DB not extracted, but missing models are allowed."
            )

    def extract_touched_together() -> None:
        try:
            utils.extract_file(
                os.path.join("data", test_scheduling.TOUCHED_TOGETHER_DB)
            )
            logger.info("Touched together DB extracted.")
        except FileNotFoundError:
            assert ALLOW_MISSING_MODELS
            logger.info(
                "Touched together DB not extracted, but missing models are allowed."
            )

    def extract_commits() -> bool:
        try:
            utils.extract_file(f"{repository.COMMITS_DB}.zst")
            logger.info("Commits DB extracted.")
            return True
        except FileNotFoundError:
            logger.info("Commits DB not extracted, but missing models are allowed.")
            assert ALLOW_MISSING_MODELS
            return False

    def extract_commit_experiences() -> None:
        try:
            utils.extract_file(os.path.join("data", repository.COMMIT_EXPERIENCES_DB))
            logger.info("Commit experiences DB extracted.")
        except FileNotFoundError:
            logger.info(
                "Commit experiences DB not extracted, but missing models are allowed."
            )
            assert ALLOW_MISSING_MODELS

    @tenacity.retry(
        stop=tenacity.stop_after_attempt(7),
        wait=tenacity.wait_exponential(multiplier=1, min=1, max=8),
    )
    def retrieve_schedulable_tasks() -> None:
        r = requests.get(
            "https://hg.mozilla.org/integration/autoland/json-pushes?version=2&tipsonly=1"
        )
        r.raise_for_status()
        revs = [
            push_obj["changesets"][0]
            for push_id, push_obj in r.json()["pushes"].items()
        ]

        logger.info(f"Retrieving known tasks from {revs}")

        # Store in a file the list of tasks in the latest autoland pushes.
        # We use more than one to protect ourselves from broken decision tasks.
        known_tasks = set()
        for rev in revs:
            r = requests.get(
                f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.autoland.revision.{rev}.taskgraph.decision/artifacts/public/target-tasks.json"
            )
            if r.ok:
                known_tasks.update(r.json())

        logger.info(f"Retrieved {len(known_tasks)} tasks")

        assert len(known_tasks) > 0

        with open("known_tasks", "w") as f:
            f.write("\n".join(known_tasks))

    with concurrent.futures.ThreadPoolExecutor() as executor:
        clone_autoland_future = executor.submit(clone_autoland)

        retrieve_schedulable_tasks_future = executor.submit(retrieve_schedulable_tasks)

        commits_db_extracted = extract_commits()
        extract_commit_experiences()
        extract_touched_together()
        extract_past_failures_label()
        extract_past_failures_group()
        extract_failing_together_label()
        extract_failing_together_config_group()

        if commits_db_extracted:
            # Update the commits DB.
            logger.info("Browsing all commits...")
            for commit in repository.get_commits():
                pass
            logger.info("All commits browsed.")

            # Wait repository to be cloned, as it's required to call repository.download_commits.
            logger.info("Waiting autoland to be cloned...")
            clone_autoland_future.result()

            rev_start = "children({})".format(commit["node"])
            logger.info("Updating commits DB...")
            commits = repository.download_commits(
                REPO_DIR, rev_start, use_single_process=True
            )
            logger.info("Commits DB updated.")

            logger.info("Updating touched together DB...")
            if len(commits) > 0:
                # Update the touched together DB.
                update_touched_together_gen = test_scheduling.update_touched_together()
                next(update_touched_together_gen)

                update_touched_together_gen.send(commits[-1]["node"])

                try:
                    update_touched_together_gen.send(None)
                except StopIteration:
                    pass
            logger.info("Touched together DB updated.")

        # Wait list of schedulable tasks to be downloaded and written to disk.
        retrieve_schedulable_tasks_future.result()

    logger.info("Worker boot done")
Example #40
0
    def loop_client_recv(self):
        # The built-in client loop_forever seems busted (it doesn't retry
        # under all exceptions, so just do it ourselves...); arg...

        def on_connect(client, userdata, flags, rc):
            if rc == mqtt.MQTT_ERR_SUCCESS:
                self.log.info("MQTT connected to %s:%s over %s",
                              self.config['firehose_host'],
                              self.config['firehose_port'],
                              self.config['firehose_transport'])
                client.subscribe('#')
            else:
                self.log.error(
                    "MQTT not connected to %s:%s over %s, rc=%s",
                    self.config['firehose_host'],
                    self.config['firehose_port'],
                    self.config['firehose_transport'], rc)

        def on_message(client, userdata, msg):
            if not msg.topic or not msg.payload:
                return
            self.log.info(("Dispatching message on topic=%s"
                           " with payload=%s"), msg.topic, msg.payload)
            try:
                payload = msg.payload
                if isinstance(payload, six.binary_type):
                    payload = payload.decode("utf8")
                details = {'event': json.loads(payload)}
            except (UnicodeError, ValueError):
                self.log.exception(
                    "Received corrupted/invalid payload: %s", msg.payload)
            else:
                self.work_queue.put(details)

        @tenacity.retry(
            wait=tenacity.wait_exponential(multiplier=1, max=30),
            before=tenacity.before_log(self.log, logging.INFO))
        def loop_forever_until_dead():
            if self.dying:
                return
            client = mqtt.Client(transport=self.config['firehose_transport'])
            client.on_connect = on_connect
            client.on_message = on_message
            try:
                client.connect(self.config['firehose_host'],
                               port=int(self.config['firehose_port']))
                max_timeout = 1
                while not self.dying:
                    rc = mqtt.MQTT_ERR_SUCCESS
                    start = time.time()
                    elapsed = 0
                    while rc == mqtt.MQTT_ERR_SUCCESS and (elapsed < max_timeout):
                        rc = client.loop(timeout=max(0, max_timeout - elapsed))
                        elapsed = time.time() - start
                    if not self.dying:
                        time.sleep(0.1)
            except Exception:
                self.log.exception("Failed mqtt client usage, retrying")
                raise

        loop_forever_until_dead()
Example #41
0
class Backend(ovs_idl.Backend):
    lookup_table = {}
    ovsdb_connection = None

    def __init__(self, connection):
        self.ovsdb_connection = connection
        super(Backend, self).__init__(connection)

    def start_connection(self, connection):
        try:
            self.ovsdb_connection.start()
        except Exception as e:
            connection_exception = OvsdbConnectionUnavailable(
                db_schema=self.schema, error=e)
            LOG.exception(connection_exception)
            raise connection_exception

    @property
    def idl(self):
        return self.ovsdb_connection.idl

    @property
    def tables(self):
        return self.idl.tables

    _tables = tables

    @n_utils.classproperty
    def connection_string(cls):
        raise NotImplementedError()

    @n_utils.classproperty
    def schema_helper(cls):
        # SchemaHelper.get_idl_schema() sets schema_json to None which is
        # called in Idl.__init__(), so if we've done that return new helper
        try:
            if cls._schema_helper.schema_json:
                return cls._schema_helper
        except AttributeError:
            pass

        ovsdb_monitor._check_and_set_ssl_files(cls.schema)
        cls._schema_helper = idlutils.get_schema_helper(cls.connection_string,
                                                        cls.schema)
        return cls._schema_helper

    @classmethod
    def schema_has_table(cls, table_name):
        return table_name in cls.schema_helper.schema_json['tables']

    def is_table_present(self, table_name):
        return table_name in self._tables

    def is_col_present(self, table_name, col_name):
        return self.is_table_present(table_name) and (
            col_name in self._tables[table_name].columns)

    def create_transaction(self, check_error=False, log_errors=True):
        return idl_trans.Transaction(
            self, self.ovsdb_connection, self.ovsdb_connection.timeout,
            check_error, log_errors)

    # Check for a column match in the table. If not found do a retry with
    # a stop delay of 10 secs. This function would be useful if the caller
    # wants to verify for the presence of a particular row in the table
    # with the column match before doing any transaction.
    # Eg. We can check if Logical_Switch row is present before adding a
    # logical switch port to it.
    @tenacity.retry(retry=tenacity.retry_if_exception_type(RuntimeError),
                    wait=tenacity.wait_exponential(),
                    stop=tenacity.stop_after_delay(10),
                    reraise=True)
    def check_for_row_by_value_and_retry(self, table, column, match):
        try:
            idlutils.row_by_value(self.idl, table, column, match)
        except idlutils.RowNotFound:
            msg = (_("%(match)s does not exist in %(column)s of %(table)s")
                   % {'match': match, 'column': column, 'table': table})
            raise RuntimeError(msg)
Example #42
0
    def request(self,
                method,
                additional_headers=None,
                retry=True,
                timeout=None,
                auth=None,
                use_gzip_encoding=None,
                params=None,
                max_attempts=None,
                **kwargs):
        """
        Make an HTTP request by calling self._request with backoff retry.

        :param method: request method
        :type method: str
        :param additional_headers: additional headers to include in the request
        :type additional_headers: dict[str, str]
        :param retry: boolean indicating whether to retry if the request fails
        :type retry: boolean
        :param timeout: timeout in seconds, overrides default_timeout_secs
        :type timeout: float
        :param timeout: timeout in seconds
        :type timeout: float
        :param auth: auth scheme for the request
        :type auth: requests.auth.AuthBase
        :param use_gzip_encoding: boolean indicating whether to pass gzip
                                  encoding in the request headers or not
        :type use_gzip_encoding: boolean | None
        :param params: additional params to include in the request
        :type params: str | dict[str, T] | None
        :param max_attempts: maximum number of attempts to try for any request
        :type max_attempts: int
        :param kwargs: additional arguments to pass to requests.request
        :type kwargs: dict[str, T]
        :return: HTTP response
        :rtype: requests.Response
        """
        request = self._request

        if retry:
            if max_attempts is None:
                max_attempts = self.default_max_attempts

            # We retry only when it makes sense: either due to a network
            # partition (e.g. connection errors) or if the request failed
            # due to a server error such as 500s, timeouts, and so on.
            request = tenacity.retry(
                stop=tenacity.stop_after_attempt(max_attempts),
                wait=tenacity.wait_exponential(),
                retry=tenacity.retry_if_exception_type((
                    requests.exceptions.Timeout,
                    requests.exceptions.ConnectionError,
                    MesosServiceUnavailableException,
                    MesosInternalServerErrorException,
                )),
                reraise=True,
            )(request)

        try:
            return request(
                method=method,
                additional_headers=additional_headers,
                timeout=timeout,
                auth=auth,
                use_gzip_encoding=use_gzip_encoding,
                params=params,
                **kwargs
            )
        # If the request itself failed, an exception subclassed from
        # RequestException will be raised. Catch this and reraise as
        # MesosException since we want the caller to be able to catch
        # and handle this.
        except requests.exceptions.RequestException as err:
            raise MesosException('Request failed', err)
Example #43
0
class PodManager(LoggingMixin):
    """
    Helper class for creating, monitoring, and otherwise interacting with Kubernetes pods
    for use with the KubernetesPodOperator
    """
    def __init__(
        self,
        kube_client: client.CoreV1Api = None,
        in_cluster: bool = True,
        cluster_context: Optional[str] = None,
    ):
        """
        Creates the launcher.

        :param kube_client: kubernetes client
        :param in_cluster: whether we are in cluster
        :param cluster_context: context of the cluster
        """
        super().__init__()
        self._client = kube_client or get_kube_client(
            in_cluster=in_cluster, cluster_context=cluster_context)
        self._watch = watch.Watch()

    def run_pod_async(self, pod: V1Pod, **kwargs) -> V1Pod:
        """Runs POD asynchronously"""
        sanitized_pod = self._client.api_client.sanitize_for_serialization(pod)
        json_pod = json.dumps(sanitized_pod, indent=2)

        self.log.debug('Pod Creation Request: \n%s', json_pod)
        try:
            resp = self._client.create_namespaced_pod(
                body=sanitized_pod, namespace=pod.metadata.namespace, **kwargs)
            self.log.debug('Pod Creation Response: %s', resp)
        except Exception as e:
            self.log.exception(
                'Exception when attempting to create Namespaced Pod: %s',
                str(json_pod).replace("\n", " "))
            raise e
        return resp

    def delete_pod(self, pod: V1Pod) -> None:
        """Deletes POD"""
        try:
            self._client.delete_namespaced_pod(pod.metadata.name,
                                               pod.metadata.namespace,
                                               body=client.V1DeleteOptions())
        except ApiException as e:
            # If the pod is already deleted
            if e.status != 404:
                raise

    @tenacity.retry(
        stop=tenacity.stop_after_attempt(3),
        wait=tenacity.wait_random_exponential(),
        reraise=True,
        retry=tenacity.retry_if_exception(should_retry_start_pod),
    )
    def create_pod(self, pod: V1Pod) -> V1Pod:
        """Launches the pod asynchronously."""
        return self.run_pod_async(pod)

    def await_pod_start(self, pod: V1Pod, startup_timeout: int = 120) -> None:
        """
        Waits for the pod to reach phase other than ``Pending``

        :param pod:
        :param startup_timeout: Timeout (in seconds) for startup of the pod
            (if pod is pending for too long, fails task)
        :return:
        """
        curr_time = datetime.now()
        while True:
            remote_pod = self.read_pod(pod)
            if remote_pod.status.phase != PodPhase.PENDING:
                break
            self.log.warning("Pod not yet started: %s", pod.metadata.name)
            delta = datetime.now() - curr_time
            if delta.total_seconds() >= startup_timeout:
                msg = (
                    f"Pod took longer than {startup_timeout} seconds to start. "
                    "Check the pod events in kubernetes to determine why.")
                raise PodLaunchFailedException(msg)
            time.sleep(1)

    def follow_container_logs(self, pod: V1Pod,
                              container_name: str) -> PodLoggingStatus:
        warnings.warn(
            "Method `follow_container_logs` is deprecated.  Use `fetch_container_logs` instead"
            "with option `follow=True`.",
            DeprecationWarning,
        )
        return self.fetch_container_logs(pod=pod,
                                         container_name=container_name,
                                         follow=True)

    def fetch_container_logs(
            self,
            pod: V1Pod,
            container_name: str,
            *,
            follow=False,
            since_time: Optional[DateTime] = None) -> PodLoggingStatus:
        """
        Follows the logs of container and streams to airflow logging.
        Returns when container exits.
        """
        def consume_logs(*,
                         since_time: Optional[DateTime] = None,
                         follow: bool = True) -> Optional[DateTime]:
            """
            Tries to follow container logs until container completes.
            For a long-running container, sometimes the log read may be interrupted
            Such errors of this kind are suppressed.

            Returns the last timestamp observed in logs.
            """
            timestamp = None
            try:
                logs = self.read_pod_logs(
                    pod=pod,
                    container_name=container_name,
                    timestamps=True,
                    since_seconds=(math.ceil(
                        (pendulum.now() -
                         since_time).total_seconds()) if since_time else None),
                    follow=follow,
                )
                for line in logs:
                    timestamp, message = self.parse_log_line(
                        line.decode('utf-8'))
                    self.log.info(message)
            except BaseHTTPError as e:
                self.log.warning(
                    "Reading of logs interrupted with error %r; will retry. "
                    "Set log level to DEBUG for traceback.",
                    e,
                )
                self.log.debug(
                    "Traceback for interrupted logs read for pod %r",
                    pod.metadata.name,
                    exc_info=True,
                )
            return timestamp or since_time

        # note: `read_pod_logs` follows the logs, so we shouldn't necessarily *need* to
        # loop as we do here. But in a long-running process we might temporarily lose connectivity.
        # So the looping logic is there to let us resume following the logs.
        last_log_time = since_time
        while True:
            last_log_time = consume_logs(since_time=last_log_time,
                                         follow=follow)
            if not self.container_is_running(pod,
                                             container_name=container_name):
                return PodLoggingStatus(running=False,
                                        last_log_time=last_log_time)
            if not follow:
                return PodLoggingStatus(running=True,
                                        last_log_time=last_log_time)
            else:
                self.log.warning(
                    'Pod %s log read interrupted but container %s still running',
                    pod.metadata.name,
                    container_name,
                )
                time.sleep(1)

    def await_container_completion(self, pod: V1Pod,
                                   container_name: str) -> None:
        while not self.container_is_running(pod=pod,
                                            container_name=container_name):
            time.sleep(1)

    def await_pod_completion(self, pod: V1Pod) -> V1Pod:
        """
        Monitors a pod and returns the final state

        :param pod: pod spec that will be monitored
        :return:  Tuple[State, Optional[str]]
        """
        while True:
            remote_pod = self.read_pod(pod)
            if remote_pod.status.phase in PodPhase.terminal_states:
                break
            self.log.info('Pod %s has phase %s', pod.metadata.name,
                          remote_pod.status.phase)
            time.sleep(2)
        return remote_pod

    def parse_log_line(self, line: str) -> Tuple[Optional[DateTime], str]:
        """
        Parse K8s log line and returns the final state

        :param line: k8s log line
        :return: timestamp and log message
        :rtype: Tuple[str, str]
        """
        split_at = line.find(' ')
        if split_at == -1:
            self.log.error(
                f"Error parsing timestamp (no timestamp in message '${line}'). "
                "Will continue execution but won't update timestamp")
            return None, line
        timestamp = line[:split_at]
        message = line[split_at + 1:].rstrip()
        try:
            last_log_time = cast(DateTime, pendulum.parse(timestamp))
        except ParserError:
            self.log.error(
                "Error parsing timestamp. Will continue execution but won't update timestamp"
            )
            return None, line
        return last_log_time, message

    def container_is_running(self, pod: V1Pod, container_name: str) -> bool:
        """Reads pod and checks if container is running"""
        remote_pod = self.read_pod(pod)
        return container_is_running(pod=remote_pod,
                                    container_name=container_name)

    @tenacity.retry(stop=tenacity.stop_after_attempt(3),
                    wait=tenacity.wait_exponential(),
                    reraise=True)
    def read_pod_logs(
        self,
        pod: V1Pod,
        container_name: str,
        tail_lines: Optional[int] = None,
        timestamps: bool = False,
        since_seconds: Optional[int] = None,
        follow=True,
    ) -> Iterable[bytes]:
        """Reads log from the POD"""
        additional_kwargs = {}
        if since_seconds:
            additional_kwargs['since_seconds'] = since_seconds

        if tail_lines:
            additional_kwargs['tail_lines'] = tail_lines

        try:
            return self._client.read_namespaced_pod_log(
                name=pod.metadata.name,
                namespace=pod.metadata.namespace,
                container=container_name,
                follow=follow,
                timestamps=timestamps,
                _preload_content=False,
                **additional_kwargs,
            )
        except BaseHTTPError:
            self.log.exception(
                'There was an error reading the kubernetes API.')
            raise

    @tenacity.retry(stop=tenacity.stop_after_attempt(3),
                    wait=tenacity.wait_exponential(),
                    reraise=True)
    def read_pod_events(self, pod: V1Pod) -> "CoreV1EventList":
        """Reads events from the POD"""
        try:
            return self._client.list_namespaced_event(
                namespace=pod.metadata.namespace,
                field_selector=f"involvedObject.name={pod.metadata.name}")
        except BaseHTTPError as e:
            raise AirflowException(
                f'There was an error reading the kubernetes API: {e}')

    @tenacity.retry(stop=tenacity.stop_after_attempt(3),
                    wait=tenacity.wait_exponential(),
                    reraise=True)
    def read_pod(self, pod: V1Pod) -> V1Pod:
        """Read POD information"""
        try:
            return self._client.read_namespaced_pod(pod.metadata.name,
                                                    pod.metadata.namespace)
        except BaseHTTPError as e:
            raise AirflowException(
                f'There was an error reading the kubernetes API: {e}')

    def extract_xcom(self, pod: V1Pod) -> str:
        """Retrieves XCom value and kills xcom sidecar container"""
        with closing(
                kubernetes_stream(
                    self._client.connect_get_namespaced_pod_exec,
                    pod.metadata.name,
                    pod.metadata.namespace,
                    container=PodDefaults.SIDECAR_CONTAINER_NAME,
                    command=['/bin/sh'],
                    stdin=True,
                    stdout=True,
                    stderr=True,
                    tty=False,
                    _preload_content=False,
                )) as resp:
            result = self._exec_pod_command(
                resp, f'cat {PodDefaults.XCOM_MOUNT_PATH}/return.json')
            self._exec_pod_command(resp, 'kill -s SIGINT 1')
        if result is None:
            raise AirflowException(
                f'Failed to extract xcom from pod: {pod.metadata.name}')
        return result

    def _exec_pod_command(self, resp, command: str) -> Optional[str]:
        if resp.is_open():
            self.log.info('Running command... %s\n', command)
            resp.write_stdin(command + '\n')
            while resp.is_open():
                resp.update(timeout=1)
                if resp.peek_stdout():
                    return resp.read_stdout()
                if resp.peek_stderr():
                    self.log.info("stderr from command: %s",
                                  resp.read_stderr())
                    break
        return None
Example #44
0
class MergifyContext(object):
    client = attr.ib()
    pull = attr.ib()
    _consolidated_data = attr.ib(init=False, default=None)

    @property
    def log(self):
        return daiquiri.getLogger(
            __name__,
            gh_owner=self.pull["user"]["login"]
            if "user" in self.pull else "<unknown-yet>",
            gh_repo=(self.pull["base"]["repo"]["name"]
                     if "base" in self.pull else "<unknown-yet>"),
            gh_private=(self.pull["base"]["repo"]["private"]
                        if "base" in self.pull else "<unknown-yet>"),
            gh_branch=self.pull["base"]["ref"]
            if "base" in self.pull else "<unknown-yet>",
            gh_pull=self.pull["number"],
            gh_pull_url=self.pull.get("html_url", "<unknown-yet>"),
            gh_pull_state=("merged" if self.pull.get("merged") else
                           (self.pull.get("mergeable_state", "unknown")
                            or "none")),
        )

    def __attrs_post_init__(self):
        self._ensure_complete()

    def has_write_permissions(self, login):
        # TODO(sileht): We should cache that, this is also used in command runner
        return self.client.item(
            f"collaborators/{login}/permission")["permission"] in [
                "admin",
                "write",
            ]

    def _get_valid_users(self):
        bots = list(
            set([
                r["user"]["login"] for r in self.reviews
                if r["user"]["type"] == "Bot"
            ]))
        collabs = set([
            r["user"]["login"] for r in self.reviews
            if r["user"]["type"] != "Bot"
        ])
        valid_collabs = [
            login for login in collabs if self.has_write_permissions(login)
        ]
        return bots + valid_collabs

    @functools_bp.cached_property
    def consolidated_reviews(self):
        # Ignore reviews that are not from someone with admin/write permissions
        # And only keep the last review for each user.
        comments = dict()
        approvals = dict()
        valid_users = self._get_valid_users()
        for review in self.reviews:
            if review["user"]["login"] not in valid_users:
                continue
            # Only keep latest review of an user
            if review["state"] == "COMMENTED":
                comments[review["user"]["login"]] = review
            else:
                approvals[review["user"]["login"]] = review
        return list(comments.values()), list(approvals.values())

    def to_dict(self):
        if self._consolidated_data is None:
            self._consolidated_data = self._get_consolidated_data()
        return self._consolidated_data

    def _get_consolidated_data(self):
        comments, approvals = self.consolidated_reviews
        return {
            # Can be used by rules too
            "assignee": [a["login"] for a in self.pull["assignees"]],
            # NOTE(sileht): We put an empty label to allow people to match
            # no label set
            "label": [l["name"] for l in self.pull["labels"]],
            "review-requested":
            ([u["login"] for u in self.pull["requested_reviewers"]] +
             ["@" + t["slug"] for t in self.pull["requested_teams"]]),
            "author":
            self.pull["user"]["login"],
            "merged-by": (self.pull["merged_by"]["login"]
                          if self.pull["merged_by"] else ""),
            "merged":
            self.pull["merged"],
            "closed":
            self.pull["state"] == "closed",
            "milestone": (self.pull["milestone"]["title"]
                          if self.pull["milestone"] else ""),
            "conflict":
            self.pull["mergeable_state"] == "dirty",
            "base":
            self.pull["base"]["ref"],
            "head":
            self.pull["head"]["ref"],
            "locked":
            self.pull["locked"],
            "title":
            self.pull["title"],
            "body":
            self.pull["body"],
            "files": [f["filename"] for f in self.files],
            "approved-reviews-by": [
                r["user"]["login"] for r in approvals
                if r["state"] == "APPROVED"
            ],
            "dismissed-reviews-by": [
                r["user"]["login"] for r in approvals
                if r["state"] == "DISMISSED"
            ],
            "changes-requested-reviews-by": [
                r["user"]["login"] for r in approvals
                if r["state"] == "CHANGES_REQUESTED"
            ],
            "commented-reviews-by": [
                r["user"]["login"] for r in comments
                if r["state"] == "COMMENTED"
            ],
            # NOTE(jd) The Check API set conclusion to None for pending.
            # NOTE(sileht): "pending" statuses are not really trackable, we
            # voluntary drop this event because CIs just sent they status every
            # minutes until the CI pass (at least Travis and Circle CI does
            # that). This was causing a big load on Mergify for nothing useful
            # tracked, and on big projects it can reach the rate limit very
            # quickly.
            "status-success": [
                name for name, state in self.checks.items()
                if state == "success"
            ],
            "status-failure": [
                name for name, state in self.checks.items()
                if state == "failure"
            ],
            "status-neutral": [
                name for name, state in self.checks.items()
                if state == "neutral"
            ],
            # NOTE(sileht): Not handled for now
            # cancelled, timed_out, or action_required
        }

    @functools_bp.cached_property
    def checks(self):
        # NOTE(sileht): conclusion can be one of success, failure, neutral,
        # cancelled, timed_out, or action_required, and  None for "pending"
        checks = dict(
            (c["name"], c["conclusion"]) for c in check_api.get_checks(self))
        # NOTE(sileht): state can be one of error, failure, pending,
        # or success.
        checks.update((s["context"], s["state"]) for s in self.client.items(
            f"commits/{self.pull['head']['sha']}/status",
            list_items="statuses"))
        return checks

    def _resolve_login(self, name):
        if not name:
            return []
        elif not isinstance(name, str):
            return [name]
        elif name[0] != "@":
            return [name]

        if "/" in name:
            organization, _, team_slug = name.partition("/")
            if not team_slug or "/" in team_slug:
                # Not a team slug
                return [name]
            organization = organization[1:]
        else:
            organization = self.pull["base"]["repo"]["owner"]["login"]
            team_slug = name[1:]

        try:
            return [
                member["login"] for member in self.client.items(
                    f"/orgs/{organization}/teams/{team_slug}/members")
            ]
        except httpx.HTTPClientSideError as e:
            self.log.warning(
                "fail to get the organization, team or members",
                team=name,
                status=e.status_code,
                detail=e.message,
            )
        return [name]

    def resolve_teams(self, values):
        if not values:
            return []
        if not isinstance(values, (list, tuple)):
            values = [values]
        values = list(
            itertools.chain.from_iterable((map(self._resolve_login, values))))
        return values

    UNUSABLE_STATES = ["unknown", None]

    # NOTE(sileht): quickly retry, if we don't get the status on time
    # the exception is recatch in worker.py, so celery will retry it later
    @tenacity.retry(
        wait=tenacity.wait_exponential(multiplier=0.2),
        stop=tenacity.stop_after_attempt(5),
        retry=tenacity.retry_if_exception_type(
            exceptions.MergeableStateUnknown),
        reraise=True,
    )
    def _ensure_complete(self):
        if not (self._is_data_complete()
                and self._is_background_github_processing_completed()):
            self.pull = self.client.item(f"pulls/{self.pull['number']}")

        if not self._is_data_complete():
            self.log.error(
                "/pulls/%s has returned an incomplete payload...",
                self.pull["number"],
                data=self.pull,
            )

        if self._is_background_github_processing_completed():
            return

        raise exceptions.MergeableStateUnknown(self)

    def _is_data_complete(self):
        # NOTE(sileht): If pull request come from /pulls listing or check-runs sometimes,
        # they are incomplete, This ensure we have the complete view
        fields_to_control = (
            "state",
            "mergeable_state",
            "merged_by",
            "merged",
            "merged_at",
        )
        for field in fields_to_control:
            if field not in self.pull:
                return False
        return True

    def _is_background_github_processing_completed(self):
        return (self.pull["state"] == "closed"
                or self.pull["mergeable_state"] not in self.UNUSABLE_STATES)

    def update(self):
        # TODO(sileht): Remove me,
        # Don't use it, because consolidated data are not updated after that.
        # Only used by merge action for posting an update report after rebase.
        self.pull = self.client.item(f"pulls/{self.pull['number']}")

    @functools_bp.cached_property
    def is_behind(self):
        branch_name_escaped = parse.quote(self.pull["base"]["ref"], safe="")
        branch = self.client.item(f"branches/{branch_name_escaped}")
        for commit in self.commits:
            for parent in commit["parents"]:
                if parent["sha"] == branch["commit"]["sha"]:
                    return False
        return True

    def get_merge_commit_message(self):
        if not self.pull["body"]:
            return

        found = False
        message_lines = []

        for line in self.pull["body"].split("\n"):
            if MARKDOWN_COMMIT_MESSAGE_RE.match(line):
                found = True
            elif found and MARKDOWN_TITLE_RE.match(line):
                break
            elif found:
                message_lines.append(line)

        if found and message_lines:
            return {
                "commit_title": message_lines[0],
                "commit_message": "\n".join(message_lines[1:]).strip(),
            }

    def __str__(self):
        return "%(login)s/%(repo)s/pull/%(number)d@%(branch)s " "s:%(pr_state)s" % {
            "login":
            self.pull["base"]["user"]["login"],
            "repo":
            self.pull["base"]["repo"]["name"],
            "number":
            self.pull["number"],
            "branch":
            self.pull["base"]["ref"],
            "pr_state": ("merged" if self.pull["merged"] else
                         (self.pull["mergeable_state"] or "none")),
        }

    @functools_bp.cached_property
    def reviews(self):
        return list(self.client.items(f"pulls/{self.pull['number']}/reviews"))

    @functools_bp.cached_property
    def commits(self):
        return list(self.client.items(f"pulls/{self.pull['number']}/commits"))

    @functools_bp.cached_property
    def files(self):
        return list(self.client.items(f"pulls/{self.pull['number']}/files"))

    @property
    def pull_from_fork(self):
        return self.pull["head"]["repo"]["id"] != self.pull["base"]["repo"][
            "id"]

    @property
    def pull_base_is_modifiable(self):
        return self.pull["maintainer_can_modify"] or not self.pull_from_fork
Example #45
0
class PodLauncher(LoggingMixin):
    """Deprecated class for launching pods. please use
    airflow.providers.cncf.kubernetes.utils.pod_launcher.PodLauncher instead
    """
    def __init__(
        self,
        kube_client: client.CoreV1Api = None,
        in_cluster: bool = True,
        cluster_context: Optional[str] = None,
        extract_xcom: bool = False,
    ):
        """
        Deprecated class for launching pods. please use
        airflow.providers.cncf.kubernetes.utils.pod_launcher.PodLauncher instead
        Creates the launcher.

        :param kube_client: kubernetes client
        :param in_cluster: whether we are in cluster
        :param cluster_context: context of the cluster
        :param extract_xcom: whether we should extract xcom
        """
        super().__init__()
        self._client = kube_client or get_kube_client(
            in_cluster=in_cluster, cluster_context=cluster_context)
        self._watch = watch.Watch()
        self.extract_xcom = extract_xcom

    def run_pod_async(self, pod: V1Pod, **kwargs):
        """Runs POD asynchronously"""
        pod_mutation_hook(pod)

        sanitized_pod = self._client.api_client.sanitize_for_serialization(pod)
        json_pod = json.dumps(sanitized_pod, indent=2)

        self.log.debug('Pod Creation Request: \n%s', json_pod)
        try:
            resp = self._client.create_namespaced_pod(
                body=sanitized_pod, namespace=pod.metadata.namespace, **kwargs)
            self.log.debug('Pod Creation Response: %s', resp)
        except Exception as e:
            self.log.exception(
                'Exception when attempting to create Namespaced Pod: %s',
                json_pod)
            raise e
        return resp

    def delete_pod(self, pod: V1Pod):
        """Deletes POD"""
        try:
            self._client.delete_namespaced_pod(pod.metadata.name,
                                               pod.metadata.namespace,
                                               body=client.V1DeleteOptions())
        except ApiException as e:
            # If the pod is already deleted
            if e.status != 404:
                raise

    def start_pod(self, pod: V1Pod, startup_timeout: int = 120):
        """
        Launches the pod synchronously and waits for completion.

        :param pod:
        :param startup_timeout: Timeout for startup of the pod (if pod is pending for too long, fails task)
        :return:
        """
        resp = self.run_pod_async(pod)
        curr_time = dt.now()
        if resp.status.start_time is None:
            while self.pod_not_started(pod):
                self.log.warning("Pod not yet started: %s", pod.metadata.name)
                delta = dt.now() - curr_time
                if delta.total_seconds() >= startup_timeout:
                    raise AirflowException("Pod took too long to start")
                time.sleep(1)

    def monitor_pod(self, pod: V1Pod,
                    get_logs: bool) -> Tuple[State, Optional[str]]:
        """
        Monitors a pod and returns the final state

        :param pod: pod spec that will be monitored
        :type pod : V1Pod
        :param get_logs: whether to read the logs locally
        :return:  Tuple[State, Optional[str]]
        """
        if get_logs:
            read_logs_since_sec = None
            last_log_time = None
            while True:
                logs = self.read_pod_logs(pod,
                                          timestamps=True,
                                          since_seconds=read_logs_since_sec)
                for line in logs:
                    timestamp, message = self.parse_log_line(
                        line.decode('utf-8'))
                    last_log_time = pendulum.parse(timestamp)
                    self.log.info(message)
                time.sleep(1)

                if not self.base_container_is_running(pod):
                    break

                self.log.warning('Pod %s log read interrupted',
                                 pod.metadata.name)
                if last_log_time:
                    delta = pendulum.now() - last_log_time
                    # Prefer logs duplication rather than loss
                    read_logs_since_sec = math.ceil(delta.total_seconds())
        result = None
        if self.extract_xcom:
            while self.base_container_is_running(pod):
                self.log.info('Container %s has state %s', pod.metadata.name,
                              State.RUNNING)
                time.sleep(2)
            result = self._extract_xcom(pod)
            self.log.info(result)
            result = json.loads(result)
        while self.pod_is_running(pod):
            self.log.info('Pod %s has state %s', pod.metadata.name,
                          State.RUNNING)
            time.sleep(2)
        return self._task_status(self.read_pod(pod)), result

    def parse_log_line(self, line: str) -> Tuple[str, str]:
        """
        Parse K8s log line and returns the final state

        :param line: k8s log line
        :type line: str
        :return: timestamp and log message
        :rtype: Tuple[str, str]
        """
        split_at = line.find(' ')
        if split_at == -1:
            raise Exception(
                f'Log not in "{{timestamp}} {{log}}" format. Got: {line}')
        timestamp = line[:split_at]
        message = line[split_at + 1:].rstrip()
        return timestamp, message

    def _task_status(self, event):
        self.log.info('Event: %s had an event of type %s', event.metadata.name,
                      event.status.phase)
        status = self.process_status(event.metadata.name, event.status.phase)
        return status

    def pod_not_started(self, pod: V1Pod):
        """Tests if pod has not started"""
        state = self._task_status(self.read_pod(pod))
        return state == State.QUEUED

    def pod_is_running(self, pod: V1Pod):
        """Tests if pod is running"""
        state = self._task_status(self.read_pod(pod))
        return state not in (State.SUCCESS, State.FAILED)

    def base_container_is_running(self, pod: V1Pod):
        """Tests if base container is running"""
        event = self.read_pod(pod)
        status = next(
            iter(
                filter(lambda s: s.name == 'base',
                       event.status.container_statuses)), None)
        if not status:
            return False
        return status.state.running is not None

    @tenacity.retry(stop=tenacity.stop_after_attempt(3),
                    wait=tenacity.wait_exponential(),
                    reraise=True)
    def read_pod_logs(
        self,
        pod: V1Pod,
        tail_lines: Optional[int] = None,
        timestamps: bool = False,
        since_seconds: Optional[int] = None,
    ):
        """Reads log from the POD"""
        additional_kwargs = {}
        if since_seconds:
            additional_kwargs['since_seconds'] = since_seconds

        if tail_lines:
            additional_kwargs['tail_lines'] = tail_lines

        try:
            return self._client.read_namespaced_pod_log(
                name=pod.metadata.name,
                namespace=pod.metadata.namespace,
                container='base',
                follow=True,
                timestamps=timestamps,
                _preload_content=False,
                **additional_kwargs,
            )
        except BaseHTTPError as e:
            raise AirflowException(
                f'There was an error reading the kubernetes API: {e}')

    @tenacity.retry(stop=tenacity.stop_after_attempt(3),
                    wait=tenacity.wait_exponential(),
                    reraise=True)
    def read_pod_events(self, pod):
        """Reads events from the POD"""
        try:
            return self._client.list_namespaced_event(
                namespace=pod.metadata.namespace,
                field_selector=f"involvedObject.name={pod.metadata.name}")
        except BaseHTTPError as e:
            raise AirflowException(
                f'There was an error reading the kubernetes API: {e}')

    @tenacity.retry(stop=tenacity.stop_after_attempt(3),
                    wait=tenacity.wait_exponential(),
                    reraise=True)
    def read_pod(self, pod: V1Pod):
        """Read POD information"""
        try:
            return self._client.read_namespaced_pod(pod.metadata.name,
                                                    pod.metadata.namespace)
        except BaseHTTPError as e:
            raise AirflowException(
                f'There was an error reading the kubernetes API: {e}')

    def _extract_xcom(self, pod: V1Pod):
        resp = kubernetes_stream(
            self._client.connect_get_namespaced_pod_exec,
            pod.metadata.name,
            pod.metadata.namespace,
            container=PodDefaults.SIDECAR_CONTAINER_NAME,
            command=['/bin/sh'],
            stdin=True,
            stdout=True,
            stderr=True,
            tty=False,
            _preload_content=False,
        )
        try:
            result = self._exec_pod_command(
                resp, f'cat {PodDefaults.XCOM_MOUNT_PATH}/return.json')
            self._exec_pod_command(resp, 'kill -s SIGINT 1')
        finally:
            resp.close()
        if result is None:
            raise AirflowException(
                f'Failed to extract xcom from pod: {pod.metadata.name}')
        return result

    def _exec_pod_command(self, resp, command):
        if resp.is_open():
            self.log.info('Running command... %s\n', command)
            resp.write_stdin(command + '\n')
            while resp.is_open():
                resp.update(timeout=1)
                if resp.peek_stdout():
                    return resp.read_stdout()
                if resp.peek_stderr():
                    self.log.info(resp.read_stderr())
                    break
        return None

    def process_status(self, job_id, status):
        """Process status information for the JOB"""
        status = status.lower()
        if status == PodStatus.PENDING:
            return State.QUEUED
        elif status == PodStatus.FAILED:
            self.log.error('Event with job id %s Failed', job_id)
            return State.FAILED
        elif status == PodStatus.SUCCEEDED:
            self.log.info('Event with job id %s Succeeded', job_id)
            return State.SUCCESS
        elif status == PodStatus.RUNNING:
            return State.RUNNING
        else:
            self.log.error('Event: Invalid state %s on job %s', status, job_id)
            return State.FAILED