def call_func(self, func, **kwargs): """General method for calling any Monasca API function.""" @tenacity.retry( wait=tenacity.wait_fixed(self._retry_interval), stop=tenacity.stop_after_attempt(self._max_retries), retry=(tenacity.retry_if_exception_type(MonascaServiceException) | tenacity.retry_if_exception_type(MonascaException))) def _inner(): try: return func(**kwargs) except (exc.http.InternalServerError, exc.http.ServiceUnavailable, exc.http.BadGateway, exc.connection.ConnectionError) as e: LOG.exception(e) msg = '%s: %s' % (e.__class__.__name__, e) raise MonascaServiceException(msg) except exc.http.HttpError as e: LOG.exception(e) msg = '%s: %s' % (e.__class__.__name__, e) status_code = e.http_status if not isinstance(status_code, int): status_code = 500 if 400 <= status_code < 500: raise MonascaInvalidParametersException(msg) else: raise MonascaException(msg) except Exception as e: LOG.exception(e) msg = '%s: %s' % (e.__class__.__name__, e) raise MonascaException(msg) return _inner()
def send_request(self, socket, request): @tenacity.retry(retry=tenacity.retry_if_exception_type(zmq.Again), stop=tenacity.stop_after_delay( self.conf.rpc_response_timeout)) def send_retrying(): self._do_send(socket, request) return send_retrying()
def upgrade(): conf = cfg.ConfigOpts() conf.register_cli_opts([ cfg.BoolOpt('skip-gnocchi-resource-types', help='Skip gnocchi resource-types upgrade.', default=False), cfg.IntOpt('retry', min=0, help='Number of times to retry on failure. ' 'Default is to retry forever.'), ]) service.prepare_service(conf=conf) if conf.skip_gnocchi_resource_types: LOG.info("Skipping Gnocchi resource types upgrade") else: LOG.debug("Upgrading Gnocchi resource types") from ceilometer import gnocchi_client from gnocchiclient import exceptions if conf.retry is None: stop = tenacity.stop_never else: stop = tenacity.stop_after_attempt(conf.retry) tenacity.Retrying( stop=stop, retry=tenacity.retry_if_exception_type(( exceptions.ConnectionFailure, exceptions.UnknownConnectionError, exceptions.ConnectionTimeout, exceptions.SSLError, )) )(gnocchi_client.upgrade_resource_types, conf)
def retry_upon_exception(exc, delay, max_delay, max_attempts): return tenacity.retry(reraise=True, retry=tenacity.retry_if_exception_type(exc), wait=tenacity.wait_exponential( multiplier=delay, max=max_delay), stop=tenacity.stop_after_attempt(max_attempts), before=_log_before_retry, after=_log_after_retry)
def join_group(self, group_id): if (not self._coordinator or not self._coordinator.is_started or not group_id): return @tenacity.retry( wait=tenacity.wait_exponential( multiplier=self.conf.coordination.retry_backoff, max=self.conf.coordination.max_retry_interval), retry=tenacity.retry_if_exception_type( ErrorJoiningPartitioningGroup)) def _inner(): try: join_req = self._coordinator.join_group(group_id) join_req.get() LOG.info(_LI('Joined partitioning group %s'), group_id) except tooz.coordination.MemberAlreadyExist: return except tooz.coordination.GroupNotCreated: create_grp_req = self._coordinator.create_group(group_id) try: create_grp_req.get() except tooz.coordination.GroupAlreadyExist: pass raise ErrorJoiningPartitioningGroup() except tooz.coordination.ToozError: LOG.exception(_LE('Error joining partitioning group %s,' ' re-trying'), group_id) raise ErrorJoiningPartitioningGroup() self._groups.add(group_id) return _inner()
def retry_on_db_error(func, retry=None): """Decorates the given function so that it retries on DB errors. Note that the decorator retries the function/method only on some of the DB errors that are considered to be worth retrying, like deadlocks and disconnections. :param func: Function to decorate. :param retry: a Retrying object :return: Decorated function. """ if not retry: retry = tenacity.Retrying( retry=tenacity.retry_if_exception_type(_RETRY_ERRORS), stop=tenacity.stop_after_attempt(50), wait=tenacity.wait_incrementing(start=0, increment=0.1, max=2) ) # The `assigned` arg should be empty as some of the default values are not # supported by simply initialized MagicMocks. The consequence may # be that the representation will contain the wrapper and not the # wrapped function. @functools.wraps(func, assigned=[]) def decorate(*args, **kw): # Retrying library decorator might potentially run a decorated # function within a new thread so it's safer not to apply the # decorator directly to a target method/function because we can # lose an authentication context. # The solution is to create one more function and explicitly set # auth context before calling it (potentially in a new thread). auth_ctx = context.ctx() if context.has_ctx() else None return retry.call(_with_auth_context, auth_ctx, func, *args, **kw) return decorate
def retry_random_upon_exception(exc, delay=0.5, max_delay=5, max_attempts=DEFAULT_MAX_ATTEMPTS): return tenacity.retry(reraise=True, retry=tenacity.retry_if_exception_type(exc), wait=tenacity.wait_random_exponential( multiplier=delay, max=max_delay), stop=tenacity.stop_after_attempt(max_attempts), before=_log_before_retry, after=_log_after_retry)
def _safe_mongo_call(max_retries, retry_interval): return tenacity.retry( retry=tenacity.retry_if_exception_type( pymongo.errors.AutoReconnect), wait=tenacity.wait_fixed(retry_interval), stop=(tenacity.stop_after_attempt(max_retries) if max_retries >= 0 else tenacity.stop_never) )
def retry_on_conflict(func): wrapper = tenacity.retry( stop=tenacity.stop_after_attempt(11), wait=tenacity.wait_random(max=0.002), retry=tenacity.retry_if_exception_type(exception.ConcurrentTransaction), reraise=True, ) return wrapper(func)
def send_request(self, socket, request): if hasattr(request, 'timeout'): _stop = tenacity.stop_after_delay(request.timeout) elif request.retry is not None and request.retry > 0: # no rpc_response_timeout option if notification _stop = tenacity.stop_after_attempt(request.retry) else: # well, now what? _stop = tenacity.stop_after_delay(60) @tenacity.retry(retry=tenacity.retry_if_exception_type(zmq.Again), stop=_stop) def send_retrying(): if request.msg_type in zmq_names.MULTISEND_TYPES: for _ in range(socket.connections_count()): self.sender.send(socket, request) else: self.sender.send(socket, request) return send_retrying()
def assert_wait(func, exc_type=AssertionError): """Waits for function to succeed (not raise `exc_type`).""" return tenacity.Retrying( stop=tenacity.stop_after_delay(constants.ux.MAX_USER_WAIT_SECONDS), retry=tenacity.retry_if_exception_type((exc_type, tenacity.TryAgain)))( func)
"w", compression=zipfile.ZIP_DEFLATED) async def get_nintendont_releases(session: aiohttp.ClientSession): async with session.get(_NINTENDONT_RELEASES_URL) as response: try: response.raise_for_status() return await response.json() except aiohttp.ClientResponseError as e: raise RuntimeError("Unable to get Nintendont releases") from e @tenacity.retry( stop=tenacity.stop_after_attempt(5), retry=tenacity.retry_if_exception_type(aiohttp.ClientConnectorError), wait=tenacity.wait_exponential(multiplier=1, min=4, max=30), ) async def download_nintendont(): headers = None if "GITHUB_TOKEN" in os.environ: headers = {"Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}"} async with aiohttp.ClientSession(headers=headers) as session: print("Fetching list of Nintendont releases.") releases = await get_nintendont_releases(session) latest_release = releases[0] download_urls = [ asset["browser_download_url"] for asset in latest_release["assets"] if asset["name"] == "boot.dol"
plural=plural, ) except ApiException as err: logging.error( "Failed to create %s %r: %s", custom_object.body["kind"], custom_object.name, err.reason, ) raise @retry( stop=stop_after_delay(KUBE_API_DELETE_TIMEOUT), wait=wait_fixed(KUBE_API_WAIT), retry=retry_if_exception_type(ResourceStillThereError), ) def await_no_resources_found(list_resources: Callable, **kwargs): try: found = list_resources(**kwargs) except ApiException as err: if err.status == STATUS_NOT_FOUND: return raise if hasattr(found, "items"): found = found.items if found: raise ResourceStillThereError( f"Resource(s): {found} still found; retrying.")
else: expect = 'Projects - OpenStack Dashboard' if expect not in response.text: msg = 'FAILURE code={} text="{}"'.format(response, response.text) logging.info("Yeah, wen't wrong: {}".format(msg)) raise FailedAuth(msg) logging.info("Logged into okay") return client, response # NOTE(ajkavanagh): it seems that apache2 doesn't start quickly enough # for the test, and so it gets reset errors; repeat until either that # stops or there is a failure @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), retry=tenacity.retry_if_exception_type( http.client.RemoteDisconnected), reraise=True) def _do_request(request, cafile=None): """Open a webpage via urlopen. :param request: A urllib request object. :type request: object :returns: HTTPResponse object :rtype: object :raises: URLError on protocol errors """ return urllib.request.urlopen(request, cafile=cafile) class OpenStackDashboardBase(): """Mixin for interacting with Horizon."""
def cleanup(bucket_cleanup): """Use this fixture to delete all unit testing resources regardless of of the failure or success of the test""" yield None iam = ck.aws.clients['iam'] ec2 = ck.aws.clients['ec2'] batch = ck.aws.clients['batch'] ecs = ck.aws.clients['ecs'] config_file = ck.config.get_config_file() section_suffix = ck.get_profile() + ' ' + ck.get_region() jq_section_name = 'job-queues ' + section_suffix ce_section_name = 'compute-environments ' + section_suffix jd_section_name = 'job-definitions ' + section_suffix roles_section_name = 'roles ' + ck.get_profile() + ' global' vpc_section_name = 'vpc ' + section_suffix sg_section_name = 'security-groups ' + section_suffix retry = tenacity.Retrying(wait=tenacity.wait_exponential(max=16), stop=tenacity.stop_after_delay(120), retry=tenacity.retry_if_exception_type( batch.exceptions.ClientException)) # Clean up job queues from AWS # ---------------------------- # Find all unit testing job queues response = batch.describe_job_queues() job_queues = [{ 'name': d['jobQueueName'], 'arn': d['jobQueueArn'], 'state': d['state'], 'status': d['status'] } for d in response.get('jobQueues')] while response.get('nextToken'): response = batch.describe_job_queues( nextToken=response.get('nextToken')) job_queues = job_queues + [{ 'name': d['jobQueueName'], 'arn': d['jobQueueArn'], 'state': d['state'], 'status': d['status'] } for d in response.get('jobQueues')] unit_test_JQs = list( filter(lambda d: UNIT_TEST_PREFIX in d['name'], job_queues)) enabled = list(filter(lambda d: d['state'] == 'ENABLED', unit_test_JQs)) for jq in enabled: ck.aws.wait_for_job_queue(name=jq['name'], max_wait_time=180) retry.call(batch.update_job_queue, jobQueue=jq['arn'], state='DISABLED') config = configparser.ConfigParser() with ck.config.rlock: config.read(config_file) requires_deletion = list( filter(lambda d: d['status'] not in ['DELETED', 'DELETING'], unit_test_JQs)) for jq in requires_deletion: ck.aws.wait_for_job_queue(name=jq['name'], max_wait_time=180) # Finally, delete the job queue retry.call(batch.delete_job_queue, jobQueue=jq['arn']) # Clean up config file try: config.remove_option(jq_section_name, jq['name']) except configparser.NoSectionError: pass with open(config_file, 'w') as f: config.write(f) # Clean up compute environments from AWS # -------------------------------------- # Find all unit testing compute environments response = batch.describe_compute_environments() comp_envs = [{ 'name': d['computeEnvironmentName'], 'arn': d['computeEnvironmentArn'], 'state': d['state'], 'status': d['status'] } for d in response.get('computeEnvironments')] while response.get('nextToken'): response = batch.describe_compute_environments( nextToken=response.get('nextToken')) comp_envs = comp_envs + [{ 'name': d['computeEnvironmentName'], 'arn': d['computeEnvironmentArn'], 'state': d['state'], 'status': d['status'] } for d in response.get('computeEnvironments')] unit_test_CEs = list( filter(lambda d: UNIT_TEST_PREFIX in d['name'], comp_envs)) enabled = list(filter(lambda d: d['state'] == 'ENABLED', unit_test_CEs)) for ce in enabled: ck.aws.wait_for_compute_environment(arn=ce['arn'], name=ce['name'], log=False) # Set the compute environment state to 'DISABLED' retry.call(batch.update_compute_environment, computeEnvironment=ce['arn'], state='DISABLED') config = configparser.ConfigParser() with ck.config.rlock: config.read(config_file) for ce in unit_test_CEs: # Then disassociate from any job queues response = batch.describe_job_queues() associated_queues = list( filter( lambda q: ce['arn'] in [ c['computeEnvironment'] for c in q['computeEnvironmentOrder'] ], response.get('jobQueues'))) for queue in associated_queues: arn = queue['jobQueueArn'] name = queue['jobQueueName'] # Disable submissions to the queue if queue['state'] == 'ENABLED': ck.aws.wait_for_job_queue(name=name, log=True, max_wait_time=180) retry.call(batch.update_job_queue, jobQueue=arn, state='DISABLED') # Delete the job queue if queue['status'] not in ['DELETED', 'DELETING']: ck.aws.wait_for_job_queue(name=name, log=True, max_wait_time=180) retry.call(batch.delete_job_queue, jobQueue=arn) # Clean up config file try: config.remove_option(jq_section_name, name) except configparser.NoSectionError: pass requires_deletion = list( filter(lambda d: d['status'] not in ['DELETED', 'DELETING'], unit_test_CEs)) for ce in requires_deletion: # Now get the associated ECS cluster response = batch.describe_compute_environments( computeEnvironments=[ce['arn']]) cluster_arn = response.get('computeEnvironments')[0]['ecsClusterArn'] # Get container instances response = ecs.list_container_instances(cluster=cluster_arn, ) instances = response.get('containerInstanceArns') for i in instances: ecs.deregister_container_instance(cluster=cluster_arn, containerInstance=i, force=True) retry_if_exception = tenacity.Retrying( wait=tenacity.wait_exponential(max=16), stop=tenacity.stop_after_delay(120), retry=tenacity.retry_if_exception_type()) retry_if_exception.call(ecs.delete_cluster, cluster=cluster_arn) ck.aws.wait_for_compute_environment(arn=ce['arn'], name=ce['name'], log=False) retry.call(batch.delete_compute_environment, computeEnvironment=ce['arn']) # Clean up config file try: config.remove_option(ce_section_name, ce['name']) except configparser.NoSectionError: pass with open(config_file, 'w') as f: config.write(f) # Clean up job definitions from AWS # --------------------------------- # Find all unit testing job definitions response = batch.describe_job_definitions(status='ACTIVE') jds = [{ 'name': d['jobDefinitionName'], 'arn': d['jobDefinitionArn'] } for d in response.get('jobDefinitions')] unit_test_jds = list(filter(lambda d: UNIT_TEST_PREFIX in d['name'], jds)) while response.get('nextToken'): response = batch.describe_job_definitions( status='ACTIVE', nextToken=response.get('nextToken')) jds = [{ 'name': d['jobDefinitionName'], 'arn': d['jobDefinitionArn'] } for d in response.get('jobDefinitions')] unit_test_jds = unit_test_jds + list( filter(lambda d: UNIT_TEST_PREFIX in d['name'], jds)) config = configparser.ConfigParser() with ck.config.rlock: config.read(config_file) for jd in unit_test_jds: # Deregister the job definition retry.call(batch.deregister_job_definition, jobDefinition=jd['arn']) # Clean up config file try: config.remove_option(jd_section_name, jd['name']) except configparser.NoSectionError: pass with open(config_file, 'w') as f: config.write(f) # Clean up security_groups from AWS # --------------------------------- # Find all unit test security groups ec2_retry = tenacity.Retrying(wait=tenacity.wait_exponential(max=16), stop=tenacity.stop_after_delay(60), retry=tenacity.retry_if_exception_type( ec2.exceptions.ClientError)) response = ec2.describe_security_groups() sgs = [{ 'name': d['GroupName'], 'id': d['GroupId'] } for d in response.get('SecurityGroups')] unit_test_sgs = filter(lambda d: UNIT_TEST_PREFIX in d['name'], sgs) config = configparser.ConfigParser() with ck.config.rlock: config.read(config_file) for sg in unit_test_sgs: # Delete role ec2_retry.call(ec2.delete_security_group, GroupId=sg['id']) # Clean up config file try: config.remove_option(sg_section_name, sg['id']) except configparser.NoSectionError: pass with open(config_file, 'w') as f: config.write(f) # Clean up VPCs from AWS # ---------------------- config = configparser.ConfigParser() with ck.config.rlock: config.read(config_file) # Find all VPCs with a Name tag key response = ec2.describe_vpcs(Filters=[{ 'Name': 'tag-key', 'Values': ['Name'] }]) for vpc in response.get('Vpcs'): # Test if the unit-test prefix is in the name if UNIT_TEST_PREFIX in [ d for d in vpc['Tags'] if d['Key'] == 'Name' ][0]['Value']: # Retrieve and delete subnets response = ec2.describe_subnets( Filters=[{ 'Name': 'vpc-id', 'Values': [vpc['VpcId']] }]) subnets = [d['SubnetId'] for d in response.get('Subnets')] for subnet_id in subnets: ec2_retry.call(ec2.delete_subnet, SubnetId=subnet_id) response = ec2.describe_network_acls( Filters=[{ 'Name': 'vpc-id', 'Values': [vpc['VpcId']] }, { 'Name': 'default', 'Values': ['false'] }]) network_acl_ids = [ n['NetworkAclId'] for n in response.get('NetworkAcls') ] # Delete the network ACL for net_id in network_acl_ids: ec2_retry.call(ec2.delete_network_acl, NetworkAclId=net_id) response = ec2.describe_route_tables( Filters=[{ 'Name': 'vpc-id', 'Values': [vpc['VpcId']] }, { 'Name': 'association.main', 'Values': ['false'] }]) route_table_ids = [ rt['RouteTableId'] for rt in response.get('RouteTables') ] # Delete the route table for rt_id in route_table_ids: ec2_retry.call(ec2.delete_route_table, RouteTableId=rt_id) # Detach and delete the internet gateway response = ec2.describe_internet_gateways( Filters=[{ 'Name': 'attachment.vpc-id', 'Values': [vpc['VpcId']] }]) gateway_ids = [ g['InternetGatewayId'] for g in response.get('InternetGateways') ] for gid in gateway_ids: ec2_retry.call(ec2.detach_internet_gateway, InternetGatewayId=gid, VpcId=vpc['VpcId']) ec2_retry.call(ec2.delete_internet_gateway, InternetGatewayId=gid) # delete the VPC ec2_retry.call(ec2.delete_vpc, VpcId=vpc['VpcId']) # Clean up config file try: config.remove_option(vpc_section_name, vpc['VpcId']) except configparser.NoSectionError: pass with open(config_file, 'w') as f: config.write(f) # Clean up roles from AWS # ----------------------- # Find all unit test roles response = iam.list_roles() role_names = [d['RoleName'] for d in response.get('Roles')] unit_test_roles = filter(lambda n: UNIT_TEST_PREFIX in n, role_names) for role_name in unit_test_roles: # Remove instance profiles response = iam.list_instance_profiles_for_role(RoleName=role_name) for ip in response.get('InstanceProfiles'): iam.remove_role_from_instance_profile( InstanceProfileName=ip['InstanceProfileName'], RoleName=role_name) iam.delete_instance_profile( InstanceProfileName=ip['InstanceProfileName']) # Detach policies from role response = iam.list_attached_role_policies(RoleName=role_name) for policy in response.get('AttachedPolicies'): iam.detach_role_policy(RoleName=role_name, PolicyArn=policy['PolicyArn']) # Delete role iam.delete_role(RoleName=role_name) # Clean up config file config = configparser.ConfigParser() with ck.config.rlock: config.read(config_file) for role_name in config.options(roles_section_name): if UNIT_TEST_PREFIX in role_name: config.remove_option(roles_section_name, role_name) with open(config_file, 'w') as f: config.write(f)
class LBAASv2Test(test_utils.OpenStackBaseTest): """LBaaSv2 service tests.""" @classmethod def setUpClass(cls): """Run class setup for running LBaaSv2 service tests.""" super(LBAASv2Test, cls).setUpClass() cls.keystone_client = openstack_utils.get_keystone_session_client( cls.keystone_session) cls.neutron_client = openstack_utils.get_neutron_session_client( cls.keystone_session) cls.octavia_client = openstack_utils.get_octavia_session_client( cls.keystone_session) cls.RESOURCE_PREFIX = 'zaza-octavia' # NOTE(fnordahl): in the event of a test failure we do not want to run # tear down code as it will make debugging a problem virtually # impossible. To alleviate each test method will set the # `run_tearDown` instance variable at the end which will let us run # tear down only when there were no failure. cls.run_tearDown = False # List of load balancers created by this test cls.loadbalancers = [] # List of floating IPs created by this test cls.fips = [] def resource_cleanup(self): """Remove resources created during test execution.""" for lb in self.loadbalancers: self.octavia_client.load_balancer_delete(lb['id'], cascade=True) try: self.wait_for_lb_resource( self.octavia_client.load_balancer_show, lb['id'], provisioning_status='DELETED') except osc_lib.exceptions.NotFound: pass for fip in self.fips: self.neutron_client.delete_floatingip(fip) # we run the parent resource_cleanup last as it will remove instances # referenced as members in the above cleaned up load balancers super(LBAASv2Test, self).resource_cleanup() @staticmethod @tenacity.retry(retry=tenacity.retry_if_exception_type(AssertionError), wait=tenacity.wait_fixed(1), reraise=True, stop=tenacity.stop_after_delay(900)) def wait_for_lb_resource(octavia_show_func, resource_id, provisioning_status=None, operating_status=None): """Wait for loadbalancer resource to reach expected status.""" provisioning_status = provisioning_status or 'ACTIVE' resp = octavia_show_func(resource_id) logging.info(resp['provisioning_status']) assert resp['provisioning_status'] == provisioning_status, ( 'load balancer resource has not reached ' 'expected provisioning status: {}'.format(resp)) if operating_status: logging.info(resp['operating_status']) assert resp['operating_status'] == operating_status, ( 'load balancer resource has not reached ' 'expected operating status: {}'.format(resp)) return resp @staticmethod def get_lb_providers(octavia_client): """Retrieve loadbalancer providers. :param octavia_client: Octavia client object :type octavia_client: OctaviaAPI :returns: Dictionary with provider information, name as keys :rtype: Dict[str,Dict[str,str]] """ providers = { provider['name']: provider for provider in octavia_client.provider_list().get( 'providers', []) if provider['name'] != 'octavia' # alias for `amphora`, skip } return providers def _create_lb_resources(self, octavia_client, provider, vip_subnet_id, member_subnet_id, payload_ips): # The `amphora` provider is required for load balancing based on # higher layer protocols if provider == 'amphora': protocol = 'HTTP' algorithm = 'ROUND_ROBIN' monitor = True else: protocol = 'TCP' algorithm = 'SOURCE_IP_PORT' monitor = False result = octavia_client.load_balancer_create( json={ 'loadbalancer': { 'description': 'Created by Zaza', 'admin_state_up': True, 'vip_subnet_id': vip_subnet_id, 'name': 'zaza-{}-0'.format(provider), 'provider': provider, } }) lb = result['loadbalancer'] lb_id = lb['id'] logging.info('Awaiting loadbalancer to reach provisioning_status ' '"ACTIVE"') resp = self.wait_for_lb_resource(octavia_client.load_balancer_show, lb_id) logging.info(resp) result = octavia_client.listener_create( json={ 'listener': { 'loadbalancer_id': lb_id, 'name': 'listener1', 'protocol': protocol, 'protocol_port': 80 }, }) listener_id = result['listener']['id'] logging.info('Awaiting listener to reach provisioning_status ' '"ACTIVE"') resp = self.wait_for_lb_resource(octavia_client.listener_show, listener_id) logging.info(resp) result = octavia_client.pool_create( json={ 'pool': { 'listener_id': listener_id, 'name': 'pool1', 'lb_algorithm': algorithm, 'protocol': protocol, }, }) pool_id = result['pool']['id'] logging.info('Awaiting pool to reach provisioning_status ' '"ACTIVE"') resp = self.wait_for_lb_resource(octavia_client.pool_show, pool_id) logging.info(resp) if monitor: result = octavia_client.health_monitor_create( json={ 'healthmonitor': { 'pool_id': pool_id, 'delay': 5, 'max_retries': 4, 'timeout': 10, 'type': 'HTTP', 'url_path': '/', }, }) healthmonitor_id = result['healthmonitor']['id'] logging.info('Awaiting healthmonitor to reach provisioning_status ' '"ACTIVE"') resp = self.wait_for_lb_resource( octavia_client.health_monitor_show, healthmonitor_id) logging.info(resp) for ip in payload_ips: result = octavia_client.member_create(pool_id=pool_id, json={ 'member': { 'subnet_id': member_subnet_id, 'address': ip, 'protocol_port': 80, }, }) member_id = result['member']['id'] logging.info('Awaiting member to reach provisioning_status ' '"ACTIVE"') resp = self.wait_for_lb_resource( lambda x: octavia_client.member_show(pool_id=pool_id, member_id=x), member_id, operating_status='ONLINE' if monitor else '') logging.info(resp) return lb @staticmethod @tenacity.retry(wait=tenacity.wait_fixed(1), reraise=True, stop=tenacity.stop_after_delay(900)) def _get_payload(ip): return subprocess.check_output( ['wget', '-O', '-', 'http://{}/'.format(ip)], universal_newlines=True) def test_create_loadbalancer(self): """Create load balancer.""" # Prepare payload instances # First we allow communication to port 80 by adding a security group # rule project_id = openstack_utils.get_project_id(self.keystone_client, 'admin', domain_name='admin_domain') openstack_utils.add_neutron_secgroup_rules(self.neutron_client, project_id, [{ 'protocol': 'tcp', 'port_range_min': '80', 'port_range_max': '80', 'direction': 'ingress' }]) # Then we request two Ubuntu instances with the Apache web server # installed instance_1, instance_2 = self.launch_guests( userdata='#cloud-config\npackages:\n - apache2\n') # Get IP of the prepared payload instances payload_ips = [] for server in (instance_1, instance_2): payload_ips.append(server.networks['private'][0]) self.assertTrue(len(payload_ips) > 0) resp = self.neutron_client.list_networks(name='private') subnet_id = resp['networks'][0]['subnets'][0] if openstack_utils.dvr_enabled(): resp = self.neutron_client.list_networks( name='private_lb_fip_network') vip_subnet_id = resp['networks'][0]['subnets'][0] else: vip_subnet_id = subnet_id for provider in self.get_lb_providers(self.octavia_client).keys(): logging.info( 'Creating loadbalancer with provider {}'.format(provider)) lb = self._create_lb_resources(self.octavia_client, provider, vip_subnet_id, subnet_id, payload_ips) self.loadbalancers.append(lb) lb_fp = openstack_utils.create_floating_ip( self.neutron_client, 'ext_net', port={'id': lb['vip_port_id']}) snippet = 'This is the default welcome page' assert snippet in self._get_payload(lb_fp['floating_ip_address']) logging.info('Found "{}" in page retrieved through load balancer ' ' (provider="{}") at "http://{}/"'.format( snippet, provider, lb_fp['floating_ip_address'])) # If we get here, it means the tests passed self.run_resource_cleanup = True
""" return hookenv.leader_get(CHARM_ACCESS_ROLE_ID) def get_client(url=None): """Provide a client for talking to the vault api :returns: vault client :rtype: hvac.Client """ return hvac.Client(url=url or get_api_url()) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), stop=tenacity.stop_after_attempt(8), retry=tenacity.retry_if_exception_type( hvac.exceptions.InternalServerError), reraise=True) def get_local_client(): """Provide a client for talking to the vault api :returns: vault client :rtype: hvac.Client """ client = get_client(url=VAULT_LOCALHOST_URL) app_role_id = get_local_charm_access_role_id() if not app_role_id: hookenv.log('Could not retrieve app_role_id', level=hookenv.DEBUG) raise VaultNotReady("Cannot initialise local client") client = hvac.Client(url=VAULT_LOCALHOST_URL) client.auth_approle(app_role_id) return client
class EntityManager(object): __metaclass__ = ABCMeta @abstractmethod def __init__(self, client=None, project_id=None): """Mother Class to Initiate Args: self : Authorized BigQuery API service instance. client (:obj:`int`, optional): project_id (str): Returns: """ self.__client = client self.__project_id = project_id self._time_out_seconds = 3600 @staticmethod def _execute_methods(request): """Method using to execute Build the calling API's Methods in Google Cloud Discovery Args: self : Authorized BigQuery API service instance. request: Returns: """ return request.execute() @tn.retry(wait=tn.wait_random_exponential(multiplier=1, max=60), retry=tn.retry_if_exception_type(), stop=tn.stop_after_attempt(5)) def _call_methods(self, http_verb, url, **kwargs): """Method using to Call API's Methods in Google Cloud Discovery Args: self : Authorized Cloud Storage API service instance. http_verb: url: **kwargs: Returns: """ return self.__client.api_request(http_verb, url, api_version="v2", **kwargs) def _check_results(self, job_id, job_type, dataset_id, table_id): """check_job_results : methods to check if execution jobs is good gone Args: self : Authorized BigQuery API service instance. job_id (str): job_type (object): Type of Job : Dataset Jobs, Table Jobs or General Jobs dataset_id (str): table_id (str): Returns: """ request = self.__client.jobs().get(projectId=self.__project_id, jobId=job_id) response = self._execute_methods(request) start_time = time.time() duration = 0 while response["status"]["state"] != "DONE" and ( duration <= self._time_out_seconds): time.sleep(1) response = self.__client.jobs().get(projectId=self.__project_id, jobId=job_id) response = self._execute_methods(response) duration = time.time() - start_time if duration > self._time_out_seconds and response["status"][ "state"] != "DONE": raise Exception("Time out %dsec exceeded for this current job" % self._time_out_seconds) try: print(response["status"]["errorResult"]) print("\tERROR %s: %s.%s " % (job_type, dataset_id, table_id)) return response["status"]["errorResult"] except KeyError: print("\tSUCCESS %s: %s.%s " % (job_type, dataset_id, table_id)) return "\t {statistics}".format(statistics=response["statistics"])
class Novel: id: int title: str author: str library: str cover: bytes status: str statusCode: int totalWords: str briefIntroduction: str copyright: bool volumeList: List[dict] @retry(stop=stop_after_attempt(3), retry=retry_if_exception_type(ConnectionError)) @retry(stop=stop_after_attempt(3), retry=retry_if_exception_type(ProtocolError)) @retry(stop=stop_after_attempt(3), retry=retry_if_exception_type(CE)) @retry(stop=stop_after_attempt(3), retry=retry_if_exception_type(TimeoutError)) def __init__(self, articleid: int): main_page_request = requests.get( f"http://www.wenku8.net/book/{articleid}.htm", headers=headers, cookies=SelfUser.cookies) main_page_request.encoding = "gbk" main_page = BeautifulSoup(main_page_request.text, features="html.parser") main_web_content = main_page.text self.id = articleid self.title = fast_regex(r"槽([\s\S]*)\[推", main_web_content).lstrip() self.statusCode = int(str(articleid)[0:1]) if len(str( articleid)) >= 4 else 0 # 这个statusCode实际上就是把文库的小说按每1000本进1的方式搞出来的 assert bool(self.title) self.author = fast_regex(r"小说作者:(.*)", main_web_content) self.library = fast_regex(r"文库分类:(.*)", main_web_content) self.status = fast_regex(r"文章状态:(.*)", main_web_content) self.totalWords = fast_regex(r"全文长度:(.*)字", main_web_content) self.copyright = True if main_web_content.find("版权问题") == -1 else False self.briefIntroduction = fast_regex("内容简介:([\s\S]*)阅读\n小说目录", main_web_content).lstrip().rstrip().replace(' ', '') \ .replace("\n\n", "").replace(" ", "") self.cover = request( f"https://img.wenku8.com/image/{self.statusCode}/{self.id}/{self.id}s.jpg", SelfUser.cookies).content read_page_request = requests.get( f"http://www.wenku8.net/novel/{self.statusCode}/{articleid}/index.htm", cookies=SelfUser.cookies, headers=headers) read_page_request.encoding = "gbk" read_page = BeautifulSoup(read_page_request.text, features="html.parser") tags = read_page.find_all("td") volumeList = [] for i in tags: if i["class"][0] == "vcss": volumeList.append({"name": str(i.string), "chapters": []}) elif i["class"][0] == "ccss" and i.string != "\xa0": volumeList[len(volumeList) - 1]["chapters"].append({ "name": str(i.string), "cid": int(i.a["href"].replace(".htm", "")) }) self.volumeList = volumeList def to_dict(self) -> dict: return { "id": self.id, "title": self.title, "author": self.author, "library": self.library, "cover": self.cover, "status": self.status, "statusCode": self.statusCode, "totalWords": self.totalWords, "briefIntroduction": self.briefIntroduction, "copyright": self.copyright, "volumeList": self.volumeList }
class NuAPI: def __init__(self, apiKey, verbose): self.__apiKey = apiKey self.__apiHeaders = {"X-NUID-API-KEY": self.__apiKey} # set our API key in the header as required. self.__v = verbose self.__api = "https://nebulous.nuid.io" self.__secrets = None self.__evtLog = WindowsEventWriter() # API route: https://nebulous.nuid.io/api/search/hash/NTLMSHA2/<sha256(ntlm)> self.__route_url = "{}/api/search/hash/NTLMSHA2".format(self.__api) # API route: https://nebulous.nuid.io/api/search/kanon/<query> self.__anon_url = "{}/api/search/kanon/NTLMSHA2".format(self.__api) @retry(retry=retry_if_exception_type(APIRetryException), stop=stop_after_attempt(10), wait=wait_random(10, 30)) def anon_api_helper(self, user): """ Helper function for check_hashes(). Pass over an iterable of the users' keys in __secrets and feed it to this. :param user: Pass in the username of the user to look up. This should be a key of __secrets we can use to lookup the users information. :return: None. :raises: APIRetryException @ if the request fails for whatever reason. """ # Broilerplate the data for the Windows Event Log. Need to do this to redact the NT hashes from the log file. audit_data = { "accoutName": user, "Status": self.__secrets[user].get("Status"), "Check": { "LastCheck": None, "Which": [], "Compromised": None } } h = sha256() # Set the timestamps now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.__secrets[user]['Check']['LastCheck'] = now audit_data['Check']['LastCheck'] = now if self.__v: logging.info(logger.Fore.LIGHTBLACK_EX + "Checking hash for account: {}".format(user, logger.Fore.RESET)) try: # First we check the accounts active NTLM hash. nthash = self.__secrets[user]['NTLM_Hash'] h.update(nthash) # Calculate SHA256(NTLM) digest = h.hexdigest() # Save this digest for comparison later. url = "{}/{}".format(self.__anon_url, digest[0:5]) resp = requests.get(url, headers=self.__apiHeaders, timeout=60) except (requests.HTTPError, requests.ConnectTimeout, requests.ConnectionError, ReadTimeout) as err: logging.warning("Had trouble connecting to the API. Will retry this request. ERR: {}".format(err)) raise APIRetryException except UnicodeDecodeError as unierr: logging.error("Malformated request URL. Did the hash decode correctly? Exiting. {}".format(unierr)) exit(1) if resp.status_code == 200: for hit in resp.json()['data'].get('matches'): if hit == digest: logging.warning( logger.Fore.LIGHTRED_EX + "CURRENT PASSWORD for account: {} is COMPROMISED!".format(user) + logger.Fore.RESET) self.__secrets[user]["Check"]["Compromised"] = True self.__secrets[user]["Check"]["Which"].append("NTLM_Hash") break if self.__secrets[user]["Check"]["Compromised"] is None: self.__secrets[user]["Check"]["Compromised"] = False if self.__v: logging.info(logger.Fore.LIGHTBLACK_EX + "Hash for account:" " {} is OK.{}".format(user, logger.Fore.RESET)) elif resp.status_code == 404: logging.error("API error. Retrying the request. Code: {}".format(resp.status_code)) raise APIRetryException elif resp.status_code == 429: logging.warning("We are being rate limited. Retrying request and throttling connection.") raise APIRetryException elif resp.status_code == 500: logging.error("API error. Retrying the request. Code: {}".format(resp.status_code)) raise APIRetryException if self.__secrets[user].get("History"): # Need to split this one up so we can retry each individual request. Should prevent log spam from retrying. self.check_anon_history(user) audit_data['Check']['Which'] = self.__secrets[user]['Check']['Which'] # Setup the data to be submitted to the audit log. audit_data['Check']['Compromised'] = self.__secrets[user]['Check']['Compromised'] audit_data['Check']['Which'] = self.__secrets[user]['Check']['Which'] # Write our results and redacted data to the Windows event log. self.__evtLog.write_event(user, audit_data['Check']['Compromised'], audit_data) return 0 @retry(retry=retry_if_exception_type(APIRetryException), stop=stop_after_attempt(10), wait=wait_random(10, 30)) def check_anon_history(self, user): """ We split up check_history to its own function. This way if we get a failure when checking an old hash, it does not restart the entire process. However, if we get a failure, it will restart all of the history over again. :param user: Pass in the username of the user to look up. This should be a key of __secrets we can use to lookup the users information. :return: None. :raises: APIRetryException @ if the request fails for whatever reason. """ # TODO: Get better exception handling so we don't retry all histories if one fails or gets rate limited. for nthistory in self.__secrets[user]["History"]: h = sha256() ntlm_hist = self.__secrets[user]["History"][nthistory] h.update(ntlm_hist) digest = h.hexdigest() if self.__v: logging.info(logger.Fore.LIGHTBLACK_EX + "Checking {} for user: {}...".format(nthistory, user) + logger.Fore.RESET) try: url = "{}/{}".format(self.__anon_url, digest[0:5]) resp = requests.get(url, headers=self.__apiHeaders, timeout=60) except (requests.HTTPError, requests.ConnectTimeout, requests.ConnectionError, ReadTimeout) as err: logging.warning("Had trouble connecting to the API. Will retry this request. ERR: {}".format(err)) raise APIRetryException if resp.status_code == 200: for hit in resp.json()['data'].get('matches'): if hit == digest: logging.warning(logger.Fore.RED + "Historic hash {} for account: {} is INACTIVE and " "COMPROMISED!".format(nthistory, user) + logger.Fore.RESET) self.__secrets[user]["Check"]["Compromised"] = True self.__secrets[user]["Check"]["Which"].append(nthistory) break if self.__secrets[user]["Check"]["Compromised"] is None: self.__secrets[user]["Check"]["Compromised"] = False if self.__v: logging.info(logger.Fore.LIGHTBLACK_EX + "Historic hash for account:" " {} is OK.{}".format(user, logger.Fore.RESET)) elif resp.status_code == 404: logging.error("API error. Retrying the request. Code: {}".format(resp.status_code)) raise APIRetryException elif resp.status_code == 429: logging.warning("We are being rate limited. Retrying request and throttling connection.") raise APIRetryException elif resp.status_code == 500: logging.error("API error. Retrying the request. Code: {}".format(resp.status_code)) raise APIRetryException @retry(retry=retry_if_exception_type(APIRetryException), stop=stop_after_attempt(10), wait=wait_random(10, 30)) def api_helper(self, user): """ Helper function for check_hashes(). Pass over an iterable of the users' keys in __secrets and feed it to this. :param user: Pass in the username of the user to look up. This should be a key of __secrets we can use to lookup the users information. :return: None. :raises: APIRetryException @ if the request fails for whatever reason. """ # Broilerplate the data for the Windows Event Log. Need to do this to redact the NT hashes from the log file. audit_data = { "accoutName": user, "Status": self.__secrets[user].get("Status"), "Check": { "LastCheck": None, "Which": [], "Compromised": None } } h = sha256() # Set the timestamps now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.__secrets[user]['Check']['LastCheck'] = now audit_data['Check']['LastCheck'] = now if self.__v: logging.info(logger.Fore.LIGHTBLACK_EX + "Checking hash for account: {}".format(user, logger.Fore.RESET)) try: # First we check the accounts active NTLM hash. nthash = self.__secrets[user]['NTLM_Hash'] h.update(nthash) # Calculate SHA256(NTLM) url = "{}/{}".format(self.__route_url, h.hexdigest()) resp = requests.get(url, headers=self.__apiHeaders, timeout=60) except (requests.HTTPError, requests.ConnectTimeout, requests.ConnectionError, ReadTimeout) as err: logging.warning("Had trouble connecting to the API. Will retry this request. ERR: {}".format(err)) raise APIRetryException except UnicodeDecodeError as unierr: logging.error("Malformated request URL. Did the hash decode correctly? Exiting. {}".format(unierr)) exit(1) if resp.status_code == 200: logging.warning(logger.Fore.LIGHTRED_EX + "CURRENT PASSWORD for account: {} is COMPROMISED!".format(user) + logger.Fore.RESET) self.__secrets[user]["Check"]["Compromised"] = True self.__secrets[user]["Check"]["Which"].append("NTLM_Hash") elif resp.status_code == 404: if self.__v: logging.info(logger.Fore.LIGHTBLACK_EX + "Hash for account:" " {} is OK.{}".format(user, logger.Fore.RESET)) self.__secrets[user]["Check"]["Compromised"] = False elif resp.status_code == 429: logging.warning("We are being rate limited. Retrying request and throttling connection.") raise APIRetryException elif resp.status_code == 500: logging.error("API error. Retrying the request. Code: {}".format(resp.status_code)) raise APIRetryException if self.__secrets[user].get("History"): # Need to split this one up so we can retry each individual request. Should prevent log spam from retrying. self.check_history(user) audit_data['Check']['Which'] = self.__secrets[user]['Check']['Which'] # Setup the data to be submitted to the audit log. audit_data['Check']['Compromised'] = self.__secrets[user]['Check']['Compromised'] audit_data['Check']['Which'] = self.__secrets[user]['Check']['Which'] # Write our results and redacted data to the Windows event log. self.__evtLog.write_event(user, audit_data['Check']['Compromised'], audit_data) return 0 @retry(retry=retry_if_exception_type(APIRetryException), stop=stop_after_attempt(10), wait=wait_random(10, 30)) def check_history(self, user): """ We split up check_history to its own function. This way if we get a failure when checking an old hash, it does not restart the entire process. However, if we get a failure, it will restart all of the history over again. :param user: Pass in the username of the user to look up. This should be a key of __secrets we can use to lookup the users information. :return: None. :raises: APIRetryException @ if the request fails for whatever reason. """ # TODO: Get better exception handling so we don't retry all histories if one fails or gets rate limited. for nthistory in self.__secrets[user]["History"]: h = sha256() ntlm_hist = self.__secrets[user]["History"][nthistory] h.update(ntlm_hist) if self.__v: logging.info(logger.Fore.LIGHTBLACK_EX + "Checking {} for user: {}...".format(nthistory, user) + logger.Fore.RESET) try: url = "{}/{}".format(self.__route_url, h.hexdigest()) resp = requests.get(url, headers=self.__apiHeaders, timeout=60) except (requests.HTTPError, requests.ConnectTimeout, requests.ConnectionError, ReadTimeout) as err: logging.warning("Had trouble connecting to the API. Will retry this request. ERR: {}".format(err)) raise APIRetryException if resp.status_code == 200: logging.warning(logger.Fore.RED + "Historic Hash {} for account: {} is INACTIVE and " "COMPROMISED!".format(nthistory, user) + logger.Fore.RESET) self.__secrets[user]["Check"]["Compromised"] = True self.__secrets[user]["Check"]["Which"].append(nthistory) elif resp.status_code == 404: if self.__v: logging.info( logger.Fore.LIGHTBLACK_EX + "Historic hash {} for account:" " {} is OK. {}".format(nthistory, user, logger.Fore.RESET)) elif resp.status_code == 429: logging.warning("We are being rate limited. Retrying request and throttling connection.") raise APIRetryException elif resp.status_code == 500: logging.error("API error. Retrying the request. Code: {}".format(resp.status_code)) raise APIRetryException def check_hashes(self, secrets, kanon): """ Feed this a secrets_dict from secretsdump.py and we can check against NuID's api if the hash exists. :param secrets: secrets_dict object from secretsdump.NTDSHashes(). :return: Return the updated secrets_dict, after all checks have completed. """ self.__secrets = secrets # Use a minumum of 16 threads for network bound I/O. More if we have them. if cpu_count() < 16: procs = 16 else: procs = cpu_count() p = Pool(processes=procs) if kanon: logging.info("Checking a total of {} accounts for compromised credentials with k-Anon".format(len(self.__secrets))) p.imap_unordered(self.anon_api_helper, self.__secrets) else: logging.info("Checking a total of {} accounts for compromised credentials.".format(len(self.__secrets))) p.imap_unordered(self.api_helper, self.__secrets) p.close() p.join() logging.info("Done checking hashes.") return self.__secrets
def render_text(self, output: str) -> str: """Exports the info to string""" console = AirflowConsole(record=True) with console.capture(): self.show(output=output, console=console) return console.export_text() class FileIoException(Exception): """Raises when error happens in FileIo.io integration""" @tenacity.retry( stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_exponential(multiplier=1, max=10), retry=tenacity.retry_if_exception_type(FileIoException), before=tenacity.before_log(log, logging.DEBUG), after=tenacity.after_log(log, logging.DEBUG), ) def _upload_text_to_fileio(content): """Upload text file to File.io service and return lnk""" resp = requests.post("https://file.io", data={"text": content}) if not resp.ok: print(resp.json()) raise FileIoException("Failed to send report to file.io service.") try: return resp.json()["link"] except ValueError as e: log.debug(e) raise FileIoException("Failed to send report to file.io service.")
import re from typing import Any, Iterable import psycopg2 import psycopg2.extensions from tenacity import before_log, retry, retry_if_exception_type, stop_after_delay, wait_random_exponential from connstr import ConnectionString log = logging.getLogger(__name__) PGConnection = psycopg2.extensions.connection @retry( retry=retry_if_exception_type(psycopg2.OperationalError), stop=stop_after_delay(300), wait=wait_random_exponential(multiplier=1, max=15), reraise=True, before=before_log(log, logging.DEBUG), ) def connect(conn_str: ConnectionString) -> PGConnection: con = psycopg2.connect(str(conn_str)) con.autocommit = True return con def ensure_user(con: PGConnection, username: str, password: str, superuser: bool = False,
return size, count @app.task(bind=True, event_type=50710) def DeleteFiles(self, path): path, = self.parse_params(path) delete_path(path) msg = "Deleted %s" % path self.create_success_event(msg) @app.task(bind=True) @retry(reraise=True, retry=retry_if_exception_type(NoSpaceLeftError), wait=wait_exponential(max=60), stop=stop_after_delay(600)) def CopyDir(self, src, dst, remote_credentials=None, block_size=DEFAULT_BLOCK_SIZE): src, dst = self.parse_params(src, dst) requests_session = None if remote_credentials: user, passw = decrypt_remote_credentials(remote_credentials) requests_session = requests.Session() requests_session.verify = settings.REQUESTS_VERIFY requests_session.auth = (user, passw)
body: str, ) -> SHA: url = make_file_content_url(doc) data = { "message": "Interactive edit", "content": base64.b64encode(body.encode()).decode(), "sha": old_file_sha, "branch": new_branch, } response = session.put(url, json=data) response.raise_for_status() return response.json()["commit"]["sha"] @tenacity.retry( retry=tenacity.retry_if_exception_type(requests.exceptions.HTTPError), wait=tenacity.wait_fixed(10), stop=tenacity.stop_after_attempt(3), ) def _create_branch(session, doc, name, parent_sha): url = doc.api_url.child("git").child("refs") r = session.post(url, json={ "ref": f"refs/heads/{name}", "sha": parent_sha }) r.raise_for_status() def create_branch(session: requests.Session, doc: interedit.app.RenderedDocument, parent_sha: SHA) -> str:
failures.add('collector %s failed: %s', name, exc) resp = call_inspector(data, failures) # Now raise everything we were delaying failures.raise_if_needed() if resp is None: raise errors.InspectionError('stopping inspection, as inspector ' 'returned an error') LOG.info('inspection finished successfully') return resp.get('uuid') @tenacity.retry(retry=tenacity.retry_if_exception_type( requests.exceptions.ConnectionError), stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_fixed(5), reraise=True) def _post_to_inspector(url, data, verify, cert): return requests.post(CONF.inspection_callback_url, data=data, verify=verify, cert=cert) def call_inspector(data, failures): """Post data to inspector.""" data['error'] = failures.get_error() LOG.info('posting collected data to %s', CONF.inspection_callback_url)
class NetworkExplorer(object): def __init__(self, this, region_name=None): session = helpers.get_execution_session() self._project_id = session.project_id self._settings = CONF.networking self._available_cidrs = self._generate_possible_cidrs() self._region = this.find_owner('io.murano.CloudRegion') self._region_name = region_name @staticmethod @session_local_storage.execution_session_memoize def _get_client(region_name): return nclient.Client(**auth_utils.get_session_client_parameters( service_type='network', region=region_name, conf='neutron')) @property def _client(self): region = self._region_name or (None if self._region is None else self._region['name']) return self._get_client(region) # NOTE(starodubcevna): to avoid simultaneous router requests we use retry # decorator with random delay 1-10 seconds between attempts and maximum # delay time 30 seconds. @tenacity.retry(retry=tenacity.retry_if_exception_type( exc.RouterInfoException), stop=tenacity.stop_after_delay(30), wait=tenacity.wait_random(min=1, max=10), reraise=True) def get_default_router(self): router_name = self._settings.router_name routers = self._client.list_routers(tenant_id=self._project_id, name=router_name).get('routers') if len(routers) == 0: LOG.debug('Router {name} not found'.format(name=router_name)) if self._settings.create_router: LOG.debug('Attempting to create Router {router}'.format( router=router_name)) external_network = self._settings.external_network kwargs = {'id': external_network} \ if uuidutils.is_uuid_like(external_network) \ else {'name': external_network} networks = self._client.list_networks(**kwargs).get('networks') ext_nets = list( filter(lambda n: n['router:external'], networks)) if len(ext_nets) == 0: raise KeyError('Router %s could not be created, ' 'no external network found' % router_name) nid = ext_nets[0]['id'] body_data = { 'router': { 'name': router_name, 'external_gateway_info': { 'network_id': nid }, 'admin_state_up': True, } } router = self._client.create_router( body=body_data).get('router') LOG.info('Created router: {id}'.format(id=router['id'])) return router['id'] else: raise KeyError('Router %s was not found' % router_name) else: if routers[0]['external_gateway_info'] is None: raise exc.RouterInfoException('Please set external gateway for' ' the router %s ' % router_name) router_id = routers[0]['id'] return router_id def get_available_cidr(self, router_id, net_id, ip_version=4): """Uses hash of network IDs to minimize the collisions Different nets will attempt to pick different cidrs out of available range. If the cidr is taken will pick another one. """ taken_cidrs = self._get_cidrs_taken_by_router(router_id) id_hash = hash(net_id) num_fails = 0 available_ipv6_cidrs = [] if ip_version == 6: for cidr in self._available_cidrs: available_ipv6_cidrs.append(cidr.ipv6()) self._available_cidrs = available_ipv6_cidrs while num_fails < len(self._available_cidrs): cidr = self._available_cidrs[(id_hash + num_fails) % len(self._available_cidrs)] if any( self._cidrs_overlap(cidr, taken_cidr) for taken_cidr in taken_cidrs): num_fails += 1 else: return str(cidr) return None def get_default_dns(self, ip_version=4): dns_list = self._settings.default_dns valid_dns = [] for ip in dns_list: if ip_version == 6 and netutils.is_valid_ipv6(ip): valid_dns.append(ip) elif ip_version == 4 and netutils.is_valid_ipv4(ip): valid_dns.append(ip) else: LOG.warning('{0} is not a vaild IPV{1} address, ' 'ingore...'.format(ip, ip_version)) return valid_dns def get_external_network_id_for_router(self, router_id): router = self._client.show_router(router_id).get('router') if not router or 'external_gateway_info' not in router: return None return router['external_gateway_info'].get('network_id') def get_external_network_id_for_network(self, network_id): network = self._client.show_network(network_id).get('network') if network.get('router:external', False): return network_id # Get router interfaces of the network router_ports = self._client.list_ports(**{ 'device_owner': 'network:router_interface', 'network_id': network_id }).get('ports') # For each router this network is connected to # check if the router has external_gateway set for router_port in router_ports: ext_net_id = self.get_external_network_id_for_router( router_port.get('device_id')) if ext_net_id: return ext_net_id return None def _get_cidrs_taken_by_router(self, router_id): if not router_id: return [] ports = self._client.list_ports(device_id=router_id)['ports'] subnet_ids = [] for port in ports: for fixed_ip in port['fixed_ips']: subnet_ids.append(fixed_ip['subnet_id']) all_subnets = self._client.list_subnets()['subnets'] filtered_cidrs = [ netaddr.IPNetwork(subnet['cidr']) for subnet in all_subnets if subnet['id'] in subnet_ids ] return filtered_cidrs @staticmethod def _cidrs_overlap(cidr1, cidr2): return (cidr1 in cidr2) or (cidr2 in cidr1) def _generate_possible_cidrs(self): bits_for_envs = int( math.ceil(math.log(self._settings.max_environments, 2))) bits_for_hosts = int(math.ceil(math.log(self._settings.max_hosts, 2))) width = ipv4.width mask_width = width - bits_for_hosts - bits_for_envs net = netaddr.IPNetwork('{0}/{1}'.format( self._settings.env_ip_template, mask_width)) return list(net.subnet(width - bits_for_hosts)) def list_networks(self): return self._client.list_networks()['networks'] def list_subnetworks(self): return self._client.list_subnets()['subnets'] def list_ports(self): return self._client.list_ports()['ports'] def list_neutron_extensions(self): return self._client.list_extensions()['extensions']
if retry_state.outcome.failed: verb, value = 'raised', retry_state.outcome.exception() else: verb, value = 'returned', retry_state.outcome.result() logger.debug( "Retrying %s in %sms (attempt: %s) as it %s %s[%s]." % (get_callback_name(retry_state.fn), int(getattr(retry_state.next_action, 'sleep') * 1000), retry_state.attempt_number, verb, value.__class__.__name__, value)) @retry(reraise=True, stop=stop_after_attempt(2), wait=wait_random(min=0.01, max=0.04), retry=retry_if_exception_type(PermissionError), before_sleep=my_before_sleep) def get_bash_line_retry(*args): return get_bash_line(*args) def get_tmux_bash_pid(session): # out = subprocess.check_output(['tmux', 'list-panes', '-s', '-t', session, '-F', "#{pane_active} #{pane_pid}"]) out = subprocess.check_output([ 'tmux', 'list-window', '-t', session, '-F', "#{window_active} #{pane_pid}" ]) # print('out', out) out2 = [l.split(' ') for l in out.decode().splitlines()] # print('out2', out2) for active, pid in out2:
class ControllerWorker(object): def __init__(self): self._amphora_repo = repo.AmphoraRepository() self._amphora_health_repo = repo.AmphoraHealthRepository() self._health_mon_repo = repo.HealthMonitorRepository() self._lb_repo = repo.LoadBalancerRepository() self._listener_repo = repo.ListenerRepository() self._member_repo = repo.MemberRepository() self._pool_repo = repo.PoolRepository() self._l7policy_repo = repo.L7PolicyRepository() self._l7rule_repo = repo.L7RuleRepository() self._flavor_repo = repo.FlavorRepository() self._az_repo = repo.AvailabilityZoneRepository() if CONF.task_flow.jobboard_enabled: persistence = tsk_driver.MysqlPersistenceDriver() self.jobboard_driver = stevedore_driver.DriverManager( namespace='octavia.worker.jobboard_driver', name=CONF.task_flow.jobboard_backend_driver, invoke_args=(persistence, ), invoke_on_load=True).driver else: self.tf_engine = base_taskflow.BaseTaskFlowEngine() @tenacity.retry( retry=(tenacity.retry_if_result(_is_provisioning_status_pending_update) | tenacity.retry_if_exception_type()), wait=tenacity.wait_incrementing( CONF.haproxy_amphora.api_db_commit_retry_initial_delay, CONF.haproxy_amphora.api_db_commit_retry_backoff, CONF.haproxy_amphora.api_db_commit_retry_max), stop=tenacity.stop_after_attempt( CONF.haproxy_amphora.api_db_commit_retry_attempts)) def _get_db_obj_until_pending_update(self, repo, id): return repo.get(db_apis.get_session(), id=id) @property def services_controller(self): return base_taskflow.TaskFlowServiceController(self.jobboard_driver) def run_flow(self, func, *args, **kwargs): if CONF.task_flow.jobboard_enabled: self.services_controller.run_poster(func, *args, **kwargs) else: tf = self.tf_engine.taskflow_load(func(*args), **kwargs) with tf_logging.DynamicLoggingListener(tf, log=LOG): tf.run() def create_amphora(self, availability_zone=None): """Creates an Amphora. This is used to create spare amphora. :returns: uuid """ try: store = { constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_SPARES_POOL_PRIORITY, constants.FLAVOR: None, constants.SERVER_GROUP_ID: None, constants.AVAILABILITY_ZONE: None } if availability_zone: store[constants.AVAILABILITY_ZONE] = ( self._az_repo.get_availability_zone_metadata_dict( db_apis.get_session(), availability_zone)) self.run_flow(flow_utils.get_create_amphora_flow, store=store, wait=True) except Exception as e: LOG.error('Failed to create an amphora due to: %s', str(e)) def delete_amphora(self, amphora_id): """Deletes an existing Amphora. :param amphora_id: ID of the amphora to delete :returns: None :raises AmphoraNotFound: The referenced Amphora was not found """ try: amphora = self._amphora_repo.get(db_apis.get_session(), id=amphora_id) store = {constants.AMPHORA: amphora.to_dict()} self.run_flow(flow_utils.get_delete_amphora_flow, store=store) except Exception as e: LOG.error('Failed to delete a amphora %s due to: %s', amphora_id, str(e)) return LOG.info('Finished deleting amphora %s.', amphora_id) @tenacity.retry(retry=tenacity.retry_if_exception_type( db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( CONF.haproxy_amphora.api_db_commit_retry_initial_delay, CONF.haproxy_amphora.api_db_commit_retry_backoff, CONF.haproxy_amphora.api_db_commit_retry_max), stop=tenacity.stop_after_attempt( CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_health_monitor(self, health_monitor): """Creates a health monitor. :param health_monitor: Provider health monitor dict :returns: None :raises NoResultFound: Unable to find the object """ db_health_monitor = self._health_mon_repo.get( db_apis.get_session(), id=health_monitor[constants.HEALTHMONITOR_ID]) if not db_health_monitor: LOG.warning( 'Failed to fetch %s %s from DB. Retrying for up to ' '60 seconds.', 'healthmonitor', health_monitor[constants.HEALTHMONITOR_ID]) raise db_exceptions.NoResultFound pool = db_health_monitor.pool pool.health_monitor = db_health_monitor load_balancer = pool.load_balancer provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( pool.listeners)) store = { constants.HEALTH_MON: health_monitor, constants.POOL_ID: pool.id, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: load_balancer.id, constants.LOADBALANCER: provider_lb } self.run_flow(flow_utils.get_create_health_monitor_flow, store=store) def delete_health_monitor(self, health_monitor): """Deletes a health monitor. :param health_monitor: Provider health monitor dict :returns: None :raises HMNotFound: The referenced health monitor was not found """ db_health_monitor = self._health_mon_repo.get( db_apis.get_session(), id=health_monitor[constants.HEALTHMONITOR_ID]) pool = db_health_monitor.pool load_balancer = pool.load_balancer provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( pool.listeners)) store = { constants.HEALTH_MON: health_monitor, constants.POOL_ID: pool.id, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: load_balancer.id, constants.LOADBALANCER: provider_lb, constants.PROJECT_ID: load_balancer.project_id } self.run_flow(flow_utils.get_delete_health_monitor_flow, store=store) def update_health_monitor(self, original_health_monitor, health_monitor_updates): """Updates a health monitor. :param original_health_monitor: Provider health monitor dict :param health_monitor_updates: Dict containing updated health monitor :returns: None :raises HMNotFound: The referenced health monitor was not found """ try: db_health_monitor = self._get_db_obj_until_pending_update( self._health_mon_repo, original_health_monitor[constants.HEALTHMONITOR_ID]) except tenacity.RetryError as e: LOG.warning( 'Health monitor did not go into %s in 60 seconds. ' 'This either due to an in-progress Octavia upgrade ' 'or an overloaded and failing database. Assuming ' 'an upgrade is in progress and continuing.', constants.PENDING_UPDATE) db_health_monitor = e.last_attempt.result() pool = db_health_monitor.pool listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( pool.listeners)) load_balancer = pool.load_balancer provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() store = { constants.HEALTH_MON: original_health_monitor, constants.POOL_ID: pool.id, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: load_balancer.id, constants.LOADBALANCER: provider_lb, constants.UPDATE_DICT: health_monitor_updates } self.run_flow(flow_utils.get_update_health_monitor_flow, store=store) @tenacity.retry(retry=tenacity.retry_if_exception_type( db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( CONF.haproxy_amphora.api_db_commit_retry_initial_delay, CONF.haproxy_amphora.api_db_commit_retry_backoff, CONF.haproxy_amphora.api_db_commit_retry_max), stop=tenacity.stop_after_attempt( CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_listener(self, listener): """Creates a listener. :param listener: A listener provider dictionary. :returns: None :raises NoResultFound: Unable to find the object """ db_listener = self._listener_repo.get( db_apis.get_session(), id=listener[constants.LISTENER_ID]) if not db_listener: LOG.warning( 'Failed to fetch %s %s from DB. Retrying for up to ' '60 seconds.', 'listener', listener[constants.LISTENER_ID]) raise db_exceptions.NoResultFound load_balancer = db_listener.load_balancer listeners = load_balancer.listeners dict_listeners = [] for li in listeners: dict_listeners.append( provider_utils.db_listener_to_provider_listener(li).to_dict()) provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() store = { constants.LISTENERS: dict_listeners, constants.LOADBALANCER: provider_lb, constants.LOADBALANCER_ID: load_balancer.id } self.run_flow(flow_utils.get_create_listener_flow, store=store) def delete_listener(self, listener): """Deletes a listener. :param listener: A listener provider dictionary to delete :returns: None :raises ListenerNotFound: The referenced listener was not found """ store = { constants.LISTENER: listener, constants.LOADBALANCER_ID: listener[constants.LOADBALANCER_ID], constants.PROJECT_ID: listener[constants.PROJECT_ID] } self.run_flow(flow_utils.get_delete_listener_flow, store=store) def update_listener(self, listener, listener_updates): """Updates a listener. :param listener: A listener provider dictionary to update :param listener_updates: Dict containing updated listener attributes :returns: None :raises ListenerNotFound: The referenced listener was not found """ db_lb = self._lb_repo.get(db_apis.get_session(), id=listener[constants.LOADBALANCER_ID]) store = { constants.LISTENER: listener, constants.UPDATE_DICT: listener_updates, constants.LOADBALANCER_ID: db_lb.id, constants.LISTENERS: [listener] } self.run_flow(flow_utils.get_update_listener_flow, store=store) @tenacity.retry(retry=tenacity.retry_if_exception_type( db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( CONF.haproxy_amphora.api_db_commit_retry_initial_delay, CONF.haproxy_amphora.api_db_commit_retry_backoff, CONF.haproxy_amphora.api_db_commit_retry_max), stop=tenacity.stop_after_attempt( CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_load_balancer(self, loadbalancer, flavor=None, availability_zone=None): """Creates a load balancer by allocating Amphorae. First tries to allocate an existing Amphora in READY state. If none are available it will attempt to build one specifically for this load balancer. :param loadbalancer: The dict of load balancer to create :returns: None :raises NoResultFound: Unable to find the object """ lb = self._lb_repo.get(db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) if not lb: LOG.warning( 'Failed to fetch %s %s from DB. Retrying for up to ' '60 seconds.', 'load_balancer', loadbalancer[constants.LOADBALANCER_ID]) raise db_exceptions.NoResultFound store = { lib_consts.LOADBALANCER_ID: loadbalancer[lib_consts.LOADBALANCER_ID], constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_NORMAL_PRIORITY, lib_consts.FLAVOR: flavor, lib_consts.AVAILABILITY_ZONE: availability_zone } topology = lb.topology if (not CONF.nova.enable_anti_affinity or topology == constants.TOPOLOGY_SINGLE): store[constants.SERVER_GROUP_ID] = None listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( lb.listeners)) store[constants.UPDATE_DICT] = {constants.TOPOLOGY: topology} self.run_flow(flow_utils.get_create_load_balancer_flow, topology, listeners=listeners_dicts, store=store) def delete_load_balancer(self, load_balancer, cascade=False): """Deletes a load balancer by de-allocating Amphorae. :param load_balancer: Dict of the load balancer to delete :returns: None :raises LBNotFound: The referenced load balancer was not found """ loadbalancer_id = load_balancer[constants.LOADBALANCER_ID] db_lb = self._lb_repo.get(db_apis.get_session(), id=loadbalancer_id) store = { constants.LOADBALANCER: load_balancer, constants.LOADBALANCER_ID: loadbalancer_id, constants.SERVER_GROUP_ID: db_lb.server_group_id, constants.PROJECT_ID: db_lb.project_id } if cascade: listeners = flow_utils.get_listeners_on_lb(db_lb) pools = flow_utils.get_pools_on_lb(db_lb) self.run_flow(flow_utils.get_cascade_delete_load_balancer_flow, load_balancer, listeners, pools, store=store) else: self.run_flow(flow_utils.get_delete_load_balancer_flow, load_balancer, store=store) def update_load_balancer(self, original_load_balancer, load_balancer_updates): """Updates a load balancer. :param original_load_balancer: Dict of the load balancer to update :param load_balancer_updates: Dict containing updated load balancer :returns: None :raises LBNotFound: The referenced load balancer was not found """ store = { constants.LOADBALANCER: original_load_balancer, constants.LOADBALANCER_ID: original_load_balancer[constants.LOADBALANCER_ID], constants.UPDATE_DICT: load_balancer_updates } self.run_flow(flow_utils.get_update_load_balancer_flow, store=store) @tenacity.retry(retry=tenacity.retry_if_exception_type( db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( CONF.haproxy_amphora.api_db_commit_retry_initial_delay, CONF.haproxy_amphora.api_db_commit_retry_backoff, CONF.haproxy_amphora.api_db_commit_retry_max), stop=tenacity.stop_after_attempt( CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_member(self, member): """Creates a pool member. :param member: A member provider dictionary to create :returns: None :raises NoSuitablePool: Unable to find the node pool """ pool = self._pool_repo.get(db_apis.get_session(), id=member[constants.POOL_ID]) load_balancer = pool.load_balancer provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( pool.listeners)) store = { constants.MEMBER: member, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: load_balancer.id, constants.LOADBALANCER: provider_lb, constants.POOL_ID: pool.id } if load_balancer.availability_zone: store[constants.AVAILABILITY_ZONE] = ( self._az_repo.get_availability_zone_metadata_dict( db_apis.get_session(), load_balancer.availability_zone)) else: store[constants.AVAILABILITY_ZONE] = {} self.run_flow(flow_utils.get_create_member_flow, store=store) def delete_member(self, member): """Deletes a pool member. :param member: A member provider dictionary to delete :returns: None :raises MemberNotFound: The referenced member was not found """ pool = self._pool_repo.get(db_apis.get_session(), id=member[constants.POOL_ID]) load_balancer = pool.load_balancer provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( pool.listeners)) store = { constants.MEMBER: member, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: load_balancer.id, constants.LOADBALANCER: provider_lb, constants.POOL_ID: pool.id, constants.PROJECT_ID: load_balancer.project_id } if load_balancer.availability_zone: store[constants.AVAILABILITY_ZONE] = ( self._az_repo.get_availability_zone_metadata_dict( db_apis.get_session(), load_balancer.availability_zone)) else: store[constants.AVAILABILITY_ZONE] = {} self.run_flow(flow_utils.get_delete_member_flow, store=store) def batch_update_members(self, old_members, new_members, updated_members): updated_members = [(provider_utils.db_member_to_provider_member( self._member_repo.get(db_apis.get_session(), id=m.get(constants.ID))).to_dict(), m) for m in updated_members] provider_old_members = [ provider_utils.db_member_to_provider_member( self._member_repo.get(db_apis.get_session(), id=m.get(constants.ID))).to_dict() for m in old_members ] if old_members: pool = self._pool_repo.get(db_apis.get_session(), id=old_members[0][constants.POOL_ID]) elif new_members: pool = self._pool_repo.get(db_apis.get_session(), id=new_members[0][constants.POOL_ID]) else: pool = self._pool_repo.get( db_apis.get_session(), id=updated_members[0][0][constants.POOL_ID]) load_balancer = pool.load_balancer listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( pool.listeners)) provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() store = { constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: load_balancer.id, constants.LOADBALANCER: provider_lb, constants.POOL_ID: pool.id, constants.PROJECT_ID: load_balancer.project_id } if load_balancer.availability_zone: store[constants.AVAILABILITY_ZONE] = ( self._az_repo.get_availability_zone_metadata_dict( db_apis.get_session(), load_balancer.availability_zone)) else: store[constants.AVAILABILITY_ZONE] = {} self.run_flow(flow_utils.get_batch_update_members_flow, provider_old_members, new_members, updated_members, store=store) def update_member(self, member, member_updates): """Updates a pool member. :param member_id: A member provider dictionary to update :param member_updates: Dict containing updated member attributes :returns: None :raises MemberNotFound: The referenced member was not found """ # TODO(ataraday) when other flows will use dicts - revisit this pool = self._pool_repo.get(db_apis.get_session(), id=member[constants.POOL_ID]) load_balancer = pool.load_balancer provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( pool.listeners)) store = { constants.MEMBER: member, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: load_balancer.id, constants.LOADBALANCER: provider_lb, constants.POOL_ID: pool.id, constants.UPDATE_DICT: member_updates } if load_balancer.availability_zone: store[constants.AVAILABILITY_ZONE] = ( self._az_repo.get_availability_zone_metadata_dict( db_apis.get_session(), load_balancer.availability_zone)) else: store[constants.AVAILABILITY_ZONE] = {} self.run_flow(flow_utils.get_update_member_flow, store=store) @tenacity.retry(retry=tenacity.retry_if_exception_type( db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( CONF.haproxy_amphora.api_db_commit_retry_initial_delay, CONF.haproxy_amphora.api_db_commit_retry_backoff, CONF.haproxy_amphora.api_db_commit_retry_max), stop=tenacity.stop_after_attempt( CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_pool(self, pool): """Creates a node pool. :param pool: Provider pool dict to create :returns: None :raises NoResultFound: Unable to find the object """ # TODO(ataraday) It seems we need to get db pool here anyway to get # proper listeners db_pool = self._pool_repo.get(db_apis.get_session(), id=pool[constants.POOL_ID]) if not db_pool: LOG.warning( 'Failed to fetch %s %s from DB. Retrying for up to ' '60 seconds.', 'pool', pool[constants.POOL_ID]) raise db_exceptions.NoResultFound load_balancer = db_pool.load_balancer provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( db_pool.listeners)) store = { constants.POOL_ID: pool[constants.POOL_ID], constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: load_balancer.id, constants.LOADBALANCER: provider_lb } self.run_flow(flow_utils.get_create_pool_flow, store=store) def delete_pool(self, pool): """Deletes a node pool. :param pool: Provider pool dict to delete :returns: None :raises PoolNotFound: The referenced pool was not found """ db_pool = self._pool_repo.get(db_apis.get_session(), id=pool[constants.POOL_ID]) listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( db_pool.listeners)) load_balancer = db_pool.load_balancer provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() store = { constants.POOL_ID: pool[constants.POOL_ID], constants.LISTENERS: listeners_dicts, constants.LOADBALANCER: provider_lb, constants.LOADBALANCER_ID: load_balancer.id, constants.PROJECT_ID: db_pool.project_id } self.run_flow(flow_utils.get_delete_pool_flow, store=store) def update_pool(self, origin_pool, pool_updates): """Updates a node pool. :param origin_pool: Provider pool dict to update :param pool_updates: Dict containing updated pool attributes :returns: None :raises PoolNotFound: The referenced pool was not found """ try: db_pool = self._get_db_obj_until_pending_update( self._pool_repo, origin_pool[constants.POOL_ID]) except tenacity.RetryError as e: LOG.warning( 'Pool did not go into %s in 60 seconds. ' 'This either due to an in-progress Octavia upgrade ' 'or an overloaded and failing database. Assuming ' 'an upgrade is in progress and continuing.', constants.PENDING_UPDATE) db_pool = e.last_attempt.result() load_balancer = db_pool.load_balancer provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( load_balancer).to_dict() listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( db_pool.listeners)) store = { constants.POOL_ID: db_pool.id, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER: provider_lb, constants.LOADBALANCER_ID: load_balancer.id, constants.UPDATE_DICT: pool_updates } self.run_flow(flow_utils.get_update_pool_flow, store=store) @tenacity.retry(retry=tenacity.retry_if_exception_type( db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( CONF.haproxy_amphora.api_db_commit_retry_initial_delay, CONF.haproxy_amphora.api_db_commit_retry_backoff, CONF.haproxy_amphora.api_db_commit_retry_max), stop=tenacity.stop_after_attempt( CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_l7policy(self, l7policy): """Creates an L7 Policy. :param l7policy: Provider dict of the l7policy to create :returns: None :raises NoResultFound: Unable to find the object """ db_listener = self._listener_repo.get( db_apis.get_session(), id=l7policy[constants.LISTENER_ID]) listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( [db_listener])) store = { constants.L7POLICY: l7policy, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: db_listener.load_balancer.id } self.run_flow(flow_utils.get_create_l7policy_flow, store=store) def delete_l7policy(self, l7policy): """Deletes an L7 policy. :param l7policy: Provider dict of the l7policy to delete :returns: None :raises L7PolicyNotFound: The referenced l7policy was not found """ db_listener = self._listener_repo.get( db_apis.get_session(), id=l7policy[constants.LISTENER_ID]) listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( [db_listener])) store = { constants.L7POLICY: l7policy, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: db_listener.load_balancer.id } self.run_flow(flow_utils.get_delete_l7policy_flow, store=store) def update_l7policy(self, original_l7policy, l7policy_updates): """Updates an L7 policy. :param l7policy: Provider dict of the l7policy to update :param l7policy_updates: Dict containing updated l7policy attributes :returns: None :raises L7PolicyNotFound: The referenced l7policy was not found """ db_listener = self._listener_repo.get( db_apis.get_session(), id=original_l7policy[constants.LISTENER_ID]) listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( [db_listener])) store = { constants.L7POLICY: original_l7policy, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: db_listener.load_balancer.id, constants.UPDATE_DICT: l7policy_updates } self.run_flow(flow_utils.get_update_l7policy_flow, store=store) @tenacity.retry(retry=tenacity.retry_if_exception_type( db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( CONF.haproxy_amphora.api_db_commit_retry_initial_delay, CONF.haproxy_amphora.api_db_commit_retry_backoff, CONF.haproxy_amphora.api_db_commit_retry_max), stop=tenacity.stop_after_attempt( CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_l7rule(self, l7rule): """Creates an L7 Rule. :param l7rule: Provider dict l7rule :returns: None :raises NoResultFound: Unable to find the object """ db_l7policy = self._l7policy_repo.get(db_apis.get_session(), id=l7rule[constants.L7POLICY_ID]) load_balancer = db_l7policy.listener.load_balancer listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( [db_l7policy.listener])) l7policy_dict = provider_utils.db_l7policy_to_provider_l7policy( db_l7policy) store = { constants.L7RULE: l7rule, constants.L7POLICY: l7policy_dict.to_dict(), constants.L7POLICY_ID: db_l7policy.id, constants.LISTENERS: listeners_dicts, constants.LOADBALANCER_ID: load_balancer.id } self.run_flow(flow_utils.get_create_l7rule_flow, store=store) def delete_l7rule(self, l7rule): """Deletes an L7 rule. :param l7rule: Provider dict of the l7rule to delete :returns: None :raises L7RuleNotFound: The referenced l7rule was not found """ db_l7policy = self._l7policy_repo.get(db_apis.get_session(), id=l7rule[constants.L7POLICY_ID]) l7policy = provider_utils.db_l7policy_to_provider_l7policy(db_l7policy) load_balancer = db_l7policy.listener.load_balancer listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( [db_l7policy.listener])) store = { constants.L7RULE: l7rule, constants.L7POLICY: l7policy.to_dict(), constants.LISTENERS: listeners_dicts, constants.L7POLICY_ID: db_l7policy.id, constants.LOADBALANCER_ID: load_balancer.id } self.run_flow(flow_utils.get_delete_l7rule_flow, store=store) def update_l7rule(self, original_l7rule, l7rule_updates): """Updates an L7 rule. :param l7rule: Origin dict of the l7rule to update :param l7rule_updates: Dict containing updated l7rule attributes :returns: None :raises L7RuleNotFound: The referenced l7rule was not found """ db_l7policy = self._l7policy_repo.get( db_apis.get_session(), id=original_l7rule[constants.L7POLICY_ID]) load_balancer = db_l7policy.listener.load_balancer listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( [db_l7policy.listener])) l7policy_dict = provider_utils.db_l7policy_to_provider_l7policy( db_l7policy) store = { constants.L7RULE: original_l7rule, constants.L7POLICY: l7policy_dict.to_dict(), constants.LISTENERS: listeners_dicts, constants.L7POLICY_ID: db_l7policy.id, constants.LOADBALANCER_ID: load_balancer.id, constants.UPDATE_DICT: l7rule_updates } self.run_flow(flow_utils.get_update_l7rule_flow, store=store) def failover_amphora(self, amphora_id): """Perform failover operations for an amphora. Note: This expects the load balancer to already be in provisioning_status=PENDING_UPDATE state. :param amphora_id: ID for amphora to failover :returns: None :raises octavia.common.exceptions.NotFound: The referenced amphora was not found """ amphora = None try: amphora = self._amphora_repo.get(db_apis.get_session(), id=amphora_id) if amphora is None: LOG.error( 'Amphora failover for amphora %s failed because ' 'there is no record of this amphora in the ' 'database. Check that the [house_keeping] ' 'amphora_expiry_age configuration setting is not ' 'too short. Skipping failover.', amphora_id) raise exceptions.NotFound(resource=constants.AMPHORA, id=amphora_id) if amphora.status == constants.DELETED: LOG.warning( 'Amphora %s is marked DELETED in the database but ' 'was submitted for failover. Deleting it from the ' 'amphora health table to exclude it from health ' 'checks and skipping the failover.', amphora.id) self._amphora_health_repo.delete(db_apis.get_session(), amphora_id=amphora.id) return loadbalancer = None if amphora.load_balancer_id: loadbalancer = self._lb_repo.get(db_apis.get_session(), id=amphora.load_balancer_id) lb_amp_count = None if loadbalancer: if loadbalancer.topology == constants.TOPOLOGY_ACTIVE_STANDBY: lb_amp_count = 2 elif loadbalancer.topology == constants.TOPOLOGY_SINGLE: lb_amp_count = 1 az_metadata = {} flavor_dict = {} lb_id = None vip_dict = {} server_group_id = None if loadbalancer: lb_id = loadbalancer.id # Even if the LB doesn't have a flavor, create one and # pass through the topology. if loadbalancer.flavor_id: flavor_dict = self._flavor_repo.get_flavor_metadata_dict( db_apis.get_session(), loadbalancer.flavor_id) flavor_dict[constants.LOADBALANCER_TOPOLOGY] = ( loadbalancer.topology) else: flavor_dict = { constants.LOADBALANCER_TOPOLOGY: loadbalancer.topology } if loadbalancer.availability_zone: az_metadata = ( self._az_repo.get_availability_zone_metadata_dict( db_apis.get_session(), loadbalancer.availability_zone)) vip_dict = loadbalancer.vip.to_dict() server_group_id = loadbalancer.server_group_id provider_lb_dict = ( provider_utils.db_loadbalancer_to_provider_loadbalancer )(loadbalancer).to_dict() if loadbalancer else loadbalancer stored_params = { constants.AVAILABILITY_ZONE: az_metadata, constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_FAILOVER_PRIORITY, constants.FLAVOR: flavor_dict, constants.LOADBALANCER: provider_lb_dict, constants.SERVER_GROUP_ID: server_group_id, constants.LOADBALANCER_ID: lb_id, constants.VIP: vip_dict, constants.AMPHORA_ID: amphora_id } self.run_flow(flow_utils.get_failover_amphora_flow, amphora.to_dict(), lb_amp_count, store=stored_params, wait=True) LOG.info( "Successfully completed the failover for an amphora: %s", { "id": amphora_id, "load_balancer_id": lb_id, "lb_network_ip": amphora.lb_network_ip, "compute_id": amphora.compute_id, "role": amphora.role }) except Exception as e: with excutils.save_and_reraise_exception(reraise=False): LOG.exception("Amphora %s failover exception: %s", amphora_id, str(e)) self._amphora_repo.update(db_apis.get_session(), amphora_id, status=constants.ERROR) if amphora and amphora.load_balancer_id: self._lb_repo.update(db_apis.get_session(), amphora.load_balancer_id, provisioning_status=constants.ERROR) @staticmethod def _get_amphorae_for_failover(load_balancer): """Returns an ordered list of amphora to failover. :param load_balancer: The load balancer being failed over. :returns: An ordered list of amphora to failover, first amp to failover is last in the list :raises octavia.common.exceptions.InvalidTopology: LB has an unknown topology. """ if load_balancer.topology == constants.TOPOLOGY_SINGLE: # In SINGLE topology, amp failover order does not matter return [ a.to_dict() for a in load_balancer.amphorae if a.status != constants.DELETED ] if load_balancer.topology == constants.TOPOLOGY_ACTIVE_STANDBY: # In Active/Standby we should preference the standby amp # for failover first in case the Active is still able to pass # traffic. # Note: The active amp can switch at any time and in less than a # second, so this is "best effort". amphora_driver = utils.get_amphora_driver() timeout_dict = { constants.CONN_MAX_RETRIES: CONF.haproxy_amphora.failover_connection_max_retries, constants.CONN_RETRY_INTERVAL: CONF.haproxy_amphora.failover_connection_retry_interval } amps = [] selected_amp = None for amp in load_balancer.amphorae: if amp.status == constants.DELETED: continue if selected_amp is None: try: if amphora_driver.get_interface_from_ip( amp, load_balancer.vip.ip_address, timeout_dict): # This is a potential ACTIVE, add it to the list amps.append(amp.to_dict()) else: # This one doesn't have the VIP IP, so start # failovers here. selected_amp = amp LOG.debug( "Selected amphora %s as the initial " "failover amphora.", amp.id) except Exception: # This amphora is broken, so start failovers here. selected_amp = amp else: # We have already found a STANDBY, so add the rest to the # list without querying them. amps.append(amp.to_dict()) # Put the selected amphora at the end of the list so it is # first to failover. if selected_amp: amps.append(selected_amp.to_dict()) return amps LOG.error( 'Unknown load balancer topology found: %s, aborting ' 'failover.', load_balancer.topology) raise exceptions.InvalidTopology(topology=load_balancer.topology) def failover_loadbalancer(self, load_balancer_id): """Perform failover operations for a load balancer. Note: This expects the load balancer to already be in provisioning_status=PENDING_UPDATE state. :param load_balancer_id: ID for load balancer to failover :returns: None :raises octavia.commom.exceptions.NotFound: The load balancer was not found. """ try: lb = self._lb_repo.get(db_apis.get_session(), id=load_balancer_id) if lb is None: raise exceptions.NotFound(resource=constants.LOADBALANCER, id=load_balancer_id) # Get the ordered list of amphorae to failover for this LB. amps = self._get_amphorae_for_failover(lb) if lb.topology == constants.TOPOLOGY_SINGLE: if len(amps) != 1: LOG.warning( '%d amphorae found on load balancer %s where ' 'one should exist. Repairing.', len(amps), load_balancer_id) elif lb.topology == constants.TOPOLOGY_ACTIVE_STANDBY: if len(amps) != 2: LOG.warning( '%d amphorae found on load balancer %s where ' 'two should exist. Repairing.', len(amps), load_balancer_id) else: LOG.error( 'Unknown load balancer topology found: %s, aborting ' 'failover!', lb.topology) raise exceptions.InvalidTopology(topology=lb.topology) # We must provide a topology in the flavor definition # here for the amphora to be created with the correct # configuration. if lb.flavor_id: flavor = self._flavor_repo.get_flavor_metadata_dict( db_apis.get_session(), lb.flavor_id) flavor[constants.LOADBALANCER_TOPOLOGY] = lb.topology else: flavor = {constants.LOADBALANCER_TOPOLOGY: lb.topology} provider_lb_dict = ( provider_utils.db_loadbalancer_to_provider_loadbalancer( lb).to_dict() if lb else lb) provider_lb_dict[constants.FLAVOR] = flavor stored_params = { constants.LOADBALANCER: provider_lb_dict, constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_FAILOVER_PRIORITY, constants.SERVER_GROUP_ID: lb.server_group_id, constants.LOADBALANCER_ID: lb.id, constants.FLAVOR: flavor } if lb.availability_zone: stored_params[constants.AVAILABILITY_ZONE] = ( self._az_repo.get_availability_zone_metadata_dict( db_apis.get_session(), lb.availability_zone)) else: stored_params[constants.AVAILABILITY_ZONE] = {} self.run_flow(flow_utils.get_failover_LB_flow, amps, provider_lb_dict, store=stored_params, wait=True) LOG.info('Failover of load balancer %s completed successfully.', lb.id) except Exception as e: with excutils.save_and_reraise_exception(reraise=False): LOG.exception("LB %(lbid)s failover exception: %(exc)s", { 'lbid': load_balancer_id, 'exc': str(e) }) self._lb_repo.update(db_apis.get_session(), load_balancer_id, provisioning_status=constants.ERROR) def amphora_cert_rotation(self, amphora_id): """Perform cert rotation for an amphora. :param amphora_id: ID for amphora to rotate :returns: None :raises AmphoraNotFound: The referenced amphora was not found """ amp = self._amphora_repo.get(db_apis.get_session(), id=amphora_id) LOG.info("Start amphora cert rotation, amphora's id is: %s", amphora_id) store = { constants.AMPHORA: amp.to_dict(), constants.AMPHORA_ID: amphora_id } self.run_flow(flow_utils.cert_rotate_amphora_flow, store=store) LOG.info("Finished amphora cert rotation, amphora's id was: %s", amphora_id) def update_amphora_agent_config(self, amphora_id): """Update the amphora agent configuration. Note: This will update the amphora agent configuration file and update the running configuration for mutatable configuration items. :param amphora_id: ID of the amphora to update. :returns: None """ LOG.info( "Start amphora agent configuration update, amphora's id " "is: %s", amphora_id) amp = self._amphora_repo.get(db_apis.get_session(), id=amphora_id) lb = self._amphora_repo.get_lb_for_amphora(db_apis.get_session(), amphora_id) flavor = {} if lb.flavor_id: flavor = self._flavor_repo.get_flavor_metadata_dict( db_apis.get_session(), lb.flavor_id) store = {constants.AMPHORA: amp.to_dict(), constants.FLAVOR: flavor} self.run_flow(flow_utils.update_amphora_config_flow, store=store) LOG.info( "Finished amphora agent configuration update, amphora's id " "was: %s", amphora_id)
schema_file (str): Path to a file holding a JSONschema document classname (str, optional): Name of the instantiated class to return. Optional if only one class is defined by the schema document. Returns: class: A Python class named after and defined by the schema """ schema_dict = json.load(open(schema_file, 'r')) actual_classname = classname_from_long_title( title_from_schema(schema_dict)) return get_class_object_from_dict(schema_dict, classname=actual_classname, use_cache=use_cache) @retry(retry=retry_if_exception_type(RefResolutionError), stop=(stop_after_delay(15) | stop_after_attempt(5)), wait=wait_random(min=1, max=3), reraise=True) def get_class_object_from_uri(schema_uri, classname=None, use_cache=True): """Instantiate a Python class object from a JSONschema URI Args: schema_uri (str): URI for a JSONschema document classname (str, optional): Name of the instantiated class to return. Optional if only one class is defined by the schema document. Returns: class: A Python class named after and defined by the schema """ resp = requests.get(schema_uri, allow_redirects=True, timeout=1)
return decorator # NOTE I don't like this but slack client is annoying (kglisson) SLACK_GET_ENDPOINTS = [ "conversations.history", "conversations.info", "users.conversations", "users.info", "users.lookupByEmail", "users.profile.get", ] @retry(stop=stop_after_attempt(5), retry=retry_if_exception_type(TryAgain)) def make_call(client: Any, endpoint: str, **kwargs): """Make an slack client api call.""" try: if endpoint in SLACK_GET_ENDPOINTS: response = client.api_call(endpoint, http_verb="GET", params=kwargs) else: response = client.api_call(endpoint, json=kwargs) except slack.errors.SlackApiError as e: log.error(f"SlackError. Response: {e.response} Endpoint: {endpoint} kwargs: {kwargs}") # NOTE we've seen some eventual consistency problems with channel creation if e.response["error"] == "channel_not_found": raise TryAgain
def __init__( self, logger_name: str, max_meter_queue_size: int, current_dir: pathlib.Path, must_exit_after_24h: bool, messaging_thread_provider: typing.Callable[[queue.Queue], MessagingThreadProtocol], meter_values_generator: typing.Callable[[], typing.Generator[int, None, None]], default_action_handler: typing.Callable[[], None] = None, tests_modules_names_provider: typing.Callable[ [], typing.List[str]] = lambda: []): """ Parameters ---------- logger_name : str Name of the logger to be used max_meter_queue_size : int Size of the `queue.Queue` that will hold the Meter's generated values. current_dir : pathlib.Path Refers the parent directory must_exit_after_24h : bool If True, the app will abort its execution after 24h of being running messaging_thread_provider : callable Method to be executed to acquire an instance of a `MessagingThread`. meter_values_generator : generator Generates a value between [0, 9000] everytime it's iterated over. default_action_handler: callable, optional When provided, specifies the default action handler to be used when a specified action is not recognized. tests_modules_names_provider : callable, optional Provides the list of tests modules to run. """ # Maps Command Line Interface allowed actions to actions handlers. self._action_handlers: typing.Dict[str, typing.Callable] = { "help": self._usage_handler, "test": self._run_tests_handler, "start": self._run_meter_simulator_handler } self._log: logging.Logger = logging.getLogger(logger_name) self._log.debug(f"{self.__class__.__name__}.__init__()") self._max_meter_queue_size = max_meter_queue_size self._meter_values_queue: typing.Optional[queue.Queue] = None # Establish how to re-try to add a meter value to the queue when it's full. # Also specifies when it should stop from re-trying. Re-tries would be aborted, if was specified to abort execution # after 24h and they have already lapsed: self._retry_policy = tenacity.Retrying( wait=tenacity.wait_fixed( 0.5 ), # Wait 0.5sec before try again to add the generated meter value to the queue retry=tenacity.retry_if_exception_type(queue.Full), stop=tenacity.stop_any(self._must_exit), after=lambda _, __, ___: self._log.warning( "Meter's values queue is Full!!!")) self._current_dir = current_dir self._must_exit_after_24h: bool = must_exit_after_24h self._start_datetime: datetime = datetime.datetime.now() self._stop_datetime: datetime = self._start_datetime + datetime.timedelta( hours=24) # self._stop_datetime: datetime = self._start_datetime + datetime.timedelta(minutes=2) self._messaging_thread_provider: typing.Callable[[queue.Queue], MessagingThreadProtocol] = \ messaging_thread_provider self._meter_values_generator: typing.Callable[[], typing.Generator[int, None, None]] = \ meter_values_generator self._default_action_handler: typing.Callable[[], None] = \ default_action_handler or self._usage_handler self._tests_modules_names_provider: typing.Callable[[], typing.List[str]] = \ tests_modules_names_provider self._messaging_thread: typing.Optional[MessagingThreadProtocol] = None self._sys_argv: typing.Optional[typing.List[str]] = None
class NeutronNetworkingBase(test_utils.OpenStackBaseTest): """Base for checking openstack instances have valid networking.""" RESOURCE_PREFIX = 'zaza-neutrontests' @classmethod def setUpClass(cls): """Run class setup for running Neutron API Networking tests.""" super(NeutronNetworkingBase, cls).setUpClass( application_name='neutron-api') cls.neutron_client = ( openstack_utils.get_neutron_session_client(cls.keystone_session)) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(8)) def validate_instance_can_reach_other(self, instance_1, instance_2, verify, mtu=None): """ Validate that an instance can reach a fixed and floating of another. :param instance_1: The instance to check networking from :type instance_1: nova_client.Server :param instance_2: The instance to check networking from :type instance_2: nova_client.Server :param verify: callback to verify result :type verify: callable :param mtu: Check that we can send non-fragmented packets of given size :type mtu: Optional[int] """ floating_1 = floating_ips_from_instance(instance_1)[0] floating_2 = floating_ips_from_instance(instance_2)[0] address_2 = fixed_ips_from_instance(instance_2)[0] username = guest.boot_tests['bionic']['username'] password = guest.boot_tests['bionic'].get('password') privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) cmds = [ 'ping -c 1', ] if mtu: # the on-wire packet will be 28 bytes larger than the value # provided to ping(8) -s parameter packetsize = mtu - 28 cmds.append( 'ping -M do -s {} -c 1'.format(packetsize)) for cmd in cmds: openstack_utils.ssh_command( username, floating_1, 'instance-1', '{} {}'.format(cmd, address_2), password=password, privkey=privkey, verify=verify) openstack_utils.ssh_command( username, floating_1, 'instance-1', '{} {}'.format(cmd, floating_2), password=password, privkey=privkey, verify=verify) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(8)) def validate_instance_can_reach_router(self, instance, verify, mtu=None): """ Validate that an instance can reach it's primary gateway. We make the assumption that the router's IP is 192.168.0.1 as that's the network that is setup in neutron.setup.basic_overcloud_network which is used in all Zaza Neutron validations. :param instance: The instance to check networking from :type instance: nova_client.Server :param verify: callback to verify result :type verify: callable :param mtu: Check that we can send non-fragmented packets of given size :type mtu: Optional[int] """ address = floating_ips_from_instance(instance)[0] username = guest.boot_tests['bionic']['username'] password = guest.boot_tests['bionic'].get('password') privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) cmds = [ 'ping -c 1', ] if mtu: # the on-wire packet will be 28 bytes larger than the value # provided to ping(8) -s parameter packetsize = mtu - 28 cmds.append( 'ping -M do -s {} -c 1'.format(packetsize)) for cmd in cmds: openstack_utils.ssh_command( username, address, 'instance', '{} 192.168.0.1'.format(cmd), password=password, privkey=privkey, verify=verify) @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60), reraise=True, stop=tenacity.stop_after_attempt(8), retry=tenacity.retry_if_exception_type(AssertionError)) def check_server_state(self, nova_client, state, server_id=None, server_name=None): """Wait for server to reach desired state. :param nova_client: Nova client to use when checking status :type nova_client: nova client :param state: Target state for server :type state: str :param server_id: UUID of server to check :type server_id: str :param server_name: Name of server to check :type server_name: str :raises: AssertionError """ if server_name: server_id = nova_client.servers.find(name=server_name).id server = nova_client.servers.find(id=server_id) assert server.status == state @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60), reraise=True, stop=tenacity.stop_after_attempt(8), retry=tenacity.retry_if_exception_type(AssertionError)) def check_neutron_agent_up(self, neutron_client, host_name): """Wait for agents to come up. :param neutron_client: Neutron client to use when checking status :type neutron_client: neutron client :param host_name: The name of the host whose agents need checking :type host_name: str :raises: AssertionError """ for agent in neutron_client.list_agents()['agents']: if agent['host'] == host_name: assert agent['admin_state_up'] assert agent['alive'] def effective_network_mtu(self, network_name): """Retrieve effective MTU for a network. If the `instance-mtu` configuration option is set to a value lower than the network MTU this method will return the value of that. Otherwise Neutron's value for MTU on a network will be returned. :param network_name: Name of network to query :type network_name: str :returns: MTU for network :rtype: int """ cfg_instance_mtu = None for app in ('neutron-gateway', 'neutron-openvswitch'): try: cfg = zaza.model.get_application_config(app) cfg_instance_mtu = int(cfg['instance-mtu']['value']) break except KeyError: pass networks = self.neutron_client.show_network('', name=network_name) network_mtu = int(next(iter(networks['networks']))['mtu']) if cfg_instance_mtu and cfg_instance_mtu < network_mtu: logging.info('Using MTU from application "{}" config: {}' .format(app, cfg_instance_mtu)) return cfg_instance_mtu else: logging.info('Using MTU from network "{}": {}' .format(network_name, network_mtu)) return network_mtu def check_connectivity(self, instance_1, instance_2): """Run North/South and East/West connectivity tests.""" def verify(stdin, stdout, stderr): """Validate that the SSH command exited 0.""" self.assertEqual(stdout.channel.recv_exit_status(), 0) try: mtu_1 = self.effective_network_mtu( network_name_from_instance(instance_1)) mtu_2 = self.effective_network_mtu( network_name_from_instance(instance_2)) mtu_min = min(mtu_1, mtu_2) except neutronexceptions.NotFound: # Older versions of OpenStack cannot look up network by name, just # skip the check if that is the case. mtu_1 = mtu_2 = mtu_min = None # Verify network from 1 to 2 self.validate_instance_can_reach_other( instance_1, instance_2, verify, mtu_min) # Verify network from 2 to 1 self.validate_instance_can_reach_other( instance_2, instance_1, verify, mtu_min) # Validate tenant to external network routing self.validate_instance_can_reach_router(instance_1, verify, mtu_1) self.validate_instance_can_reach_router(instance_2, verify, mtu_2)
def request(self, method, additional_headers=None, retry=True, timeout=None, auth=None, use_gzip_encoding=None, params=None, max_attempts=None, **kwargs): """ Make an HTTP request by calling self._request with backoff retry. :param method: request method :type method: str :param additional_headers: additional headers to include in the request :type additional_headers: dict[str, str] :param retry: boolean indicating whether to retry if the request fails :type retry: boolean :param timeout: timeout in seconds, overrides default_timeout_secs :type timeout: float :param timeout: timeout in seconds :type timeout: float :param auth: auth scheme for the request :type auth: requests.auth.AuthBase :param use_gzip_encoding: boolean indicating whether to pass gzip encoding in the request headers or not :type use_gzip_encoding: boolean | None :param params: additional params to include in the request :type params: str | dict[str, T] | None :param max_attempts: maximum number of attempts to try for any request :type max_attempts: int :param kwargs: additional arguments to pass to requests.request :type kwargs: dict[str, T] :return: HTTP response :rtype: requests.Response """ request = self._request if retry: if max_attempts is None: max_attempts = self.default_max_attempts # We retry only when it makes sense: either due to a network # partition (e.g. connection errors) or if the request failed # due to a server error such as 500s, timeouts, and so on. request = tenacity.retry( stop=tenacity.stop_after_attempt(max_attempts), wait=tenacity.wait_exponential(), retry=tenacity.retry_if_exception_type(( requests.exceptions.Timeout, requests.exceptions.ConnectionError, MesosServiceUnavailableException, MesosInternalServerErrorException, )), reraise=True, )(request) try: return request( method=method, additional_headers=additional_headers, timeout=timeout, auth=auth, use_gzip_encoding=use_gzip_encoding, params=params, **kwargs ) # If the request itself failed, an exception subclassed from # RequestException will be raised. Catch this and reraise as # MesosException since we want the caller to be able to catch # and handle this. except requests.exceptions.RequestException as err: raise MesosException('Request failed', err)
class Mega: def __init__(self, options=None): self.schema = 'https' self.domain = 'mega.co.nz' self.timeout = 160 # max secs to wait for resp from api requests self.sid = None self.sequence_num = random.randint(0, 0xFFFFFFFF) self.request_id = make_id(10) self._trash_folder_node_id = None if options is None: options = {} self.options = options def login(self, email=None, password=None): if email: self._login_user(email, password) else: self.login_anonymous() self._trash_folder_node_id = self.get_node_by_type(4)[0] logger.info('Login complete') return self def _login_user(self, email, password): logger.info('Logging in user...') email = email.lower() get_user_salt_resp = self._api_request({'a': 'us0', 'user': email}) user_salt = None try: user_salt = base64_to_a32(get_user_salt_resp['s']) except KeyError: # v1 user account password_aes = prepare_key(str_to_a32(password)) user_hash = stringhash(email, password_aes) else: # v2 user account pbkdf2_key = hashlib.pbkdf2_hmac(hash_name='sha512', password=password.encode(), salt=a32_to_str(user_salt), iterations=100000, dklen=32) password_aes = str_to_a32(pbkdf2_key[:16]) user_hash = base64_url_encode(pbkdf2_key[-16:]) resp = self._api_request({'a': 'us', 'user': email, 'uh': user_hash}) if isinstance(resp, int): raise RequestError(resp) self._login_process(resp, password_aes) def login_anonymous(self): logger.info('Logging in anonymous temporary user...') master_key = [random.randint(0, 0xFFFFFFFF)] * 4 password_key = [random.randint(0, 0xFFFFFFFF)] * 4 session_self_challenge = [random.randint(0, 0xFFFFFFFF)] * 4 user = self._api_request({ 'a': 'up', 'k': a32_to_base64(encrypt_key(master_key, password_key)), 'ts': base64_url_encode( a32_to_str(session_self_challenge) + a32_to_str(encrypt_key(session_self_challenge, master_key))) }) resp = self._api_request({'a': 'us', 'user': user}) if isinstance(resp, int): raise RequestError(resp) self._login_process(resp, password_key) def _login_process(self, resp, password): encrypted_master_key = base64_to_a32(resp['k']) self.master_key = decrypt_key(encrypted_master_key, password) if 'tsid' in resp: tsid = base64_url_decode(resp['tsid']) key_encrypted = a32_to_str( encrypt_key(str_to_a32(tsid[:16]), self.master_key)) if key_encrypted == tsid[-16:]: self.sid = resp['tsid'] elif 'csid' in resp: encrypted_rsa_private_key = base64_to_a32(resp['privk']) rsa_private_key = decrypt_key(encrypted_rsa_private_key, self.master_key) private_key = a32_to_str(rsa_private_key) # The private_key contains 4 MPI integers concatenated together. rsa_private_key = [0, 0, 0, 0] for i in range(4): # An MPI integer has a 2-byte header which describes the number # of bits in the integer. bitlength = (private_key[0] * 256) + private_key[1] bytelength = math.ceil(bitlength / 8) # Add 2 bytes to accommodate the MPI header bytelength += 2 rsa_private_key[i] = mpi_to_int(private_key[:bytelength]) private_key = private_key[bytelength:] first_factor_p = rsa_private_key[0] second_factor_q = rsa_private_key[1] private_exponent_d = rsa_private_key[2] # In MEGA's webclient javascript, they assign [3] to a variable # called u, but I do not see how it corresponds to pycryptodome's # RSA.construct and it does not seem to be necessary. rsa_modulus_n = first_factor_p * second_factor_q phi = (first_factor_p - 1) * (second_factor_q - 1) public_exponent_e = modular_inverse(private_exponent_d, phi) rsa_components = ( rsa_modulus_n, public_exponent_e, private_exponent_d, first_factor_p, second_factor_q, ) rsa_decrypter = RSA.construct(rsa_components) encrypted_sid = mpi_to_int(base64_url_decode(resp['csid'])) sid = '%x' % rsa_decrypter._decrypt(encrypted_sid) sid = binascii.unhexlify('0' + sid if len(sid) % 2 else sid) self.sid = base64_url_encode(sid[:43]) @retry(retry=retry_if_exception_type(RuntimeError), wait=wait_exponential(multiplier=2, min=2, max=60)) def _api_request(self, data): params = {'id': self.sequence_num} self.sequence_num += 1 if self.sid: params.update({'sid': self.sid}) # ensure input data is a list if not isinstance(data, list): data = [data] url = f'{self.schema}://g.api.{self.domain}/cs' req = requests.post( url, params=params, data=json.dumps(data), timeout=self.timeout, ) json_resp = json.loads(req.text) if isinstance(json_resp, int): if json_resp == -3: msg = 'Request failed, retrying' logger.info(msg) raise RuntimeError(msg) raise RequestError(json_resp) return json_resp[0] def _parse_url(self, url): # parse file id and key from url if '!' in url: match = re.findall(r'/#!(.*)', url) path = match[0] return path else: raise RequestError('Url key missing') def _process_file(self, file, shared_keys): if file['t'] == 0 or file['t'] == 1: keys = dict( keypart.split(':', 1) for keypart in file['k'].split('/') if ':' in keypart) uid = file['u'] key = None # my objects if uid in keys: key = decrypt_key(base64_to_a32(keys[uid]), self.master_key) # shared folders elif 'su' in file and 'sk' in file and ':' in file['k']: shared_key = decrypt_key(base64_to_a32(file['sk']), self.master_key) key = decrypt_key(base64_to_a32(keys[file['h']]), shared_key) if file['su'] not in shared_keys: shared_keys[file['su']] = {} shared_keys[file['su']][file['h']] = shared_key # shared files elif file['u'] and file['u'] in shared_keys: for hkey in shared_keys[file['u']]: shared_key = shared_keys[file['u']][hkey] if hkey in keys: key = keys[hkey] key = decrypt_key(base64_to_a32(key), shared_key) break if file['h'] and file['h'] in shared_keys.get('EXP', ()): shared_key = shared_keys['EXP'][file['h']] encrypted_key = str_to_a32( base64_url_decode(file['k'].split(':')[-1])) key = decrypt_key(encrypted_key, shared_key) file['shared_folder_key'] = shared_key if key is not None: # file if file['t'] == 0: k = (key[0] ^ key[4], key[1] ^ key[5], key[2] ^ key[6], key[3] ^ key[7]) file['iv'] = key[4:6] + (0, 0) file['meta_mac'] = key[6:8] # folder else: k = key file['key'] = key file['k'] = k attributes = base64_url_decode(file['a']) attributes = decrypt_attr(attributes, k) file['a'] = attributes # other => wrong object elif file['k'] == '': file['a'] = False elif file['t'] == 2: self.root_id = file['h'] file['a'] = {'n': 'Cloud Drive'} elif file['t'] == 3: self.inbox_id = file['h'] file['a'] = {'n': 'Inbox'} elif file['t'] == 4: self.trashbin_id = file['h'] file['a'] = {'n': 'Rubbish Bin'} return file def _init_shared_keys(self, files, shared_keys): """ Init shared key not associated with a user. Seems to happen when a folder is shared, some files are exchanged and then the folder is un-shared. Keys are stored in files['s'] and files['ok'] """ ok_dict = {} for ok_item in files['ok']: shared_key = decrypt_key(base64_to_a32(ok_item['k']), self.master_key) ok_dict[ok_item['h']] = shared_key for s_item in files['s']: if s_item['u'] not in shared_keys: shared_keys[s_item['u']] = {} if s_item['h'] in ok_dict: shared_keys[s_item['u']][s_item['h']] = ok_dict[s_item['h']] self.shared_keys = shared_keys def find_path_descriptor(self, path, files=()): """ Find descriptor of folder inside a path. i.e.: folder1/folder2/folder3 Params: path, string like folder1/folder2/folder3 Return: Descriptor (str) of folder3 if exists, None otherwise """ paths = path.split('/') files = files or self.get_files() parent_desc = self.root_id found = False for foldername in paths: if foldername != '': for file in files.items(): if (file[1]['a'] and file[1]['t'] and file[1]['a']['n'] == foldername): if parent_desc == file[1]['p']: parent_desc = file[0] found = True if found: found = False else: return None return parent_desc def find(self, filename=None, handle=None, exclude_deleted=False): """ Return file object from given filename """ files = self.get_files() if handle: return files[handle] path = Path(filename) filename = path.name parent_dir_name = path.parent.name for file in list(files.items()): parent_node_id = None if parent_dir_name: parent_node_id = self.find_path_descriptor(parent_dir_name, files=files) if (filename and parent_node_id and file[1]['a'] and file[1]['a']['n'] == filename and parent_node_id == file[1]['p']): if (exclude_deleted and self._trash_folder_node_id == file[1]['p']): continue return file if (filename and file[1]['a'] and file[1]['a']['n'] == filename): if (exclude_deleted and self._trash_folder_node_id == file[1]['p']): continue return file def get_files(self): logger.info('Getting all files...') files = self._api_request({'a': 'f', 'c': 1, 'r': 1}) files_dict = {} shared_keys = {} self._init_shared_keys(files, shared_keys) for file in files['f']: processed_file = self._process_file(file, shared_keys) # ensure each file has a name before returning if processed_file['a']: files_dict[file['h']] = processed_file return files_dict def get_upload_link(self, file): """ Get a files public link inc. decrypted key Requires upload() response as input """ if 'f' in file: file = file['f'][0] public_handle = self._api_request({'a': 'l', 'n': file['h']}) file_key = file['k'][file['k'].index(':') + 1:] decrypted_key = a32_to_base64( decrypt_key(base64_to_a32(file_key), self.master_key)) return (f'{self.schema}://{self.domain}' f'/#!{public_handle}!{decrypted_key}') else: raise ValueError('''Upload() response required as input, use get_link() for regular file input''') def get_link(self, file): """ Get a file public link from given file object """ file = file[1] if 'h' in file and 'k' in file: public_handle = self._api_request({'a': 'l', 'n': file['h']}) if public_handle == -11: raise RequestError("Can't get a public link from that file " "(is this a shared file?)") decrypted_key = a32_to_base64(file['key']) return (f'{self.schema}://{self.domain}' f'/#!{public_handle}!{decrypted_key}') else: raise ValidationError('File id and key must be present') def _node_data(self, node): try: return node[1] except (IndexError, KeyError): return node def get_folder_link(self, file): try: file = file[1] except (IndexError, KeyError): pass if 'h' in file and 'k' in file: public_handle = self._api_request({'a': 'l', 'n': file['h']}) if public_handle == -11: raise RequestError("Can't get a public link from that file " "(is this a shared file?)") decrypted_key = a32_to_base64(file['shared_folder_key']) return (f'{self.schema}://{self.domain}' f'/#F!{public_handle}!{decrypted_key}') else: raise ValidationError('File id and key must be present') def get_user(self): user_data = self._api_request({'a': 'ug'}) return user_data def get_node_by_type(self, type): """ Get a node by it's numeric type id, e.g: 0: file 1: dir 2: special: root cloud drive 3: special: inbox 4: special trash bin """ nodes = self.get_files() for node in list(nodes.items()): if node[1]['t'] == type: return node def get_files_in_node(self, target): """ Get all files in a given target, e.g. 4=trash """ if type(target) == int: # convert special nodes (e.g. trash) node_id = self.get_node_by_type(target) else: node_id = [target] files = self._api_request({'a': 'f', 'c': 1}) files_dict = {} shared_keys = {} self._init_shared_keys(files, shared_keys) for file in files['f']: processed_file = self._process_file(file, shared_keys) if processed_file['a'] and processed_file['p'] == node_id[0]: files_dict[file['h']] = processed_file return files_dict def get_id_from_public_handle(self, public_handle): # get node data node_data = self._api_request({'a': 'f', 'f': 1, 'p': public_handle}) node_id = self.get_id_from_obj(node_data) return node_id def get_id_from_obj(self, node_data): """ Get node id from a file object """ node_id = None for i in node_data['f']: if i['h'] != '': node_id = i['h'] return node_id def get_quota(self): """ Get current remaining disk quota in MegaBytes """ json_resp = self._api_request({ 'a': 'uq', 'xfer': 1, 'strg': 1, 'v': 1 }) # convert bytes to megabyes return json_resp['mstrg'] / 1048576 def get_storage_space(self, giga=False, mega=False, kilo=False): """ Get the current storage space. Return a dict containing at least: 'used' : the used space on the account 'total' : the maximum space allowed with current plan All storage space are in bytes unless asked differently. """ if sum(1 if x else 0 for x in (kilo, mega, giga)) > 1: raise ValueError("Only one unit prefix can be specified") unit_coef = 1 if kilo: unit_coef = 1024 if mega: unit_coef = 1048576 if giga: unit_coef = 1073741824 json_resp = self._api_request({'a': 'uq', 'xfer': 1, 'strg': 1}) return { 'used': json_resp['cstrg'] / unit_coef, 'total': json_resp['mstrg'] / unit_coef, } def get_balance(self): """ Get account monetary balance, Pro accounts only """ user_data = self._api_request({"a": "uq", "pro": 1}) if 'balance' in user_data: return user_data['balance'] def delete(self, public_handle): """ Delete a file by its public handle """ return self.move(public_handle, 4) def delete_url(self, url): """ Delete a file by its url """ path = self._parse_url(url).split('!') public_handle = path[0] file_id = self.get_id_from_public_handle(public_handle) return self.move(file_id, 4) def destroy(self, file_id): """ Destroy a file by its private id """ return self._api_request({ 'a': 'd', 'n': file_id, 'i': self.request_id }) def destroy_url(self, url): """ Destroy a file by its url """ path = self._parse_url(url).split('!') public_handle = path[0] file_id = self.get_id_from_public_handle(public_handle) return self.destroy(file_id) def empty_trash(self): # get list of files in rubbish out files = self.get_files_in_node(4) # make a list of json if files != {}: post_list = [] for file in files: post_list.append({"a": "d", "n": file, "i": self.request_id}) return self._api_request(post_list) def download(self, file, dest_path=None, dest_filename=None): """ Download a file by it's file object """ return self._download_file(file_handle=None, file_key=None, file=file[1], dest_path=dest_path, dest_filename=dest_filename, is_public=False) def _export_file(self, node): node_data = self._node_data(node) self._api_request([{ 'a': 'l', 'n': node_data['h'], 'i': self.request_id }]) return self.get_link(node) def export(self, path=None, node_id=None): nodes = self.get_files() if node_id: node = nodes[node_id] else: node = self.find(path) node_data = self._node_data(node) is_file_node = node_data['t'] == 0 if is_file_node: return self._export_file(node) if node: try: # If already exported return self.get_folder_link(node) except (RequestError, KeyError): pass master_key_cipher = AES.new(a32_to_str(self.master_key), AES.MODE_ECB) ha = base64_url_encode( master_key_cipher.encrypt(node_data['h'].encode("utf8") + node_data['h'].encode("utf8"))) share_key = secrets.token_bytes(16) ok = base64_url_encode(master_key_cipher.encrypt(share_key)) share_key_cipher = AES.new(share_key, AES.MODE_ECB) node_key = node_data['k'] encrypted_node_key = base64_url_encode( share_key_cipher.encrypt(a32_to_str(node_key))) node_id = node_data['h'] request_body = [{ 'a': 's2', 'n': node_id, 's': [{ 'u': 'EXP', 'r': 0 }], 'i': self.request_id, 'ok': ok, 'ha': ha, 'cr': [[node_id], [node_id], [0, 0, encrypted_node_key]] }] self._api_request(request_body) nodes = self.get_files() return self.get_folder_link(nodes[node_id]) def download_url(self, url, dest_path=None, dest_filename=None): """ Download a file by it's public url """ path = self._parse_url(url).split('!') file_id = path[0] file_key = path[1] return self._download_file( file_handle=file_id, file_key=file_key, dest_path=dest_path, dest_filename=dest_filename, is_public=True, ) def _download_file(self, file_handle, file_key, dest_path=None, dest_filename=None, is_public=False, file=None): if file is None: if is_public: file_key = base64_to_a32(file_key) file_data = self._api_request({ 'a': 'g', 'g': 1, 'p': file_handle }) else: file_data = self._api_request({ 'a': 'g', 'g': 1, 'n': file_handle }) k = (file_key[0] ^ file_key[4], file_key[1] ^ file_key[5], file_key[2] ^ file_key[6], file_key[3] ^ file_key[7]) iv = file_key[4:6] + (0, 0) meta_mac = file_key[6:8] else: file_data = self._api_request({'a': 'g', 'g': 1, 'n': file['h']}) k = file['k'] iv = file['iv'] meta_mac = file['meta_mac'] # Seems to happens sometime... When this occurs, files are # inaccessible also in the official also in the official web app. # Strangely, files can come back later. if 'g' not in file_data: raise RequestError('File not accessible anymore') file_url = file_data['g'] file_size = file_data['s'] attribs = base64_url_decode(file_data['at']) attribs = decrypt_attr(attribs, k) if dest_filename is not None: file_name = dest_filename else: file_name = attribs['n'] input_file = requests.get(file_url, stream=True).raw if dest_path is None: dest_path = '' else: dest_path += '/' with tempfile.NamedTemporaryFile(mode='w+b', prefix='megapy_', delete=False) as temp_output_file: k_str = a32_to_str(k) counter = Counter.new(128, initial_value=((iv[0] << 32) + iv[1]) << 64) aes = AES.new(k_str, AES.MODE_CTR, counter=counter) mac_str = '\0' * 16 mac_encryptor = AES.new(k_str, AES.MODE_CBC, mac_str.encode("utf8")) iv_str = a32_to_str([iv[0], iv[1], iv[0], iv[1]]) for chunk_start, chunk_size in get_chunks(file_size): chunk = input_file.read(chunk_size) chunk = aes.decrypt(chunk) temp_output_file.write(chunk) encryptor = AES.new(k_str, AES.MODE_CBC, iv_str) for i in range(0, len(chunk) - 16, 16): block = chunk[i:i + 16] encryptor.encrypt(block) # fix for files under 16 bytes failing if file_size > 16: i += 16 else: i = 0 block = chunk[i:i + 16] if len(block) % 16: block += b'\0' * (16 - (len(block) % 16)) mac_str = mac_encryptor.encrypt(encryptor.encrypt(block)) file_info = os.stat(temp_output_file.name) logger.info('%s of %s downloaded', file_info.st_size, file_size) file_mac = str_to_a32(mac_str) # check mac integrity if (file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]) != meta_mac: raise ValueError('Mismatched mac') output_path = Path(dest_path + file_name) shutil.move(temp_output_file.name, output_path) return output_path def upload(self, filename, dest=None, dest_filename=None): # determine storage node if dest is None: # if none set, upload to cloud drive node if not hasattr(self, 'root_id'): self.get_files() dest = self.root_id # request upload url, call 'u' method with open(filename, 'rb') as input_file: file_size = os.path.getsize(filename) ul_url = self._api_request({'a': 'u', 's': file_size})['p'] # generate random aes key (128) for file ul_key = [random.randint(0, 0xFFFFFFFF) for _ in range(6)] k_str = a32_to_str(ul_key[:4]) count = Counter.new( 128, initial_value=((ul_key[4] << 32) + ul_key[5]) << 64) aes = AES.new(k_str, AES.MODE_CTR, counter=count) upload_progress = 0 completion_file_handle = None mac_str = '\0' * 16 mac_encryptor = AES.new(k_str, AES.MODE_CBC, mac_str.encode("utf8")) iv_str = a32_to_str([ul_key[4], ul_key[5], ul_key[4], ul_key[5]]) if file_size > 0: for chunk_start, chunk_size in get_chunks(file_size): chunk = input_file.read(chunk_size) upload_progress += len(chunk) encryptor = AES.new(k_str, AES.MODE_CBC, iv_str) for i in range(0, len(chunk) - 16, 16): block = chunk[i:i + 16] encryptor.encrypt(block) # fix for files under 16 bytes failing if file_size > 16: i += 16 else: i = 0 block = chunk[i:i + 16] if len(block) % 16: block += makebyte('\0' * (16 - len(block) % 16)) mac_str = mac_encryptor.encrypt(encryptor.encrypt(block)) # encrypt file and upload chunk = aes.encrypt(chunk) output_file = requests.post(ul_url + "/" + str(chunk_start), data=chunk, timeout=self.timeout) completion_file_handle = output_file.text logger.info('%s of %s uploaded', upload_progress, file_size) else: output_file = requests.post(ul_url + "/0", data='', timeout=self.timeout) completion_file_handle = output_file.text logger.info('Chunks uploaded') logger.info('Setting attributes to complete upload') logger.info('Computing attributes') file_mac = str_to_a32(mac_str) # determine meta mac meta_mac = (file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3]) dest_filename = dest_filename or os.path.basename(filename) attribs = {'n': dest_filename} encrypt_attribs = base64_url_encode( encrypt_attr(attribs, ul_key[:4])) key = [ ul_key[0] ^ ul_key[4], ul_key[1] ^ ul_key[5], ul_key[2] ^ meta_mac[0], ul_key[3] ^ meta_mac[1], ul_key[4], ul_key[5], meta_mac[0], meta_mac[1] ] encrypted_key = a32_to_base64(encrypt_key(key, self.master_key)) logger.info('Sending request to update attributes') # update attributes data = self._api_request({ 'a': 'p', 't': dest, 'i': self.request_id, 'n': [{ 'h': completion_file_handle, 't': 0, 'a': encrypt_attribs, 'k': encrypted_key }] }) logger.info('Upload complete') return data def _mkdir(self, name, parent_node_id): # generate random aes key (128) for folder ul_key = [random.randint(0, 0xFFFFFFFF) for _ in range(6)] # encrypt attribs attribs = {'n': name} encrypt_attribs = base64_url_encode(encrypt_attr(attribs, ul_key[:4])) encrypted_key = a32_to_base64(encrypt_key(ul_key[:4], self.master_key)) # update attributes data = self._api_request({ 'a': 'p', 't': parent_node_id, 'n': [{ 'h': 'xxxxxxxx', 't': 1, 'a': encrypt_attribs, 'k': encrypted_key }], 'i': self.request_id }) return data def _root_node_id(self): if not hasattr(self, 'root_id'): self.get_files() return self.root_id def create_folder(self, name, dest=None): dirs = tuple(dir_name for dir_name in str(name).split('/') if dir_name) folder_node_ids = {} for idx, directory_name in enumerate(dirs): existing_node_id = self.find_path_descriptor(directory_name) if existing_node_id: folder_node_ids[idx] = existing_node_id continue if idx == 0: if dest is None: parent_node_id = self._root_node_id() else: parent_node_id = dest else: parent_node_id = folder_node_ids[idx - 1] created_node = self._mkdir(name=directory_name, parent_node_id=parent_node_id) node_id = created_node['f'][0]['h'] folder_node_ids[idx] = node_id return dict(zip(dirs, folder_node_ids.values())) def rename(self, file, new_name): file = file[1] # create new attribs attribs = {'n': new_name} # encrypt attribs encrypt_attribs = base64_url_encode(encrypt_attr(attribs, file['k'])) encrypted_key = a32_to_base64(encrypt_key(file['key'], self.master_key)) # update attributes return self._api_request([{ 'a': 'a', 'attr': encrypt_attribs, 'key': encrypted_key, 'n': file['h'], 'i': self.request_id }]) def move(self, file_id, target): """ Move a file to another parent node params: a : command n : node we're moving t : id of target parent node, moving to i : request id targets 2 : root 3 : inbox 4 : trash or... target's id or... target's structure returned by find() """ # determine target_node_id if type(target) == int: target_node_id = str(self.get_node_by_type(target)[0]) elif type(target) in (str, ): target_node_id = target else: file = target[1] target_node_id = file['h'] return self._api_request({ 'a': 'm', 'n': file_id, 't': target_node_id, 'i': self.request_id }) def add_contact(self, email): """ Add another user to your mega contact list """ return self._edit_contact(email, True) def remove_contact(self, email): """ Remove a user to your mega contact list """ return self._edit_contact(email, False) def _edit_contact(self, email, add): """ Editing contacts """ if add is True: l = '1' # add command elif add is False: l = '0' # remove command else: raise ValidationError('add parameter must be of type bool') if not re.match(r"[^@]+@[^@]+\.[^@]+", email): ValidationError('add_contact requires a valid email address') else: return self._api_request({ 'a': 'ur', 'u': email, 'l': l, 'i': self.request_id }) def get_public_url_info(self, url): """ Get size and name from a public url, dict returned """ file_handle, file_key = self._parse_url(url).split('!') return self.get_public_file_info(file_handle, file_key) def import_public_url(self, url, dest_node=None, dest_name=None): """ Import the public url into user account """ file_handle, file_key = self._parse_url(url).split('!') return self.import_public_file(file_handle, file_key, dest_node=dest_node, dest_name=dest_name) def get_public_file_info(self, file_handle, file_key): """ Get size and name of a public file """ data = self._api_request({'a': 'g', 'p': file_handle, 'ssm': 1}) if isinstance(data, int): raise RequestError(data) if 'at' not in data or 's' not in data: raise ValueError("Unexpected result", data) key = base64_to_a32(file_key) k = (key[0] ^ key[4], key[1] ^ key[5], key[2] ^ key[6], key[3] ^ key[7]) size = data['s'] unencrypted_attrs = decrypt_attr(base64_url_decode(data['at']), k) if not unencrypted_attrs: return None result = {'size': size, 'name': unencrypted_attrs['n']} return result def import_public_file(self, file_handle, file_key, dest_node=None, dest_name=None): """ Import the public file into user account """ # Providing dest_node spare an API call to retrieve it. if dest_node is None: # Get '/Cloud Drive' folder no dest node specified dest_node = self.get_node_by_type(2)[1] # Providing dest_name spares an API call to retrieve it. if dest_name is None: pl_info = self.get_public_file_info(file_handle, file_key) dest_name = pl_info['name'] key = base64_to_a32(file_key) k = (key[0] ^ key[4], key[1] ^ key[5], key[2] ^ key[6], key[3] ^ key[7]) encrypted_key = a32_to_base64(encrypt_key(key, self.master_key)) encrypted_name = base64_url_encode(encrypt_attr({'n': dest_name}, k)) return self._api_request({ 'a': 'p', 't': dest_node['h'], 'n': [{ 'ph': file_handle, 't': 0, 'a': encrypted_name, 'k': encrypted_key }] })
import os import logging import mysql.connector import postgresql import postgresql.driver.dbapi20 as pgdb import time from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_random_exponential logging.basicConfig(level=logging.DEBUG) @retry(retry=retry_if_exception_type(mysql.connector.errors.InterfaceError), stop=stop_after_attempt(10), wait=wait_random_exponential(multiplier=1, max=10)) def get_mysql_connection(host: str, user: str, password: str, database: str): logging.debug( f'Getting bigbluebutton-auto-approver MySQL connection on host={host} database={database} user={user} password={password}...' ) db = mysql.connector.connect(host=host, user=user, passwd=password, db=database) logging.debug(f'Got bigbluebutton-auto-approver MySQL connection: {db}') return db def get_postgres_connection(host: str, user: str, password: str, database: str): logging.debug( f'Getting Greenlight PostgreSQL connection on host={host} database={database} user={user} password={password}...' )
def retry_on_stale_data_error(func): wrapper = tenacity.retry( stop=tenacity.stop_after_attempt(3), retry=tenacity.retry_if_exception_type(exc.StaleDataError), reraise=True) return wrapper(func)
class MasakariTest(test_utils.OpenStackBaseTest): """Encapsulate Masakari tests.""" @classmethod def setUpClass(cls): """Run class setup for running tests.""" super(MasakariTest, cls).setUpClass(application_name="masakari") cls.current_release = openstack_utils.get_os_release() cls.keystone_session = openstack_utils.get_overcloud_keystone_session() cls.model_name = zaza.model.get_juju_model() cls.nova_client = openstack_utils.get_nova_session_client( cls.keystone_session) @classmethod def tearDown(cls): """Bring hypervisors and services back up.""" logging.info('Running teardown') for unit in zaza.model.get_units('nova-compute', model_name=cls.model_name): zaza.openstack.configure.masakari.simulate_compute_host_recovery( unit.entity_id, model_name=cls.model_name) openstack_utils.enable_all_nova_services(cls.nova_client) zaza.openstack.configure.masakari.enable_hosts() def ensure_guest(self, vm_name): """Return the existing guest or boot a new one. :param vm_name: Name of guest to lookup :type vm_name: str :returns: Guest matching name. :rtype: novaclient.v2.servers.Server """ try: guest = self.nova_client.servers.find(name=vm_name) logging.info('Found existing guest') except novaclient.exceptions.NotFound: logging.info('Launching new guest') guest = zaza.openstack.configure.guest.launch_instance( 'bionic', use_boot_volume=True, meta={'HA_Enabled': 'True'}, vm_name=vm_name) return guest def get_guests_compute_info(self, vm_name): """Return the hostname & juju unit of compute host hosting vm. :param vm_name: Name of guest to lookup :type vm_name: str :returns: Hypervisor name and juju unit name :rtype: (str, str) """ current_hypervisor = openstack_utils.get_hypervisor_for_guest( self.nova_client, vm_name) unit_name = juju_utils.get_unit_name_from_host_name( current_hypervisor, 'nova-compute') return current_hypervisor, unit_name def get_guest_qemu_pid(self, compute_unit_name, vm_uuid, model_name=None): """Return the qemu pid of process running guest. :param compute_unit_name: Juju unit name of hypervisor running guest :type compute_unit_name: str :param vm_uuid: Guests UUID :type vm_uuid: str :param model_name: Name of model running cloud. :type model_name: str :returns: PID of qemu process :rtype: int :raises: ValueError """ pid_find_cmd = 'pgrep -u libvirt-qemu -f {}'.format(vm_uuid) out = zaza.model.run_on_unit(compute_unit_name, pid_find_cmd, model_name=self.model_name) return int(out['Stdout'].strip()) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), reraise=True, stop=tenacity.stop_after_attempt(5), retry=tenacity.retry_if_exception_type(ValueError)) def wait_for_guest_pid(self, compute_unit_name, vm_uuid, model_name=None): """Wait for the qemu process running guest to appear & return its pid. :param compute_unit_name: Juju unit name of hypervisor running guest :type compute_unit_name: str :param vm_uuid: Guests UUID :type vm_uuid: str :param model_name: Name of model running cloud. :type model_name: str :returns: PID of qemu process :rtype: int :raises: ValueError """ return self.get_guest_qemu_pid(compute_unit_name, vm_uuid, model_name=self.model_name) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), reraise=True, stop=tenacity.stop_after_attempt(5), retry=tenacity.retry_if_exception_type(AssertionError)) def wait_for_guest_ready(self, vm_name): """Wait for the guest to be ready. :param vm_name: Name of guest to check. :type vm_name: str """ guest_ready_attr_checks = [('OS-EXT-STS:task_state', None), ('status', 'ACTIVE'), ('OS-EXT-STS:power_state', 1), ('OS-EXT-STS:vm_state', 'active')] guest = self.nova_client.servers.find(name=vm_name) logging.info('Checking guest {} attributes'.format(vm_name)) for (attr, required_state) in guest_ready_attr_checks: logging.info('Checking {} is {}'.format(attr, required_state)) assert getattr(guest, attr) == required_state def test_instance_failover(self): """Test masakari managed guest migration.""" # Workaround for Bug #1874719 zaza.openstack.configure.hacluster.remove_node('masakari', 'node1') # Launch guest self.assertTrue( zaza.openstack.configure.hacluster.check_all_nodes_online( 'masakari')) vm_name = 'zaza-test-instance-failover' self.ensure_guest(vm_name) # Locate hypervisor hosting guest and shut it down current_hypervisor, unit_name = self.get_guests_compute_info(vm_name) zaza.openstack.configure.masakari.simulate_compute_host_failure( unit_name, model_name=self.model_name) # Wait for instance move logging.info('Waiting for guest to move away from {}'.format( current_hypervisor)) # wait_for_server_migration will throw an exception if migration fails openstack_utils.wait_for_server_migration(self.nova_client, vm_name, current_hypervisor) # Bring things back zaza.openstack.configure.masakari.simulate_compute_host_recovery( unit_name, model_name=self.model_name) openstack_utils.enable_all_nova_services(self.nova_client) zaza.openstack.configure.masakari.enable_hosts() self.wait_for_guest_ready(vm_name) def test_instance_restart_on_fail(self): """Test single guest crash and recovery.""" if self.current_release < openstack_utils.get_os_release( 'bionic_ussuri'): raise unittest.SkipTest("Not supported on {}. Bug #1866638".format( self.current_release)) vm_name = 'zaza-test-instance-failover' vm = self.ensure_guest(vm_name) self.wait_for_guest_ready(vm_name) _, unit_name = self.get_guests_compute_info(vm_name) logging.info('{} is running on {}'.format(vm_name, unit_name)) guest_pid = self.get_guest_qemu_pid(unit_name, vm.id, model_name=self.model_name) logging.info('{} pid is {}'.format(vm_name, guest_pid)) inital_update_time = datetime.strptime(vm.updated, "%Y-%m-%dT%H:%M:%SZ") logging.info('Simulating vm crash of {}'.format(vm_name)) zaza.openstack.configure.masakari.simulate_guest_crash( guest_pid, unit_name, model_name=self.model_name) logging.info( 'Waiting for {} to be updated and become active'.format(vm_name)) openstack_utils.wait_for_server_update_and_active( self.nova_client, vm_name, inital_update_time) new_guest_pid = self.wait_for_guest_pid(unit_name, vm.id, model_name=self.model_name) logging.info('{} pid is now {}'.format(vm_name, new_guest_pid)) assert new_guest_pid and new_guest_pid != guest_pid, ( "Restart failed or never happened")
def retry_upon_exception(exc, delay, max_delay, max_attempts): return tenacity.retry(reraise=True, retry=tenacity.retry_if_exception_type(exc), wait=tenacity.wait_exponential(multiplier=delay, max=max_delay), stop=tenacity.stop_after_attempt(max_attempts))
class StatusManager(object): def __init__(self, bigip): self.bigip = bigip self._octavia_driver_lib = driver_lib.DriverLibrary( status_socket=CONF.driver_agent.status_socket_path, stats_socket=CONF.driver_agent.stats_socket_path ) def set_active(self, obj): """Set provisioning_state of octavia object and all ancestors to ACTIVE. :param obj: octavia object like loadbalancer, pools, etc. """ self._set_obj_and_ancestors(obj) def set_deleted(self, obj): """Set provisioning_state of octavia object to DELETED and all ancestors to ACTIVE. :param obj: octavia object like loadbalancer, pools, etc. """ self._set_obj_and_ancestors(obj, lib_consts.DELETED) def set_error(self, obj): """Set provisioning_state of octavia object to ERROR and all ancestors to ACTIVE. :param obj: octavia object like loadbalancer, pools, etc. """ self._set_obj_and_ancestors(obj, lib_consts.ERROR) def update_status(self, loadbalancers): """Set provisioning_state of loadbalancers and all it's children to ACTIVE if PENDING_UPDATE or PENDING_CREATE, else DELETED for PENDING_DELETED. :param loadbalancers: octavia loadbalancers list """ def _set_deleted_or_active(obj): """Sets octavia object to deleted if status was PENDING_DELETE :param obj: octavia object """ if utils.pending_delete(obj): self.set_deleted(obj) else: self.set_active(obj) for loadbalancer in loadbalancers: _set_deleted_or_active(loadbalancer) for listener in loadbalancer.listeners: _set_deleted_or_active(listener) for l7policy in listener.l7policies: _set_deleted_or_active(l7policy) for l7rule in l7policy.l7rules: _set_deleted_or_active(l7rule) for pool in loadbalancer.pools: _set_deleted_or_active(pool) for member in pool.members: _set_deleted_or_active(member) if pool.health_monitor: _set_deleted_or_active(pool.health_monitor) def _set_obj_and_ancestors(self, obj, state=lib_consts.ACTIVE): """Set provisioning_state of octavia object to state and set all ancestors to ACTIVE. :param obj: octavia object like loadbalancer, pools, etc. """ obj_status = self._status_obj(obj, state) # Load Balancer if isinstance(obj, data_models.LoadBalancer): self._update_status_to_octavia({ lib_consts.LOADBALANCERS: [obj_status] }) # Listener if isinstance(obj, data_models.Listener): self._update_status_to_octavia({ lib_consts.LISTENERS: [obj_status], lib_consts.LOADBALANCERS: [self._status_obj(obj.load_balancer)] }) # Pool if isinstance(obj, data_models.Pool): self._update_status_to_octavia({ lib_consts.POOLS: [obj_status], lib_consts.LOADBALANCERS: [self._status_obj(obj.load_balancer)] }) # Member if isinstance(obj, data_models.Member): self._update_status_to_octavia({ lib_consts.MEMBERS: [obj_status], lib_consts.POOLS: [self._status_obj(obj.pool)], lib_consts.LOADBALANCERS: [self._status_obj(obj.pool.load_balancer)] }) # Health Monitor if isinstance(obj, data_models.HealthMonitor): self._update_status_to_octavia({ lib_consts.HEALTHMONITORS: [obj_status], lib_consts.POOLS: [self._status_obj(obj.pool)], lib_consts.LOADBALANCERS: [self._status_obj(obj.pool.load_balancer)] }) # L7Policy if isinstance(obj, data_models.L7Policy): self._update_status_to_octavia({ lib_consts.L7POLICIES: [obj_status], lib_consts.LISTENERS: [self._status_obj(obj.listener)], lib_consts.LOADBALANCERS: [self._status_obj(obj.listener.load_balancer)] }) # L7Rule if isinstance(obj, data_models.L7Rule): self._update_status_to_octavia({ lib_consts.L7RULES: [obj_status], lib_consts.L7POLICIES: [self._status_obj(obj.l7policy)], lib_consts.LISTENERS: [self._status_obj(obj.l7policy.listener)], lib_consts.LOADBALANCERS: [self._status_obj( obj.l7policy.listener.load_balancer)] }) @staticmethod def _status_obj(obj, provisioning_status=lib_consts.ACTIVE): """Return status object for statup update api consumption :param obj: octavia object containing ID :param provisioning_status: provisioning status :return: status object """ return { lib_consts.ID: obj.id, lib_consts.PROVISIONING_STATUS: provisioning_status } @tenacity.retry( retry=tenacity.retry_if_exception_type(), wait=tenacity.wait_incrementing(start=1, increment=10), stop=tenacity.stop_after_attempt(max_attempt_number=3)) def _update_status_to_octavia(self, status): try: self._octavia_driver_lib.update_loadbalancer_status(status) except driver_exceptions.UpdateStatusError as e: msg = ("Error while updating status to octavia: " "%s") % e.fault_string LOG.error(msg) raise driver_exceptions.UpdateStatusError(msg)
import random import tenacity def do_something(): if random.randint(0, 1) == 0: print('Failure') raise IOError print('Success') return True r = tenacity.Retrying( wait=tenacity.wait_fixed(1), retry=tenacity.retry_if_exception_type(IOError)) r.call(do_something)
import os import sys import uuid import httpx from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed import urllib3.exceptions @retry( retry=(retry_if_exception_type(httpx.HTTPError) | retry_if_exception_type(urllib3.exceptions.HTTPError)), stop=stop_after_attempt(10), wait=wait_fixed(60), ) def download_file(*, url, path, client=None): """ Atomically download a file from ``url`` to ``path``. If ``path`` already exists, the file will not be downloaded again. This means that different URLs should be saved to different paths. This function is meant to be used in cases where the contents of ``url`` is immutable -- calling it more than once should always return the same bytes. Returns the download path. """ # If the URL has already been downloaded, we can skip downloading it again. if os.path.exists(path): print("path exists") return path
class ExchangeInterface(): """Interface for performing queries against exchange API's """ def __init__(self, exchange_config): """Initializes ExchangeInterface class Args: exchange_config (dict): A dictionary containing configuration for the exchanges. """ self.logger = structlog.get_logger() self.exchanges = dict() # Loads the exchanges using ccxt. for exchange in exchange_config: if exchange_config[exchange]['required']['enabled']: new_exchange = getattr(ccxt, exchange)({ "enableRateLimit": True }) # sets up api permissions for user if given if new_exchange: self.exchanges[new_exchange.id] = new_exchange else: self.logger.error("Unable to load exchange %s", new_exchange) @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3)) def get_historical_data(self, market_pair, exchange, time_unit, start_date=None, max_periods=100): """Get historical OHLCV for a symbol pair Decorators: retry Args: market_pair (str): Contains the symbol pair to operate on i.e. BURST/BTC exchange (str): Contains the exchange to fetch the historical data from. time_unit (str): A string specifying the ccxt time unit i.e. 5m or 1d. start_date (int, optional): Timestamp in milliseconds. max_periods (int, optional): Defaults to 100. Maximum number of time periods back to fetch data for. Returns: list: Contains a list of lists which contain timestamp, open, high, low, close, volume. """ try: if time_unit not in self.exchanges[exchange].timeframes: raise ValueError( "{} does not support {} timeframe for OHLCV data. Possible values are: {}" .format(exchange, time_unit, list(self.exchanges[exchange].timeframes))) except AttributeError: self.logger.error( '%s interface does not support timeframe queries! We are unable to fetch data!', exchange) raise AttributeError(sys.exc_info()) if not start_date: timeframe_regex = re.compile('([0-9]+)([a-zA-Z])') timeframe_matches = timeframe_regex.match(time_unit) time_quantity = timeframe_matches.group(1) time_period = timeframe_matches.group(2) timedelta_values = { 'm': 'minutes', 'h': 'hours', 'd': 'days', 'w': 'weeks', 'M': 'months', 'y': 'years' } timedelta_args = { timedelta_values[time_period]: int(time_quantity) } start_date_delta = timedelta(**timedelta_args) max_days_date = datetime.now() - (max_periods * start_date_delta) start_date = int( max_days_date.replace(tzinfo=timezone.utc).timestamp() * 1000) historical_data = self.exchanges[exchange].fetch_ohlcv( market_pair, timeframe=time_unit, since=start_date) if not historical_data: raise ValueError( 'No historical data provided returned by exchange.') # Sort by timestamp in ascending order historical_data.sort(key=lambda d: d[0]) time.sleep(self.exchanges[exchange].rateLimit / 1000) return historical_data @retry(retry=retry_if_exception_type(ccxt.NetworkError), stop=stop_after_attempt(3)) def get_exchange_markets(self, exchanges=[], markets=[]): """Get market data for all symbol pairs listed on all configured exchanges. Args: markets (list, optional): A list of markets to get from the exchanges. Default is all markets. exchanges (list, optional): A list of exchanges to collect market data from. Default is all enabled exchanges. Decorators: retry Returns: dict: A dictionary containing market data for all symbol pairs. """ if not exchanges: exchanges = self.exchanges exchange_markets = dict() for exchange in exchanges: exchange_markets[exchange] = self.exchanges[exchange].load_markets( ) if markets: curr_markets = exchange_markets[exchange] # Only retrieve markets the users specified exchange_markets[exchange] = { key: curr_markets[key] for key in curr_markets if key in markets } for market in markets: if market not in exchange_markets[exchange]: self.logger.info('%s has no market %s, ignoring.', exchange, market) time.sleep(self.exchanges[exchange].rateLimit / 1000) return exchange_markets def get_default_exchanges(self): return self.exchanges
class Uploads: """Class dedicated to all "uploads" related endpoints""" # Retry until the unpack agent is finished @retry(retry=retry_if_exception_type(TryAgain), stop=stop_after_attempt(10)) def detail_upload(self, upload_id: int, group: str = None, wait_time: int = 0) -> Upload: """Get detailled information about an upload API Endpoint: GET /uploads/{id} Get information about a given upload. If the upload is not ready wait another ``wait_time`` seconds or look at the ``Retry-After`` to determine how long the wait period shall be. If ``wait_time`` is 0, the time interval specified by the ``Retry-After`` header is used. The function stops trying after **10 attempts**. :Examples: >>> # Wait up to 20 minutes until the upload is ready >>> long_upload = detail_upload(1, 120) >>> # Wait up to 5 minutes until the upload is ready >>> long_upload = detail_upload(1, 30) :param upload_id: the id of the upload :param group: the group the upload shall belong to :param wait_time: use a customized upload wait time instead of Retry-After (in seconds, default: 0) :type upload_id: int :type group: string :type wait_time: int :return: the upload data :rtype: Upload :raises FossologyApiError: if the REST call failed :raises AuthorizationError: if the user can't access the group """ headers = {} if group: headers["groupName"] = group response = self.session.get(f"{self.api}/uploads/{upload_id}", headers=headers) if response.status_code == 200: logger.debug(f"Got details for upload {upload_id}") return Upload.from_json(response.json()) elif response.status_code == 403: description = f"Getting details for upload {upload_id} {get_options(group)}not authorized" raise AuthorizationError(description, response) elif response.status_code == 503: if not wait_time: wait_time = response.headers["Retry-After"] logger.debug( f"Retry GET upload {upload_id} after {wait_time} seconds: {response.json()['message']}" ) time.sleep(int(wait_time)) raise TryAgain else: description = f"Error while getting details for upload {upload_id}" raise FossologyApiError(description, response) def upload_file( # noqa: C901 self, folder, file=None, vcs=None, url=None, server=None, description=None, access_level=None, ignore_scm=False, group=None, wait_time=0, ): """Upload a package to FOSSology API Endpoint: POST /uploads Perform a file, VCS or URL upload and get information about the upload using :func:`~fossology.uploads.Uploads.detail_upload` and passing the ``wait_time`` argument. See description of :func:`~fossology.uploads.Uploads.detail_upload` to configure how long the client shall wait for the upload to be ready. :Example for a file upload: >>> from fossology import Fossology >>> from fossology.obj import AccessLevel >>> foss = Fossology(FOSS_URL, FOSS_TOKEN, username) >>> my_upload = foss.upload_file( foss.rootFolder, file="my-package.zip", description="My product package", access_level=AccessLevel.PUBLIC, ) :Example for a VCS upload: >>> vcs = { "vcsType": "git", "vcsUrl": "https://github.com/fossology/fossology-python", "vcsName": "fossology-python-github-master", "vcsUsername": "", "vcsPassword": "", } >>> vcs_upload = foss.upload_file( foss.rootFolder, vcs=vcs, description="Upload from VCS", access_level=AccessLevel.PUBLIC, ) :Example for a URL upload: >>> url = { "url": "https://github.com/fossology/fossology-python/archive/master.zip", "name": "fossology-python-master.zip", "accept": "zip", "reject": "", "maxRecursionDepth": "1", } >>> url_upload = foss.upload_file( foss.rootFolder, url=url, description="Upload from URL", access_level=AccessLevel.PUBLIC, ) :Example for a SERVER upload: >>> server = { "path": "/tmp/fossology-python", "name": "fossology-python", } >>> server_upload = foss.upload_file( foss.rootFolder, server=server, description="Upload from SERVER", access_level=AccessLevel.PUBLIC, ) :param folder: the upload Fossology folder :param file: the local path of the file to be uploaded :param vcs: the VCS specification to upload from an online repository :param url: the URL specification to upload from a url :param server: the SERVER specification to upload from fossology server :param description: description of the upload (default: None) :param access_level: access permissions of the upload (default: protected) :param ignore_scm: ignore SCM files (Git, SVN, TFS) (default: True) :param group: the group name to chose while uploading the file (default: None) :param wait_time: use a customized upload wait time instead of Retry-After (in seconds, default: 0) :type folder: Folder :type file: string :type vcs: dict() :type url: dict() :type server: dict() :type description: string :type access_level: AccessLevel :type ignore_scm: boolean :type group: string :type wait_time: int :return: the upload data :rtype: Upload :raises FossologyApiError: if the REST call failed :raises AuthorizationError: if the user can't access the group """ headers = {"folderId": str(folder.id)} if description: headers["uploadDescription"] = description if access_level: headers["public"] = access_level.value if ignore_scm: headers["ignoreScm"] = "false" if group: headers["groupName"] = group if file: headers["uploadType"] = "server" with open(file, "rb") as fp: files = {"fileInput": fp} response = self.session.post(f"{self.api}/uploads", files=files, headers=headers) elif vcs or url or server: if vcs: headers["uploadType"] = "vcs" data = json.dumps(vcs) elif url: headers["uploadType"] = "url" data = json.dumps(url) elif server: headers["uploadType"] = "server" data = json.dumps(server) headers["Content-Type"] = "application/json" response = self.session.post(f"{self.api}/uploads", data=data, headers=headers) else: logger.info( "Neither VCS, or Url or filename option given, not uploading anything" ) return if file: source = f"{file}" elif vcs: source = vcs.get("vcsName") elif url: source = url.get("name") elif server: source = server.get("name") if response.status_code == 201: try: upload = self.detail_upload(response.json()["message"], wait_time) if upload.filesize: logger.info( f"Upload {upload.uploadname} ({upload.filesize}) " f"has been uploaded on {upload.uploaddate}") else: logger.info( f"Upload {upload.uploadname} ({upload.hash.size}) " f"has been uploaded on {upload.uploaddate}") return upload except TryAgain: description = f"Upload of {source} failed" raise FossologyApiError(description, response) elif response.status_code == 403: description = ( f"Upload of {source} {get_options(group, folder)}not authorized" ) raise AuthorizationError(description, response) elif server and response.status_code == 500: description = ( f"Upload {description} could not be performed; " f"did you add a prefix for '{server['path']}' in Fossology config " f"variable 'Admin->Customize->Whitelist for serverupload'? " f"Has fossy user read access to {server['path']}?") raise FossologyApiError(description, response) else: description = f"Upload {description} could not be performed" raise FossologyApiError(description, response) @retry(retry=retry_if_exception_type(TryAgain), stop=stop_after_attempt(3)) def upload_summary(self, upload, group=None): """Get clearing information about an upload API Endpoint: GET /uploads/{id}/summary :param upload: the upload to gather data from :param group: the group name to chose while accessing an upload (default: None) :type: Upload :type group: string :return: the upload summary data :rtype: Summary :raises FossologyApiError: if the REST call failed :raises AuthorizationError: if the user can't access the group """ headers = {} if group: headers["groupName"] = group response = self.session.get(f"{self.api}/uploads/{upload.id}/summary", headers=headers) if response.status_code == 200: return Summary.from_json(response.json()) elif response.status_code == 403: description = f"Getting summary of upload {upload.id} {get_options(group)}not authorized" raise AuthorizationError(description, response) elif response.status_code == 503: logger.debug( f"Unpack agent for {upload.uploadname} (id={upload.id}) didn't start yet" ) time.sleep(3) raise TryAgain else: description = f"No summary for upload {upload.uploadname} (id={upload.id})" raise FossologyApiError(description, response) @retry(retry=retry_if_exception_type(TryAgain), stop=stop_after_attempt(3)) def upload_licenses(self, upload, group: str = None, agent=None, containers=False): """Get clearing information about an upload API Endpoint: GET /uploads/{id}/licenses The response does not generate Python objects yet, the plain JSON data is simply returned. :param upload: the upload to gather data from :param agent: the license agents to use (e.g. "nomos,monk,ninka,ojo,reportImport", default: "nomos") :param containers: wether to show containers or not (default: False) :param group: the group name to chose while accessing the upload (default: None) :type upload: Upload :type agent: string :type containers: boolean :type group: string :return: the list of licenses findings for the specified agent :rtype: list of Licenses :raises FossologyApiError: if the REST call failed :raises AuthorizationError: if the user can't access the group """ headers = {} params = {} headers = {} if group: headers["groupName"] = group if agent: params["agent"] = agent else: params["agent"] = agent = "nomos" if containers: params["containers"] = "true" if group: headers["groupName"] = group response = self.session.get(f"{self.api}/uploads/{upload.id}/licenses", params=params, headers=headers) if response.status_code == 200: all_licenses = [] scanned_files = response.json() for file_with_findings in scanned_files: file_licenses = Licenses.from_json(file_with_findings) all_licenses.append(file_licenses) return all_licenses elif response.status_code == 403: description = f"Getting license for upload {upload.id} {get_options(group)}not authorized" raise AuthorizationError(description, response) elif response.status_code == 412: description = f"Unable to get licenses from {agent} for {upload.uploadname} (id={upload.id})" raise FossologyApiError(description, response) elif response.status_code == 503: logger.debug( f"Unpack agent for {upload.uploadname} (id={upload.id}) didn't start yet" ) time.sleep(3) raise TryAgain else: description = f"No licenses for upload {upload.uploadname} (id={upload.id})" raise FossologyApiError(description, response) def delete_upload(self, upload, group=None): """Delete an upload API Endpoint: DELETE /uploads/{id} :param upload: the upload to be deleted :param group: the group name to chose while deleting the upload (default: None) :type upload: Upload :type group: string :raises FossologyApiError: if the REST call failed :raises AuthorizationError: if the user can't access the group """ headers = {} if group: headers["groupName"] = group response = self.session.delete(f"{self.api}/uploads/{upload.id}", headers=headers) if response.status_code == 202: logger.info(f"Upload {upload.id} has been scheduled for deletion") elif response.status_code == 403: description = ( f"Deleting upload {upload.id} {get_options(group)}not authorized" ) raise AuthorizationError(description, response) else: description = f"Unable to delete upload {upload.id}" raise FossologyApiError(description, response) def list_uploads( self, folder=None, group=None, recursive=True, page_size=100, page=1, all_pages=False, ): """Get all uploads available to the registered user API Endpoint: GET /uploads :param folder: only list uploads from the given folder :param group: list uploads from a specific group (not only your own uploads) (default: None) :param recursive: wether to list uploads from children folders or not (default: True) :param page_size: limit the number of uploads per page (default: 100) :param page: the number of the page to fetch uploads from (default: 1) :param all_pages: get all uploads (default: False) :type folder: Folder :type group: string :type recursive: boolean :type page_size: int :type page: int :type all_pages: boolean :return: a tuple containing the list of uploads and the total number of pages :rtype: Tuple(list of Upload, int) :raises FossologyApiError: if the REST call failed :raises AuthorizationError: if the user can't access the group """ params = {} headers = {"limit": str(page_size)} if group: headers["groupName"] = group if folder: params["folderId"] = folder.id if not recursive: params["recursive"] = "false" uploads_list = list() if all_pages: # will be reset after the total number of pages has been retrieved from the API x_total_pages = 2 else: x_total_pages = page while page <= x_total_pages: headers["page"] = str(page) response = self.session.get(f"{self.api}/uploads", headers=headers, params=params) if response.status_code == 200: for upload in response.json(): uploads_list.append(Upload.from_json(upload)) x_total_pages = int(response.headers.get("X-TOTAL-PAGES", 0)) if not all_pages or x_total_pages == 0: logger.info( f"Retrieved page {page} of uploads, {x_total_pages} pages are in total available" ) return uploads_list, x_total_pages page += 1 elif response.status_code == 403: description = f"Retrieving list of uploads {get_options(group, folder)}not authorized" raise AuthorizationError(description, response) else: description = f"Unable to retrieve the list of uploads from page {page}" raise FossologyApiError(description, response) logger.info(f"Retrieved all {x_total_pages} of uploads") return uploads_list, x_total_pages def move_upload(self, upload, folder, group=None): """Move an upload to another folder API Endpoint: PATCH /uploads/{id} :param upload: the Upload to be copied in another folder :param folder: the destination Folder :param group: the group name to chose while changing the upload (default: None) :type upload: Upload :type folder: Folder :type group: string :raises FossologyApiError: if the REST call failed :raises AuthorizationError: if the user can't access the group or folder """ headers = {"folderId": str(folder.id)} if group: headers["groupName"] = group response = self.session.patch(f"{self.api}/uploads/{upload.id}", headers=headers) if response.status_code == 202: logger.info( f"Upload {upload.uploadname} has been moved to {folder.name}") elif response.status_code == 403: description = ( f"Moving upload {upload.id} {get_options(group, folder)}not authorized" ) raise AuthorizationError(description, response) else: description = f"Unable to move upload {upload.uploadname} to {folder.name}" raise FossologyApiError(description, response) def copy_upload(self, upload, folder): """Copy an upload in another folder API Endpoint: PUT /uploads/{id} :param upload: the Upload to be copied in another folder :param folder: the destination Folder :type upload: Upload :type folder: Folder :raises FossologyApiError: if the REST call failed """ headers = {"folderId": str(folder.id)} response = self.session.put(f"{self.api}/uploads/{upload.id}", headers=headers) if response.status_code == 202: logger.info( f"Upload {upload.uploadname} has been copied to {folder.name}") elif response.status_code == 403: description = f"Copy upload {upload.id} {get_options(folder)}not authorized" raise AuthorizationError(description, response) else: description = f"Unable to copy upload {upload.uploadname} to {folder.name}" raise FossologyApiError(description, response)
return int(round(time.time() * 1000)) @retry(wait=tenacity.wait_fixed(0.05), retry=tenacity.retry_if_result(lambda result: result is None)) def _retryable_test_with_wait(thing): return thing.go() @retry(stop=tenacity.stop_after_attempt(3), retry=tenacity.retry_if_result(lambda result: result is None)) def _retryable_test_with_stop(thing): return thing.go() @retry(retry=tenacity.retry_if_exception_type(IOError)) def _retryable_test_with_exception_type_io(thing): return thing.go() @retry( stop=tenacity.stop_after_attempt(3), retry=tenacity.retry_if_exception_type(IOError)) def _retryable_test_with_exception_type_io_attempt_limit(thing): return thing.go() @retry(retry=tenacity.retry_unless_exception_type(NameError)) def _retryable_test_with_unless_exception_type_name(thing): return thing.go()
import random import tenacity def do_something(): if random.randint(0, 1) == 0: print('Failure') raise RuntimeError print('Success') @tenacity.retry(wait=tenacity.wait_fixed(1), retry=tenacity.retry_if_exception_type(RuntimeError)) def do_something_and_retry(): do_something() do_something_and_retry()