def test_wait_backward_compat(self):
"""Ensure Retrying object accepts both old and newstyle wait funcs."""
def wait1(previous_attempt_number, delay_since_first_attempt):
wait1.calls.append(
(previous_attempt_number, delay_since_first_attempt))
return 0
wait1.calls = []
def wait2(previous_attempt_number, delay_since_first_attempt,
last_result):
wait2.calls.append((previous_attempt_number,
delay_since_first_attempt, last_result))
return 0
wait2.calls = []
def dying():
raise Exception("Broken")
retrying1 = Retrying(wait=wait1, stop=tenacity.stop_after_attempt(4))
with reports_deprecation_warning():
self.assertRaises(Exception, lambda: retrying1.call(dying))
self.assertEqual([t[0] for t in wait1.calls], [1, 2, 3])
# This assumes that 3 iterations complete within 1 second.
self.assertTrue(all(t[1] < 1 for t in wait1.calls))
retrying2 = Retrying(wait=wait2, stop=tenacity.stop_after_attempt(4))
with reports_deprecation_warning():
self.assertRaises(Exception, lambda: retrying2.call(dying))
self.assertEqual([t[0] for t in wait2.calls], [1, 2, 3])
# This assumes that 3 iterations complete within 1 second.
self.assertTrue(all(t[1] < 1 for t in wait2.calls))
self.assertEqual([str(t[2].exception()) for t in wait2.calls],
['Broken'] * 3)
def test_retry_child_class_with_override_backward_compat(self):
class MyStop(tenacity.stop_after_attempt):
def __init__(self):
super(MyStop, self).__init__(1)
def __call__(self, attempt_number, seconds_since_start):
return super(MyStop, self).__call__(attempt_number,
seconds_since_start)
retrying = Retrying(wait=tenacity.wait_fixed(0.01), stop=MyStop())
def failing():
raise NotImplementedError()
with pytest.raises(RetryError):
retrying.call(failing)
def test_retry_child_class_with_override_backward_compat(self):
def always_true(_):
return True
class MyRetry(tenacity.retry_if_exception):
def __init__(self):
super(MyRetry, self).__init__(always_true)
def __call__(self, attempt):
return super(MyRetry, self).__call__(attempt)
retrying = Retrying(wait=tenacity.wait_fixed(0.01),
stop=tenacity.stop_after_attempt(1),
retry=MyRetry())
def failing():
raise NotImplementedError()
with pytest.raises(RetryError):
retrying.call(failing)
def test_wait_retry_state_attributes(self):
class ExtractCallState(Exception):
pass
# retry_state is mutable, so return it as an exception to extract the
# exact values it has when wait is called and bypass any other logic.
def waitfunc(retry_state):
raise ExtractCallState(retry_state)
retrying = Retrying(
wait=waitfunc,
retry=(tenacity.retry_if_exception_type()
| tenacity.retry_if_result(lambda result: result == 123)))
def returnval():
return 123
try:
retrying.call(returnval)
except ExtractCallState as err:
retry_state = err.args[0]
self.assertIs(retry_state.fn, returnval)
self.assertEqual(retry_state.args, ())
self.assertEqual(retry_state.kwargs, {})
self.assertEqual(retry_state.outcome.result(), 123)
self.assertEqual(retry_state.attempt_number, 1)
self.assertGreater(retry_state.outcome_timestamp,
retry_state.start_time)
def dying():
raise Exception("Broken")
try:
retrying.call(dying)
except ExtractCallState as err:
retry_state = err.args[0]
self.assertIs(retry_state.fn, dying)
self.assertEqual(retry_state.args, ())
self.assertEqual(retry_state.kwargs, {})
self.assertEqual(str(retry_state.outcome.exception()), 'Broken')
self.assertEqual(retry_state.attempt_number, 1)
self.assertGreater(retry_state.outcome_timestamp,
retry_state.start_time)
def test_before_sleep_log_raises(self):
thing = NoIOErrorAfterCount(2)
logger = logging.getLogger(self.id())
logger.propagate = False
logger.setLevel(logging.INFO)
handler = CapturingHandler()
logger.addHandler(handler)
try:
_before_sleep = tenacity.before_sleep_log(logger, logging.INFO)
retrying = Retrying(wait=tenacity.wait_fixed(0.01),
stop=tenacity.stop_after_attempt(3),
before_sleep=_before_sleep)
retrying.call(thing.go)
finally:
logger.removeHandler(handler)
etalon_re = r'Retrying .* in 0\.01 seconds as it raised .*\.'
self.assertEqual(len(handler.records), 2)
self.assertRegexpMatches(handler.records[0].getMessage(), etalon_re)
self.assertRegexpMatches(handler.records[1].getMessage(), etalon_re)
def test_before_sleep_log_returns(self):
thing = NoneReturnUntilAfterCount(2)
logger = logging.getLogger(self.id())
logger.propagate = False
logger.setLevel(logging.INFO)
handler = CapturingHandler()
logger.addHandler(handler)
try:
_before_sleep = tenacity.before_sleep_log(logger, logging.INFO)
_retry = tenacity.retry_if_result(lambda result: result is None)
retrying = Retrying(wait=tenacity.wait_fixed(0.01),
stop=tenacity.stop_after_attempt(3),
retry=_retry, before_sleep=_before_sleep)
retrying.call(thing.go)
finally:
logger.removeHandler(handler)
self.assertEqual(len(handler.records), 2)
etalon_re = r'Retrying .* in 0\.01 seconds as it returned None'
self.assertRegexpMatches(handler.records[0].getMessage(), etalon_re)
self.assertRegexpMatches(handler.records[1].getMessage(), etalon_re)
def upload_sliced_table(
table: Union[pa.Table, pd.DataFrame, np.ndarray],
s3_url_prefix: str,
s3_file_system: s3fs.S3FileSystem,
max_records_per_entry: Optional[int],
s3_table_writer_func: Callable,
table_slicer_func: Callable,
s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
content_type: ContentType = ContentType.PARQUET,
**s3_client_kwargs) \
-> List[Dict[str, Any]]:
# @retry decorator can't be pickled by Ray, so wrap upload in Retrying
retrying = Retrying(wait=wait_random_exponential(multiplier=1, max=60),
stop=stop_after_delay(30 * 60),
retry=retry_if_exception_type(RetryableError))
manifest_entries = []
table_record_count = len(table)
if max_records_per_entry is None or not table_record_count:
# write the whole table to a single s3 file
manifest_entry = retrying.call(upload_table, table,
f"{s3_url_prefix}/{uuid4()}",
s3_file_system, s3_table_writer_func,
s3_table_writer_kwargs, content_type,
**s3_client_kwargs)
manifest_entries.append(manifest_entry)
else:
# iteratively write table slices
table_slices = table_slicer_func(table, max_records_per_entry)
for table_slice in table_slices:
manifest_entry = retrying.call(upload_table, table_slice,
f"{s3_url_prefix}/{uuid4()}",
s3_file_system,
s3_table_writer_func,
s3_table_writer_kwargs,
**s3_client_kwargs)
manifest_entries.append(manifest_entry)
return manifest_entries
def get_engagement(self, url):
r = Retrying(
retry=retry_if_exception_type((
RateLimitError,
OSError,
requests.exceptions.ConnectionError,
urllib3.exceptions.MaxRetryError,
urllib3.exceptions.NewConnectionError,
)),
before_sleep=self._switch_token,
wait=wait_fixed(5),
)
return r.call(self.api_call, url)
class FetchOperations:
client: BaseClient
dedicated_host_manager: DedicatedHostManager
load_balancer_manager: LoadBalancerManager
firewall_manager: FirewallManager
network_manager: NetworkManager
image_manager: ImageManager
ssh_manager: SshKeyManager
ssl_manager: SSLManager
vs_manager: VSManager
def __init__(self, client, username):
self.client = client
self.username = username
self.retry = self.requests_retry()
def requests_retry(self):
self.retry = Retrying(
stop=stop_after_attempt(RETRY),
retry=retry_if_exception_type(SLRateLimitExceededError),
wait=wait_random_exponential(multiplier=BACK_OFF_FACTOR,
max=MAX_INTERVAL),
reraise=True)
return self.retry
def authenticate_sl_account(self):
"""
Authenticate SL account with provided credentials
:return:
"""
try:
return self.retry.call(self.client.call, "Account", "getObject")
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def list_private_subnets(self,
vlan_no=None,
network_identifier=None) -> list:
"""List all the private Subnetworks associated with an Account."""
subnets_list = list()
try:
self.network_manager = NetworkManager(self.client)
vlans = self.retry.call(
self.network_manager.list_vlans,
filter={
"networkVlans": {
"subnets": {
"addressSpace": {
"operation": "PRIVATE"
}
}
}
},
mask="mask{subnets}".format(subnets=SUBNET_MASK))
for vlan in vlans:
if vlan_no and vlan_no != vlan.get("vlanNumber"):
continue
vlan_name = "{}-{{}}".format(
vlan.get("name")
or "subnet-{}".format(vlan.get("vlanNumber")))
count = 1
for subnet in vlan.get("subnets", []):
sl_subnet = SoftLayerSubnet(
name=vlan_name.format(count),
vif_id=vlan.get("vlanNumber"),
address="{}/{}".format(subnet.get("gateway"),
subnet.get("cidr")),
network_id=subnet["networkIdentifier"])
count = count + 1
if not (network_identifier
and network_identifier != sl_subnet.network_id):
subnets_list.append(sl_subnet)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
return subnets_list
def get_instance_by_id(self, instance_id):
"""
Get a softlayer instance, with provided ID
:return:
"""
try:
self.vs_manager = VSManager(self.client)
instances = self.retry.call(self.vs_manager.get_instance,
instance_id=instance_id)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
else:
return instances
def wait_instance_for_ready(self, instance_id, limit=10):
try:
self.vs_manager = VSManager(self.client)
return self.vs_manager.wait_for_ready(instance_id, limit=limit)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
return False
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def get_image_by_name(self, image_name, create_date=None):
"""
Get image on the basis of image name.
:param image_name: Name of the image.
:param create_date: Creation Date of image
:return: Return the filtered image on the basis of above params.
"""
try:
image_manager = ImageManager(self.client)
images = self.retry.call(image_manager.list_private_images,
name=image_name)
if not images:
return
if not create_date or len(images) == 1:
return images[0]
else:
try:
time_list = [
abs((parser.parse(image["createDate"]) -
parser.parse(create_date)).total_seconds())
for image in images
]
minimum_index = time_list.index(min(time_list))
return images[minimum_index]
except (parser.ParserError, KeyError, IndexError):
return images[0]
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def get_image_by_id(self, image_id):
"""
Get image on the basis of image id.
:param image_id: Classical ID of the image.
:return: Return the filtered image on the basis of above params.
"""
try:
image_manager = ImageManager(self.client)
image = self.retry.call(image_manager.get_image, image_id=image_id)
if image:
return image
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def __parse_to_softlayer(self,
vs_instance,
subnets=None,
address=None,
ssh_keys_required=True,
security_groups_required=True):
if not (vs_instance["status"].get("keyName") == "ACTIVE"
and vs_instance.get("operatingSystem")):
return
sl_instance = SoftLayerInstance.from_softlayer_json(
instance_json=vs_instance)
sl_instance.auto_scale_group = get_auto_scale_group(vs_instance)
sl_instance.network_attached_storages = list_network_attached_storages(
vs_instance.get('allowedNetworkStorage'))
instance_profile, family = get_ibm_instance_profile(
vs_instance["maxCpu"], vs_instance["maxMemory"])
sl_instance.instance_profile = SoftLayerInstanceProfile(
name=instance_profile, family=family)
os = vs_instance["operatingSystem"]["softwareLicense"][
"softwareDescription"]
sl_instance.image = SoftLayerImage.from_softlayer_json(
operating_system=os)
if ssh_keys_required:
sl_instance.ssh_keys.extend([
SoftLayerSshKey.from_softlayer_json(ssh_key)
for ssh_key in vs_instance.get("sshKeys", [])
])
for volume in vs_instance.get("blockDevices", []):
if not volume.get("diskImage"):
continue
SWAP = "SWAP" in volume["diskImage"].get("description", "")
MB = "MB" == volume["diskImage"].get("units")
CLOUD_INIT_DISK = 64 == volume["diskImage"].get("capacity")
if not ((CLOUD_INIT_DISK and MB) or SWAP):
volume_name = f"{sl_instance.name}-{volume['diskImage'].get('name')}"
sl_instance.volumes.append(
SoftLayerVolume.from_softlayer_json(name=volume_name,
volume_json=volume))
for network in vs_instance.get("networkComponents", []):
if not network.get("primarySubnet"):
continue
interface_name = "{name}{port}".format(name=network.get("name"),
port=network.get("port"))
sl_interface = SoftLayerNetworkInterface(
interface_name, network.get("primaryIpAddress"))
if interface_name == "eth0":
sl_interface.is_primary = True
if network["primarySubnet"].get("addressSpace") == "PUBLIC":
sl_interface.is_public_interface = True
if subnets:
attached_subnet = [
subnet for subnet in subnets
if subnet.vif_id == network["networkVlan"].get(
"vlanNumber") and subnet.network_id ==
network["primarySubnet"].get("networkIdentifier")
]
else:
attached_subnet = self.list_private_subnets(
vlan_no=network["networkVlan"].get("vlanNumber"),
network_identifier=network["primarySubnet"].get(
"networkIdentifier"))
if attached_subnet:
sl_interface.subnet = attached_subnet[0]
if security_groups_required:
for security_group in network.get("securityGroupBindings", []):
if not security_group.get("securityGroup"):
continue
if not security_group["securityGroup"].get("name"):
continue
sl_security_group = SoftLayerSecurityGroup(
name=security_group["securityGroup"]["name"])
for rule in security_group["securityGroup"].get(
"rules", []):
if rule["ethertype"] == "IPv6":
continue
sl_security_group_rule = SoftLayerSecurityGroupRule(
direction=rule["direction"],
protocol=rule.get("protocol", "all"),
port_max=rule.get("portRangeMax"),
port_min=rule.get("portRangeMin"),
address=network["primaryIpAddress"])
sl_security_group.rules.append(sl_security_group_rule)
sl_interface.security_groups.append(sl_security_group)
sl_instance.network_interfaces.append(sl_interface)
else:
sl_instance.network_interfaces.append(sl_interface)
if not (address and address not in [
interface.private_ip
for interface in sl_instance.network_interfaces
]):
return sl_instance
def get_instance_details(self,
instance_id,
ssh_keys_required=False,
security_groups_required=False):
details = {
'ssh_keys_required': ssh_keys_required,
'security_groups_required': security_groups_required
}
self.vs_manager = VSManager(self.client)
try:
instance = self.retry.call(self.vs_manager.get_instance,
mask=VIRTUAL_SERVER_MASK,
instance_id=instance_id)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
else:
instance = self.__parse_to_softlayer(instance, **details)
return instance
def list_virtual_servers(self,
address=None,
subnets=None,
ssh_keys_required=True,
security_groups_required=True) -> List:
"""Retrieve a list of all virtual servers on the Account."""
details = {
'address': address,
'subnets': subnets,
'ssh_keys_required': ssh_keys_required,
'security_groups_required': security_groups_required
}
instances_list = []
self.vs_manager = VSManager(self.client)
try:
instances = self.retry.call(self.vs_manager.list_instances,
mask=VIRTUAL_SERVER_MASK)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
else:
for instance in instances:
instance_obj = self.__parse_to_softlayer(instance, **details)
if instance_obj:
instances_list.append(instance_obj)
return instances_list
def list_load_balancers(self, vs_instances=None) -> list:
"""Returns a list of IBM Cloud Loadbalancers"""
load_balancers_list = list()
try:
self.load_balancer_manager = LoadBalancerManager(self.client)
load_balancers = self.retry.call(
self.load_balancer_manager.get_lbaas, mask=LOAD_BALANCER_MASK)
for lb in load_balancers:
pools_list, subnets_list = list(), list()
sl_load_balancer = SoftLayerLoadBalancer(
lb["name"], lb["isPublic"], lb["address"])
for listener in lb.get("listeners", []):
sl_listener = SoftLayerListener(
protocol=listener.get("protocol"),
port=listener.get("protocolPort"),
connection_limit=listener.get("connectionLimit"))
if listener.get("defaultPool"):
sl_pool = SoftLayerBackendPool(
port=listener["defaultPool"].get("protocolPort"),
protocol=listener["defaultPool"].get("protocol"),
algorithm=listener["defaultPool"].get(
"loadBalancingAlgorithm"))
if listener["defaultPool"].get("sessionAffinity"):
sl_pool.session_persistence = listener[
"defaultPool"]["sessionAffinity"].get("type")
if listener["defaultPool"].get("healthMonitor"):
sl_health_monitor = SoftLayerPoolHealthMonitor(
listener["defaultPool"]["healthMonitor"].get(
"maxRetries"), listener["defaultPool"]
["healthMonitor"].get("timeout"),
listener["defaultPool"]["healthMonitor"].get(
"monitorType"), listener["defaultPool"]
["healthMonitor"].get("urlPath"),
listener["defaultPool"]["healthMonitor"].get(
"interval"))
sl_pool.health_monitor = sl_health_monitor
for member in listener["defaultPool"].get(
"members", []):
sl_pool_mem = SoftLayerPoolMember(
weight=member.get("weight"),
port=listener.get("protocolPort"),
ip=member.get("address"))
if not vs_instances:
instance = self.list_virtual_servers(
address=member.get("address"))
else:
instance = [
instance for instance in vs_instances
if member.get("address") in [
interf.private_ip for interf in
instance.network_interfaces
]
]
if not instance:
continue
sl_pool_mem.instance = instance[0]
for network_interface in sl_pool_mem.instance.network_interfaces:
if not network_interface.subnet:
continue
if network_interface.subnet.name not in [
subnet.name for subnet in subnets_list
]:
subnets_list.append(
network_interface.subnet)
sl_pool.pool_members.append(sl_pool_mem)
sl_listener.backend_pool = sl_pool
pools_list.append(sl_pool)
sl_load_balancer.listeners.append(sl_listener)
sl_load_balancer.pools = pools_list
sl_load_balancer.subnets = subnets_list
load_balancers_list.append(sl_load_balancer)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
else:
return load_balancers_list
def list_security_groups(self) -> list:
"""List all the Security Groups within an Account."""
sl_security_groups_list = list()
try:
security_groups = self.retry.call(
NetworkManager(self.client).list_securitygroups,
mask="mask{mask}".format(
mask="[networkComponentBindings, rules]"))
for security_group in security_groups:
sl_security_group = SoftLayerSecurityGroup(
name=security_group["name"])
for rule in security_group.get("rules", []):
if rule["ethertype"] == "IPv6":
continue
sl_security_group_rule = SoftLayerSecurityGroupRule(
direction=rule["direction"],
protocol=rule.get("protocol", "all"),
port_max=rule.get("portRangeMax"),
port_min=rule.get("portRangeMin"))
if rule.get("remoteIp") and "/" in rule.get("remoteIp"):
sl_security_group_rule.rule_type = "cidr_block"
sl_security_group_rule.cidr_block = rule.get(
"remoteIp")
elif rule.get("remoteIp"):
sl_security_group_rule.rule_type = "address"
sl_security_group_rule.address = rule.get("remoteIp")
sl_security_group.rules.append(sl_security_group_rule)
sl_security_groups_list.append(sl_security_group)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
else:
return sl_security_groups_list
def list_images(self) -> dict:
"""List all private and public images on the Account as Dict."""
try:
self.image_manager = ImageManager(self.client)
return {
"private_images":
self.retry.call(self.image_manager.list_private_images),
"public_images":
self.retry.call(self.image_manager.list_public_images),
}
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def list_private_images_name(self):
"""List all private and public images on the Account as Dict."""
image_list = list()
try:
# TODO we should ask region from the user and send the exact name rather then first element
self.image_manager = ImageManager(self.client)
images = self.retry.call(self.image_manager.list_private_images,
mask=IMAGE_MASK)
for image in images:
if not image.get("children"):
continue
image_child = image["children"][0]
if image_child.get('transactionId'):
continue
if not image_child.get("blockDevices"):
continue
block_device = image_child["blockDevices"][0]
if not block_device.get("diskImage"):
continue
disk_image = block_device.get("diskImage")
if not disk_image.get('softwareReferences'):
continue
software_reference = disk_image["softwareReferences"][0]
if not (software_reference.get("softwareDescription")
and software_reference.get("softwareDescription").get(
"longDescription")):
continue
image_name = software_reference.get("softwareDescription").get(
"longDescription")
instance_vpc_image = classical_vpc_image_dictionary.get(
image_name)
instance_vpc_image = instance_vpc_image[
0] if instance_vpc_image else "-"
if instance_vpc_image == "-":
continue
image_list.append({
"id": image.get('id'),
"name": image.get('name'),
"vpc_image_name": instance_vpc_image,
"operating_systems": {
"architecture": "amd64"
}
})
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
else:
return image_list
def list_vsi_hostnames(self) -> List[Dict]:
"""List all private and public images on the Account as Dict."""
instances_list = list()
try:
self.vs_manager = VSManager(self.client)
instances = self.retry.call(self.vs_manager.list_instances,
mask=VSI_ID_HOSTNAME_ONLY_MASK)
for instance in instances:
instances_list.append({
"id": instance.get('id'),
"hostname": instance.get('hostname')
})
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
else:
return instances_list
def list_dedicated_hosts(self, instances=False, raw=False) -> List[Dict]:
"""List all Dedicated Hosts on the Account as Dict."""
try:
self.dedicated_host_manager = DedicatedHostManager(self.client)
dedicated_hosts = self.retry.call(
self.dedicated_host_manager.list_instances,
mask=DEDICATED_HOST_W_INSTANCES_MASK
if instances else DEDICATED_HOST_WO_INSTANCES_MASK)
dedicated_hosts_list = dedicated_hosts
if not raw:
dedicated_hosts_list = [
SoftLayerDedicatedHost.from_softlayer_json(dedicated_host)
for dedicated_host in dedicated_hosts
]
except SoftLayerAPIError as ex:
LOGGER.info(f"error_message => {ex.reason}")
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
else:
return dedicated_hosts_list
def list_ssh_keys(self) -> list:
"""Lists all SSH keys on the Account."""
try:
self.ssh_manager = SshKeyManager(self.client)
return self.retry.call(self.ssh_manager.list_keys)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def list_ssl_certs(self) -> dict:
"""A list of dictionaries representing the requested SSL certs."""
try:
self.ssl_manager = SSLManager(self.client)
return self.retry.call(self.ssl_manager.list_certs)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def get_classic_image_name(self, image_name):
"""concatenate an integer value if this image already exists"""
try:
img_manager = ImageManager(self.client)
private_images = self.retry.call(img_manager.list_private_images,
name=image_name)
num = 1
while True:
if image_name not in [
image.get("name") for image in private_images
]:
return image_name
image_name = "-".join([image_name, str(num)])
num += 1
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def list_kubernetes_clusters(self, user_name, api_key):
"""List Classic IKS Clusters on the Account."""
try:
iks_clusters_list = list()
kubernetes_client = ClassicKubernetesClient(user_name, api_key)
clusters = kubernetes_client.list_clusters()
if clusters:
for cluster in clusters:
multi_zone_cluster = False
free_tier_cluster = False
if cluster['state'] != "normal" or cluster[
'type'] == "openshift":
continue
k8s_cluster = KubernetesCluster.from_ibm_json_body(cluster)
k8s_cluster.status = CREATION_PENDING
k8s_cluster_worker_pools = kubernetes_client.get_cluster_worker_pool(
cluster["id"])
for k8s_cluster_worker_pool in k8s_cluster_worker_pools:
if len(k8s_cluster_worker_pool['zones']) > 1:
multi_zone_cluster = True
break
classic_k8s_cluster_worker_pool = KubernetesClusterWorkerPool.from_ibm_json_body(
k8s_cluster_worker_pool)
for k8s_cluster_worker_pool_zone in k8s_cluster_worker_pool[
'zones']:
classic_k8s_cluster_worker_pool_zone = KubernetesClusterWorkerPoolZone.from_ibm_json_body(
k8s_cluster_worker_pool_zone)
k8s_cluster_subnets = kubernetes_client.get_cluster_subnets(
cluster["id"], cluster["resourceGroup"])
if k8s_cluster_subnets == "Free tier cluster":
free_tier_cluster = True
break
subnets = self.list_private_subnets()
for subnet in subnets:
for k8s_cluster_subnet in k8s_cluster_subnets:
if classic_k8s_cluster_worker_pool_zone.private_vlan == k8s_cluster_subnet[
"id"]:
if subnet.network == k8s_cluster_subnet[
"subnets"][0]['cidr']:
classic_k8s_cluster_worker_pool_zone.subnets.append(
subnet.to_ibm())
classic_k8s_cluster_worker_pool.zones.append(
classic_k8s_cluster_worker_pool_zone)
k8s_cluster.worker_pools.append(
classic_k8s_cluster_worker_pool)
if multi_zone_cluster:
continue
if free_tier_cluster:
continue
kube_config = kubernetes_client.get_cluster_kube_config(
cluster["id"])
kube_config = K8s(configuration_json=kube_config)
workloads = list()
namespaces = kube_config.client.CoreV1Api().list_namespace(
)
for namespace in namespaces.items:
kubernetes_objects = {
"pod": list(),
"svc": list(),
"pvc": list()
}
if namespace.metadata.name in [
"kube-system", "kube-public", "ibm-cert-store",
"ibm-operators", "kube-node-lease",
"ibm-system", "ibm-observe"
]:
continue
kubernetes_objects[
"namespace"] = namespace.metadata.name
pods = kube_config.client.CoreV1Api(
).list_namespaced_pod(
namespace=namespace.metadata.name)
if pods.items:
for pod in pods.items:
kubernetes_objects["pod"].append(
pod.metadata.name)
svcs = kube_config.client.CoreV1Api(
).list_namespaced_service(
namespace=namespace.metadata.name)
if svcs.items:
for svc in svcs.items:
kubernetes_objects["svc"].append(
svc.metadata.name)
pvcs = kube_config.client.CoreV1Api(
).list_namespaced_persistent_volume_claim(
namespace=namespace.metadata.name)
if pvcs.items:
for pvc in pvcs.items:
kubernetes_objects["pvc"].append({
"name":
pvc.metadata.name,
"size":
pvc.spec.resources.requests['storage']
})
workloads.append(kubernetes_objects)
k8s_cluster.workloads = workloads
iks_clusters_list.append(k8s_cluster)
return iks_clusters_list
except Exception as ex:
LOGGER.info(ex)
class CreateOperations:
client: BaseClient
image_manager: ImageManager
vs_manager: VSManager
def __init__(self, client, username):
self.client = client
self.username = username
self.retry = self.requests_retry()
def requests_retry(self):
self.retry = Retrying(
stop=stop_after_attempt(RETRY),
retry=retry_if_exception_type(SLRateLimitExceededError),
wait=wait_random_exponential(multiplier=BACK_OFF_FACTOR,
max=MAX_INTERVAL),
reraise=True)
return self.retry
def create_instance(self, instance_body):
"""
Create Sofltlayer Instance
return:
"""
try:
self.vs_manager = VSManager(self.client)
return self.retry.call(self.vs_manager.create_instance,
**instance_body)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def capture_image(self, instance_id, image_name, additional_disks=False):
"""
Create and capture image template of an image belonging to classical VSI
:return:
"""
try:
self.vs_manager = VSManager(self.client)
return self.retry.call(self.vs_manager.capture, instance_id,
image_name, additional_disks)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def export_image(self, image_id, cos_url, api_key):
"""
Export Image template from Classic to specified COS
:return:
"""
try:
self.image_manager = ImageManager(self.client)
return self.retry.call(self.image_manager.export_image_to_uri,
image_id, cos_url, api_key)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def delete_image(self, image_id):
"""
Delete image template from Classical Infrastructure
:param image_id: Classical Image ID for the image
:return:
"""
try:
self.image_manager = ImageManager(self.client)
return self.retry.call(self.image_manager.delete_image, image_id)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
def delete_instance(self, instance_id):
"""
Delete instance from Classical Infrastructure
:param instance_id: Classical Instance ID for the image
:return:
"""
try:
self.vs_manager = VSManager(self.client)
return self.retry.call(self.vs_manager.cancel_instance,
instance_id)
except SoftLayerAPIError as ex:
if ex.faultCode == SL_RATE_LIMIT_FAULT_CODE:
raise SLRateLimitExceededError(ex)
elif ex.faultCode == INVALID_API_KEY_CODE:
raise SLAuthError(self.username)
raise SLExecuteError(ex)
class StrandApiClientWrapper:
"""Manage all outgoing interaction with the CoreApi"""
def __init__(self, strand_api_client):
self.strand_api_client = strand_api_client
self.logger = get_logger('StrandApiClientWrapper')
self.standard_retrier = Retrying(
reraise=True,
wait=wait_fixed(2),
stop=stop_after_attempt(5),
after=after_log(logger=self.logger,
log_level=self.logger.getEffectiveLevel()),
retry=retry_if_exception_type(StrandApiClientException))
def get_user_by_email(self, email):
operation_definition = f'''
{{
user(email: "{email}") {{
id
}}
}}
'''
response_body = self.standard_retrier.call(
self.strand_api_client.query,
operation_definition=operation_definition)
if self._did_return_not_exists(response_body=response_body):
return None
return self._deserialize_response_body(response_body=response_body,
ObjectSchema=StrandUserSchema,
path_to_object=['data', 'user'])
def create_user_with_team(self, email, first_name, last_name, team_id):
operation_definition = f'''
{{
createUserWithTeams(input: {{email: "{email}",
firstName: "{first_name}",
lastName: "{last_name}",
teamIds: [{team_id}]}}) {{
user {{
id
}}
}}
}}
'''
response_body = self.standard_retrier.call(
self.strand_api_client.mutate,
operation_definition=operation_definition)
return self._deserialize_response_body(
response_body=response_body,
ObjectSchema=StrandUserSchema,
path_to_object=['data', 'createUserWithTeams', 'user'])
def add_user_to_team(self, user_id, team_id):
operation_definition = f'''
{{
addMembersToTeam(input: {{ id: {team_id},
memberIds: [{user_id}]}}) {{
team {{
members {{
id
}}
}}
}}
}}
'''
response_body = self.standard_retrier.call(
self.strand_api_client.mutate,
operation_definition=operation_definition)
self._validate_no_response_body_errors(response_body=response_body)
def create_team(self, name):
operation_definition = f'''
{{
createTeam(input: {{name: "{name}"}}) {{
team {{
id
}}
}}
}}
'''
response_body = self.standard_retrier.call(
self.strand_api_client.mutate,
operation_definition=operation_definition)
return self._deserialize_response_body(
response_body=response_body,
ObjectSchema=StrandTeamSchema,
path_to_object=['data', 'createTeam', 'team'])
def create_strand(self, team_id, saver_user_id, body):
body = json.dumps(body)
operation_definition = f'''
{{
createStrand(input: {{ownerId: {team_id},
saverId: {saver_user_id},
body: {body}}}) {{
strand {{
id
}}
}}
}}
'''
response_body = self.standard_retrier.call(
self.strand_api_client.mutate,
operation_definition=operation_definition)
return self._deserialize_response_body(
response_body=response_body,
ObjectSchema=StrandStrandSchema,
path_to_object=['data', 'createStrand', 'strand'])
def update_strand(self, strand: StrandStrand):
title = json.dumps(strand.title)
operation_definition = f'''
{{
updateStrand(input: {{title: {title},
id: {strand.id},
tags: [{','.join([f'{{name: "{tag.name}"}}' for tag in strand.tags])}]
}}){{
strand {{
id
}}
}}
}}
'''
response_body = self.standard_retrier.call(
self.strand_api_client.mutate,
operation_definition=operation_definition)
self._validate_no_response_body_errors(response_body=response_body)
def _deserialize_response_body(self,
response_body,
ObjectSchema,
path_to_object,
many=False):
"""Deserializes response_body[**path_to_object] using ObjectSchema"""
self._validate_no_response_body_errors(response_body=response_body)
result_json = response_body
for key in path_to_object:
result_json = result_json[key]
if result_json is None:
return None
if many:
return [
ObjectSchema().load(
dict_keys_camel_case_to_underscores(x)).data
for x in result_json
]
return ObjectSchema().load(
dict_keys_camel_case_to_underscores(result_json)).data
def _validate_no_response_body_errors(self, response_body):
"""Raises an exception if there are any errors in response_body"""
if 'errors' in response_body:
message = f'Errors when calling StrandApiClient. Body: {response_body}'
self.logger.error(message)
raise StrandTranslationException(message)
def _did_return_not_exists(self, response_body):
"""Returns true if the errors exists but it looks like it's only because there's no result"""
if 'errors' in response_body and len(response_body['errors']) == 1:
return 'matching query does not exist' in response_body['errors'][
0]['message']
return False
class SlackClientWrapper:
"""Manage all outgoing interaction with Slack APIs"""
def __init__(self, SlackClientClass):
self.SlackClientClass = SlackClientClass
self.logger = get_logger('SlackClientWrapper')
self.standard_retrier = Retrying(
reraise=True,
wait=wait_fixed(2),
stop=stop_after_attempt(5),
after=after_log(logger=self.logger,
log_level=self.logger.getEffectiveLevel()),
retry=(retry_if_exception_type(ConnectionError)))
def get_channel_history(self,
slack_user_id,
slack_team_id,
slack_channel_id,
count=1000):
slack_client = self._get_slack_client(slack_team_id=slack_team_id,
is_bot=False,
slack_user_id=slack_user_id)
if slack_channel_id.startswith('C'):
response = self.standard_retrier.call(slack_client.api_call,
method='channels.history',
channel=slack_channel_id,
inclusive=True,
count=count)
elif slack_channel_id.startswith('D'):
response = self.standard_retrier.call(slack_client.api_call,
method='im.history',
channel=slack_channel_id,
inclusive=True,
count=count)
else:
response = self.standard_retrier.call(slack_client.api_call,
method='groups.history',
channel=slack_channel_id,
inclusive=True,
count=count)
self._validate_response_ok(response, 'get_channel_history',
slack_team_id, slack_channel_id, count)
return self._deserialize_response_body(response_body=response,
ObjectSchema=SlackMessageSchema,
path_to_object=['messages'],
many=True)
def send_dm_to_user(self,
slack_team_id,
slack_user_id,
text,
attachments=None):
if not attachments:
attachments = []
slack_client = self._get_slack_client(slack_team_id=slack_team_id,
is_bot=False,
slack_user_id=slack_user_id)
response = self.standard_retrier.call(slack_client.api_call,
method='im.open',
user=slack_user_id)
self._validate_response_ok(response, 'send_dm_to_user', slack_team_id,
slack_user_id, text)
slack_channel_id = response['channel']['id']
self.standard_retrier.call(slack_client.api_call,
method='chat.postMessage',
channel=slack_channel_id,
text=text,
attachments=attachments,
as_user=False)
def send_ephemeral_message(self,
slack_team_id,
slack_channel_id,
slack_user_id,
text,
attachments=None,
use_bot_token=False):
if not attachments:
attachments = []
if use_bot_token:
slack_client = self._get_slack_client(slack_team_id=slack_team_id,
is_bot=True)
else:
# If this is a DM to a non-installer, we will need to use the bot token
slack_client = self._get_slack_client(slack_team_id=slack_team_id,
slack_user_id=slack_user_id,
is_bot=False)
response = self.standard_retrier.call(slack_client.api_call,
method='chat.postEphemeral',
channel=slack_channel_id,
user=slack_user_id,
text=text,
attachments=attachments,
as_user=False)
self._validate_response_ok(response, 'send_ephemeral_message',
slack_team_id, slack_channel_id,
slack_user_id, text)
def post_to_response_url(self, response_url, payload):
response_retrier = self.standard_retrier.copy(
wait=wait_fixed(0.5), stop=stop_after_attempt(4))
response = response_retrier.call(
fn=requests.post,
url=response_url,
headers={'Content-Type': 'application/json'},
json=payload)
self._validate_response_ok(response, 'post_to_response_url',
response_url, payload)
def send_dialog(self, trigger_id, slack_team_id, dialog):
slack_client = self._get_slack_client(slack_team_id=slack_team_id)
response = self.standard_retrier.call(slack_client.api_call,
method='dialog.open',
trigger_id=trigger_id,
dialog=dialog)
self._validate_response_ok(response, 'send_dialog', trigger_id,
slack_team_id, dialog)
def get_user_info(self, slack_team_id, slack_user_id):
slack_client = self._get_slack_client(slack_team_id=slack_team_id)
response = self.standard_retrier.call(slack_client.api_call,
method='users.info',
user=slack_user_id)
self._validate_response_ok(response, 'get_user_info', slack_team_id,
slack_user_id)
return self._deserialize_response_body(response_body=response,
ObjectSchema=SlackUserSchema,
path_to_object=['user'])
def get_channel_info(self, slack_team_id, slack_channel_id):
slack_client = self._get_slack_client(slack_team_id=slack_team_id)
response = self.standard_retrier.call(slack_client.api_call,
method='channels.info',
channel=slack_channel_id)
self._validate_response_ok(response, 'get_channel_info', slack_team_id,
slack_channel_id)
return self._deserialize_response_body(response_body=response,
ObjectSchema=SlackChannelSchema,
path_to_object=['channel'])
def send_message(self,
slack_team_id,
slack_channel_id,
text,
attachments=None):
slack_client = self._get_slack_client(slack_team_id=slack_team_id)
response = self.standard_retrier.call(slack_client.api_call,
method='chat.postMessage',
channel=slack_channel_id,
text=text,
attachments=attachments)
self._validate_response_ok(response, 'send_message', slack_team_id,
slack_channel_id, text)
def submit_oauth_code(self, code) -> SlackOauthAccessResponse:
slack_client = self.SlackClientClass(token=None)
# Intentional: pulling from config directly to avoid long pass-through
response = self.standard_retrier.call(
slack_client.api_call,
method='oauth.access',
code=code,
client_id=config['CLIENT_ID'],
client_secret=config['CLIENT_SECRET'])
self._validate_response_ok(response, 'submit_oauth_code', code)
return self._deserialize_response_body(
response_body=response,
ObjectSchema=SlackOauthAccessResponseSchema)
def _deserialize_response_body(self,
response_body,
ObjectSchema,
path_to_object=None,
many=False,
**kwargs):
"""Deserializes response_body[**path_to_object] merged with **kwargs using ObjectSchema"""
if path_to_object is None:
path_to_object = []
result_json = response_body
for key in path_to_object:
result_json = result_json[key]
if many:
return [
ObjectSchema().load(dict(**x, **kwargs)).data
for x in result_json
]
return ObjectSchema().load(dict(**result_json, **kwargs)).data
@db_session
def _get_slack_client(self,
session,
slack_team_id,
slack_user_id=None,
is_bot=True):
"""Using slack_team_id's tokens from the in-memory repo, wires up a slack_client"""
assert slack_user_id or is_bot, 'Need slack_user_id to get token is is_bot is false'
token = self._get_bot_token(
slack_team_id, session) if is_bot else self._get_user_token(
slack_team_id, slack_user_id, session)
return self.SlackClientClass(token=token)
def _get_bot_token(self, slack_team_id, session):
return session.query(Bot).filter(
Bot.agent_slack_team_id == slack_team_id).one().access_token
def _get_user_token(self, slack_team_id, slack_user_id, session):
return session.query(Installation).filter(
Installation.installer_agent_slack_team_id == slack_team_id,
Installation.installer_slack_user_id ==
slack_user_id).one().access_token
def _validate_response_ok(self, response, *args):
"""All variables in *args are dumped to logger output"""
is_negative = not response[
'ok'] if 'ok' in response else response.status_code != 200
if is_negative:
self._raise_wrapper_exception(response, *args)
def _raise_wrapper_exception(self, response, *args):
message = f'Errors when calling SlackClient. \n\t{response}\n\t{args}'
self.logger.error(message)
raise WrapperException(wrapper_name='SlackClient', message=message)
