def __wait_for_redis_master(self):
"""Waits for the Redis master to startup
Connects to the Redis master pod and waits until it sees that the slaves are connected.
Ideally, this should ensure that when the Redis slaves startup they are able to talk
to the Redis master immediately and connect
"""
sentinel_started = False
while not sentinel_started:
sleep(2)
try:
out = exec_redis_command(
get_name(self.redis_master),
self.__namespace,
command=[
"redis-cli",
"-p",
"26379",
"sentinel",
"master",
"mymaster",
],
)
except Exception as _:
continue
sentinel_started = "role-reported\nmaster" in out
master_started = False
while not master_started:
sleep(2)
try:
out = exec_redis_command(
get_name(self.redis_master),
self.__namespace,
command=["redis-cli", "-p", "6379", "info"],
)
except Exception as _:
continue
master_started = "role:master" in out
def delete(self):
evaluate_request(
self.__custom_objects_api.delete_namespaced_custom_object(
namespace=self.__namespace,
body={},
name=get_name(self.redis_metrics),
async_req=True,
group=self.__group,
version=self.__version,
plural=self.__plural,
))
evaluate_request(
self.__custom_objects_api.delete_namespaced_custom_object(
namespace=self.__namespace,
body={},
name=get_name(self.bps_metrics),
async_req=True,
group=self.__group,
version=self.__version,
plural=self.__plural,
))
def attach_instance_ebs(aws_instance, tag, unix_device=u.DEFAULT_UNIX_DEVICE):
"""Attaches volume to instance. Will try to detach volume if it's already mounted somewhere else. Will retry indefinitely on error."""
ec2 = u.create_ec2_resource()
v = list(ec2.volumes.filter(Filters=[{'Name':'tag:Name', 'Values':[tag]}, {"Name":"availability-zone", 'Values':[os.environ['ZONE']]}]).all())
assert(v), f"Volume {tag} not found."
v = v[0]
volume_name = u.get_name(v)
already_attached = v.attachments and v.attachments[0]['InstanceId'] == aws_instance.id
instance_name = u.get_name(aws_instance)
# TODO: still have edge case when it doesn't report as already attached
# and keeps trying to attach to an instance that has data mounted already
if already_attached:
print(f'volume {volume_name} ({v.id}) already attached to {instance_name}')
return
while v.state != 'available':
response = v.detach_from_instance()
instance_id = v.attachments[0]['InstanceId']
instance_name = u.get_name(instance_id)
print(f'Volume {tag} is attached to {instance_name}, detaching, response={response.get("State", "none")}')
time.sleep(ATTACH_WAIT_INTERVAL_SEC)
v.reload()
while True:
try:
response = v.attach_to_instance(InstanceId=aws_instance.id,
Device=unix_device)
print(f'Attaching {volume_name} to {instance_name}: response={response.get("State", "none")}')
# sometimes have unrecoverable failure on brand new instance with
# possibly because of https://forums.aws.amazon.com/thread.jspa?threadID=66192
# Error attaching volume: (An error occurred (InvalidParameterValue) when calling the AttachVolume operation: Invalid value '/dev/xvdf' for unixDevice. Attachment point /dev/xvdf is already in use). Retrying in 5 An error occurred (InvalidParameterValue) when calling the AttachVolume operation: Invalid value '/dev/xvdf' for unixDevice. Attachment point /dev/xvdf is already in use
except Exception as e:
print(f"Failed attaching ({v.id}) to ({aws_instance.id})")
print(f'Error attaching volume: ({e}). Retrying in {ATTACH_WAIT_INTERVAL_SEC}', e)
time.sleep(ATTACH_WAIT_INTERVAL_SEC)
continue
else:
print('Attachment successful')
break
def create_vault_with_network(self):
"""
Creates a key vault with network access limited by a NetworkRuleSet
"""
vault_name = get_name('vault')
# setup vault permissions for the access policy for the sample service principle
permissions = Permissions(keys=KEY_PERMISSIONS_ALL,
secrets=SECRET_PERMISSIONS_ALL,
certificates=CERTIFICATE_PERMISSIONS_ALL,
storage=STORAGE_PERMISSIONS_ALL)
policy = AccessPolicyEntry(tenant_id=self.config.tenant_id,
object_id=self.config.client_oid,
permissions=permissions)
# Network ACL definitions
# The only action supported for virtual network and IP rules is "allow".
# To deny an address, set the default action to 'deny' and do not explicitly allow the address.
network_acls = NetworkRuleSet(
# allow bypass of network ACL rules by other azure services. Valid values are azure_services or none
bypass=NetworkRuleBypassOptions.azure_services,
# the action to take if access attempt doesn't match any rule. Valid values are allow or deny
default_action=NetworkRuleAction.deny,
# IP rules (allowed IPv4 addresses / ranges)
ip_rules=[IPRule(value='0.0.0.0/0')
], # Allow access from a IP address range
# Virtual network rules(Allows access to Azure Virtual Networks by their Azure Resource ID)
virtual_network_rules=[
# To specifically allow access to a vnet, uncomment the line below and replace the id with the correct
# resource id for your vnet
# VirtualNetworkRule(id='/subscriptions/subid/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/subnet1')
])
properties = VaultProperties(tenant_id=self.config.tenant_id,
sku=Sku(name='standard'),
access_policies=[policy],
network_acls=network_acls)
parameters = VaultCreateOrUpdateParameters(
location=self.config.location, properties=properties)
parameters.properties.enabled_for_deployment = True
parameters.properties.enabled_for_disk_encryption = True
parameters.properties.enabled_for_template_deployment = True
print('creating vault {}'.format(vault_name))
self.vault = self.mgmt_client.vaults.create_or_update(
resource_group_name=self.config.group_name,
vault_name=vault_name,
parameters=parameters).result()
def list_ebss():
"""."""
ec2 = u.create_ec2_resource()
volumes = list(ec2.volumes.all())
for vol in volumes:
if args.io1 and vol.volume_type != 'io1':
continue
vol_name = u.get_name(vol)
if not vol_name:
vol_name = vol.id
attached_to = []
if vol.attachments:
for attachment in vol.attachments:
instance_id = attachment["InstanceId"]
instance = ec2.Instance(instance_id)
attached_to.append(u.get_name(instance))
else:
attached_to.append('<unattached>')
print("%25s %s %s %s" %
(vol_name, vol.availability_zone, attached_to, vol.id))
def createfile():
body = request.data.decode("utf-8")
content = json.loads(body)['content']
id = get_next_id()
try:
write_file(id, content)
except OSError as error:
return new_error(str(error))
name = get_name(content)
return json.dumps({'status': 'Ok', 'file': new_file_object(id, name)})
def delete(self):
"""Removes the Postgres resources from the cluster
"""
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.postgres_service),
async_req=True,
)
)
evaluate_request(
self.__v1_api.delete_namespaced_config_map(
namespace=self.__namespace,
name=get_name(self.postgres_config_map),
async_req=True,
)
)
evaluate_request(
self.__v1_api.delete_namespaced_persistent_volume_claim(
namespace=self.__namespace,
name=get_name(self.postgres_pvc),
async_req=True,
)
)
evaluate_request(
self.__v1_apps_api.delete_namespaced_deployment(
namespace=self.__namespace, name=get_name(self.postgres), async_req=True
)
)
evaluate_request(
self.__v1_apps_api.delete_namespaced_deployment(
namespace=self.__namespace,
name=get_name(self.postgres_connector),
async_req=True,
)
)
def main():
if len(sys.argv) < 2:
mode = 'list'
else:
mode = sys.argv[1]
if mode == 'list':
list_instances()
elif mode == 'reboot':
task_fragment = sys.argv[2]
instance = get_instance(task_fragment)
print("Rebooting " + u.get_name(instance.tags))
instance.reboot()
else:
assert False, "Unknown mode " + mode
def list_efss():
for region in ['us-west-2', 'us-east-1']:
print()
print('=' * 80)
print(region)
print('=' * 80)
efs_client = boto3.client('efs', region_name=region)
response = efs_client.describe_file_systems()
assert u.is_good_response(response)
for efs_response in response['FileSystems']:
# {'CreationTime': datetime.datetime(2017, 12, 19, 10, 3, 44, tzinfo=tzlocal()),
# 'CreationToken': '1513706624330134',
# 'Encrypted': False,
# 'FileSystemId': 'fs-0f95ab46',
# 'LifeCycleState': 'available',
# 'Name': 'nexus01',
# 'NumberOfMountTargets': 0,
# 'OwnerId': '316880547378',
# 'PerformanceMode': 'generalPurpose',
# 'SizeInBytes': {'Value': 6144}},
efs_id = efs_response['FileSystemId']
tags_response = efs_client.describe_tags(FileSystemId=efs_id)
assert u.is_good_response(tags_response)
key = u.get_name(tags_response.get('Tags', ''))
print("%-16s %-16s" % (efs_id, key))
print('-' * 40)
# list mount points
response = efs_client.describe_mount_targets(FileSystemId=efs_id)
ec2 = boto3.resource('ec2', region_name=region)
if not response['MountTargets']:
print("<no mount targets>")
else:
for mount_response in response['MountTargets']:
subnet = ec2.Subnet(mount_response['SubnetId'])
zone = subnet.availability_zone
state = mount_response['LifeCycleState']
id = mount_response['MountTargetId']
ip = mount_response['IpAddress']
print('%-16s %-16s %-16s %-16s' % (
zone,
ip,
id,
state,
))
print()
def __wait_for_redis_slaves(self):
"""Waits for the Redis slaves to register themselves against the master
"""
slaves_started = False
while not slaves_started:
sleep(2)
try:
out = exec_redis_command(
get_name(self.redis_master),
self.__namespace,
command=["redis-cli", "info"],
)
except Exception as _:
continue
slaves_started = "connected_slaves:3" in out
def delete(self):
"""Removes the Elasticsearch resources from the cluster
"""
evaluate_request(
self.v1_api.delete_namespaced_config_map(
namespace=self.namespace,
name=get_name(self.elasticsearch_config_map),
async_req=True,
))
evaluate_request(
self.v1_apps_api.delete_namespaced_deployment(
namespace=self.namespace,
name=get_name(self.elasticsearch_master),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.v1_api.delete_namespaced_service(
namespace=self.namespace,
name=get_name(self.elasticsearch_discovery),
async_req=True,
))
evaluate_request(
self.v1_apps_api.delete_namespaced_stateful_set(
namespace=self.namespace,
name=get_name(self.elasticsearch_data),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.v1_api.delete_namespaced_service(
namespace=self.namespace,
name=get_name(self.elasticsearch_data_service),
async_req=True,
))
evaluate_request(
self.v1_apps_api.delete_namespaced_deployment(
namespace=self.namespace,
name=get_name(self.elasticsearch_client),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.v1_api.delete_namespaced_service(
namespace=self.namespace,
name=get_name(self.elasticsearch_client_service),
async_req=True,
))
self.logstash_manager.delete()
def lookup_aws_instances(name):
"""Returns all AWS instances for given job."""
ec2 = boto3.resource('ec2')
# TODO: add waiting so that instances that "initializing" are supported
instances = ec2.instances.filter(Filters=[{
'Name': 'instance-state-name',
'Values': ['running']
}])
result = []
for i in instances.all():
inst_name = u.get_name(i.tags)
key_name = i.key_name
if inst_name == name:
# if key_name != KEY_NAME:
# print("name matches, but key name %s doesn't match %s, skipping"%(key_name, KEY_NAME))
# continue
result.append(i)
ec2 = boto3.resource('ec2')
instances = ec2.instances.filter(Filters=[{
'Name': 'instance-state-name',
'Values': ['running']
}])
result = []
for i in instances:
names = []
if i.tags:
names = [tag['Value'] for tag in i.tags if tag['Key'] == 'Name']
key_name = i.key_name
assert len(names) <= 1
if names:
inst_name = names[0]
else:
inst_name = ''
if inst_name == name:
# if key_name != KEY_NAME:
# print("name matches, but key name %s doesn't match %s, skipping"%(key_name, KEY_NAME))
# continue
result.append(i)
return result
def list_instances():
ec2 = u.create_ec2_resource()
instances = [(u.seconds_from_datetime(i.launch_time), i)
for i in ec2.instances.all()]
sorted_instances = sorted(instances, key=itemgetter(0))
for (seconds, instance) in sorted_instances:
hours_ago = (time.time() - seconds) / 3600
hours_ago += 8 # adjust for time being in UTC
if instance.state['Name'] != 'running':
continue
if not (LIMIT_TO_KEY in instance.key_name):
continue
print("%4s %20s %10s %20s %s %s" %
(int(hours_ago), u.get_name(instance.tags),
instance.instance_type, instance.public_ip_address,
instance.private_ip_address, instance.id))
def placeholder(shape, name=None, dtype=None):
"""Construct a HeteroCL placeholder for inputs/outputs.
If the shape is an empty tuple, the returned value is a scalar.
Parameters
----------
shape : tuple
The shape of the placeholder
name : str, optional
The name of the placeholder
dtype : Type, optional
The data type of the placeholder
Returns
-------
Scalar or Tensor
Examples
--------
.. code-block:: python
# scalar - cannot be updated
a = hcl.placeholder((), "a")
# 1-dimensional tensor - can be updated
A = hcl.placeholder((1,), "A")
"""
name = util.get_name("placeholder", name)
dtype = util.get_dtype(dtype)
if shape == ():
return Scalar(tvm.tir.Var(name, dtype))
tensor = Tensor(shape, dtype, name)
tensor.tensor = tvm_api.Placeholder(tensor.buf.shape, dtype, name)
# placeholder is also a stage
stage = Stage(name)
stage._op = tensor.tensor
stage._buf = tensor._buf
tensor.first_update = stage
tensor.last_update = stage
return tensor
def main():
fragment = ''
if len(sys.argv) > 1:
fragment = sys.argv[1]
def get_name(instance_response):
names = [
entry['Value'] for entry in instance_response.get('Tags', [])
if entry['Key'] == 'Name'
]
if not names:
names = ['']
assert len(names) == 1
return names[0]
region = u.get_region()
client = boto3.client('ec2', region_name=region)
ec2 = boto3.resource('ec2', region_name=region)
response = client.describe_instances()
username = os.environ.get("EC2_USER", "ubuntu")
print("Using username '%s'" % (username, ))
instance_list = []
for instance in ec2.instances.all():
if instance.state['Name'] != 'running':
continue
name = u.get_name(instance.tags)
if (fragment in name or fragment in instance.public_ip_address
or fragment in instance.id
or fragment in instance.private_ip_address):
print("Uninitializing %s %s %s" %
(name, instance.public_ip_address,
instance.private_ip_address))
key_file = u.get_keypair_fn(instance.key_name)
ssh_client = u.SshClient(hostname=instance.public_ip_address,
ssh_key=key_file,
username=username)
ssh_client.run('rm /tmp/is_initialized || echo "failed 1"')
ssh_client.run('rm /tmp/nv_setup_complete || echo "failed 2"')
ssh_client.run('rm *.sh') # remove install scripts
def __configure_kafka(self):
"""Modifies kube config items for Kafka based on the internal configuration
Currently, the updates to kafka include:
- Configuring Kafka to use a persistent volume claim
- Modifying the Kafka metrics container to use the correct version of the Kapture container
"""
if self.__config["usePersistentVolume"]:
volumes = self.kafka["spec"]["template"]["spec"]["volumes"]
empty_dir = volumes.pop(0)
volumes.append({
"name": empty_dir["name"],
"persistentVolumeClaim": {
"claimName": get_name(self.kafka_pvc)
},
})
self.kafka_metrics["spec"]["template"]["spec"]["containers"][0][
"image"] = ("carbonrelay/kapture:" +
self.__config["kapture_version"])
def updatefile(id):
body = request.data.decode("utf-8")
content = json.loads(body)['content']
if not id_exists(id):
return 'File does not exist'
try:
write_file(id, content)
except OSError as error:
return new_error(str(error))
return json.dumps({
'status': 'Ok',
'file': {
'id': id,
'name': get_name(content)
}
})
def list_ebss():
"""Print list of instances with their attached volume id/size to console, ie
master-us-east-1a.masters.df86c4e8-pachydermcluster.kubernetes.com: vol-0f0e841d0cc657002 (20),vol-06fb03280cf2598fb (20),vol-0e7ef0896b234db53 (64)
nodes.df86c4e8-pachydermcluster.kubernetes.com: vol-012367900cd8dae8c (128)
nodes.df86c4e8-pachydermcluster.kubernetes.com: vol-0a98ee5f7f155b2b7 (128),vol-048e29f604d2900a7 (100)
imagenet: vol-024347797a6ab11e8 (1500)
api_service_prod: vol-0c36c9f21bb6be8a6 (8)
box00.gpubox.0: vol-0c69c68295a89cde5 (50)
"""
ec2 = u.create_ec2_resource()
instances = [(u.seconds_from_datetime(i.launch_time), i)
for i in ec2.instances.all()]
sorted_instances = sorted(instances, key=itemgetter(0))
for (seconds, instance) in sorted_instances:
volumes = instance.volumes.all()
volume_strs = []
for v in volumes:
volume_strs.append("%s (%s)" % (v.id, v.size))
print("%s: %s" % (u.get_name(instance.tags), ','.join(volume_strs)))
def __init__(self, name=None, dtype=None, shape=()):
# Attributes related to a single stage
self.name = util.get_name("stage", name)
self.stmt_stack = [[]]
self.var_dict = {}
self.axis_list = []
self.has_break = False
self.has_return = False
self.ret_dtype = None
self.for_level = 0
self.for_ID = 0
# Attributes for cross-stage relation
self.input_stages = set([])
self.lhs_tensors = set([])
self.last_substages = set([])
self.name_with_prefix = self.name if Stage.get_len() == 0 \
else Stage.get_current().name_with_prefix + "." + self.name
# Private attributes for building a stage
self._op = None
self._dtype = util.get_dtype(dtype, self.name_with_prefix)
self._buf = decl_buffer(shape, self._dtype, self.name)
self._shape = self._buf.shape
def post(self, request, format=None):
form = ShortClientForm(request.POST)
if not form.is_valid():
return Response({'errors': get_errors(form)[0]})
self.get_initial_data()
self.user.first_name, self.user.last_name = get_name(
form.cleaned_data['first_name'])
self.user.save()
client = self.user.client
if form.cleaned_data['about']:
client.about = form.cleaned_data['about']
client.save()
if form.cleaned_data['location']:
location = Location(title=form.cleaned_data['location'],
address_line1=form.cleaned_data['location'])
location.google_maps_request(force=True)
location.save()
ClientLocation.objects.create(client=client, location=location)
return Response({'url': request.POST.get('next_url', '')})
def resync(self):
procs = []
for sync in self.sync:
instance = u.get_instances(args.name, verbose=False)[0]
print("Syncing with ", u.get_name(instance))
command = sync.command(instance)
popen = subprocess.Popen(command)
procs.append({
'popen': popen,
'command': command,
})
# Wait
for proc in procs:
print(proc["command"])
proc['popen'].communicate()
for proc in procs:
if proc['popen'].returncode != 0:
raise Error('Bad returncode from %s: %d', proc['command'],
proc['popen'].returncode)
logger.info('Resync %d complete', self.counter)
self.counter += 1
def delete(self):
"""Remove the Redis resources from the cluster
"""
evaluate_request(
self.__v1_api.delete_namespaced_pod(
namespace=self.__namespace,
name=get_name(self.redis_master),
async_req=True,
))
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.redis_service),
async_req=True,
))
evaluate_request(
self.__v1_api.delete_namespaced_replication_controller(
namespace=self.__namespace,
name=get_name(self.redis_slave_controller),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.__v1_api.delete_namespaced_replication_controller(
namespace=self.__namespace,
name=get_name(self.redis_sentinel_controller),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.redis_metrics_service),
async_req=True,
))
evaluate_request(
self.__v1_apps_api.delete_namespaced_deployment(
namespace=self.__namespace,
name=get_name(self.redis_connector),
async_req=True,
propagation_policy="Background",
))
def delete(self):
"""Delete all Kafka artifacts within the cluster
Removes all the Kafka artifacts within the cluster safely - if something does not already
exist within the cluster, it will be ignored.
"""
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.kafka_service),
async_req=True,
))
evaluate_request(
self.__v1_policy_api.delete_namespaced_pod_disruption_budget(
namespace=self.__namespace,
name=get_name(self.kafka_pdb),
async_req=True,
))
evaluate_request(
self.__v1_apps_api.delete_namespaced_stateful_set(
namespace=self.__namespace,
name=get_name(self.kafka),
async_req=True,
propagation_policy="Background",
))
evaluate_request(
self.__v1_api.delete_namespaced_service(
namespace=self.__namespace,
name=get_name(self.kafka_metrics_service),
async_req=True,
))
evaluate_request(
self.__v1_apps_api.delete_namespaced_deployment(
namespace=self.__namespace,
name=get_name(self.kafka_metrics),
async_req=True,
))
evaluate_request(
self.__v1_api.delete_namespaced_persistent_volume_claim(
namespace=self.__namespace,
name=get_name(self.kafka_pvc),
async_req=True,
))
import os
import sys
module_path = os.path.dirname(os.path.abspath(__file__))
sys.path.append(module_path + '/..')
import util as u
if __name__ == '__main__':
ec2 = u.create_ec2_resource()
assert args.zone
vols = {}
for vol in ec2.volumes.all():
if vol.availability_zone == args.zone:
vols[u.get_name(vol)] = vol
print(f"Deleting {args.replicas} replicas in {args.zone}")
for i in range(args.volume_offset, args.replicas + args.volume_offset):
vol_name = 'imagenet_%02d' % (i)
print(f"Deleting {vol_name}")
if not vol_name in vols:
print(" Not found")
continue
vol = vols[vol_name]
assert vol.volume_type == 'io1', "Safety check to prevent killing XView volumes (they are gp2)"
if not args.dryrun:
try:
vol.delete()
except Exception as e:
print(f"Failed with {e}")
def validate_cache(url):
print("Validating Cache of [" + url + "]")
if url not in cachedata["valid_caches"]:
cachedata["valid_caches"].append(url)
cachedata["name_mappings"][url] = util.get_name(url)
def create(self):
"""Create kafka items in the cluster.
Creates all the parts of the kafka deployment in the cluster. If items already exist in
the cluster, those items will be quietly ignored.
"""
if self.__config["usePersistentVolume"]:
evaluate_request(
self.__v1_api.create_namespaced_persistent_volume_claim(
namespace=self.__namespace,
body=self.kafka_pvc,
async_req=True),
allowed_statuses=[409],
)
else:
evaluate_request(
self.__v1_api.delete_namespaced_persistent_volume_claim(
namespace=self.__namespace,
name=get_name(self.kafka_pvc),
async_req=True,
))
evaluate_request(
self.__v1_api.create_namespaced_service(namespace=self.__namespace,
body=self.kafka_service,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_policy_api.create_namespaced_pod_disruption_budget(
namespace=self.__namespace,
body=self.kafka_pdb,
async_req=True),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_apps_api.create_namespaced_stateful_set(
namespace=self.__namespace, body=self.kafka, async_req=True),
allowed_statuses=[409],
)
# Wait for Kafka to register itself with zookeeper before proceeding
kafka_started = False
while not kafka_started:
try:
out = stream(
self.__v1_api.connect_get_namespaced_pod_exec,
"zk-0",
self.__namespace,
command=[
"bash",
"-c",
"echo dump | nc localhost 2181 | grep brokers",
],
stderr=False,
stdin=False,
stdout=True,
tty=False,
)
kafka_started = len(out.split()) == 3
except Exception as _:
sleep(2)
evaluate_request(
self.__v1_api.create_namespaced_service(
namespace=self.__namespace,
body=self.kafka_metrics_service,
async_req=True,
),
allowed_statuses=[409],
)
evaluate_request(
self.__v1_apps_api.create_namespaced_deployment(
namespace=self.__namespace,
body=self.kafka_metrics,
async_req=True),
allowed_statuses=[409],
)
def add_storage_account(self):
"""
Creates a storage account then adds the storage account to the vault to manage its keys.
"""
from azure.mgmt.storage import StorageManagementClient
from azure.mgmt.storage.models import StorageAccountCreateParameters, Sku, SkuName, Kind
from msrestazure.azure_active_directory import AADTokenCredentials
from azure.mgmt.authorization.models import RoleAssignmentCreateParameters
from azure.keyvault.models import StorageAccountAttributes
from azure.mgmt.authorization import AuthorizationManagementClient
self.config.storage_account_name = get_name('sa', '')
# only user accounts with access to the storage account keys can add a storage account to
# a vault. For this reason this sample creates the storage account with a user account
# authenticated through device login rather than the service principal credentials used
# in other samples.
# create the StorageManagementClient with user token credentials
user_token_creds = AADTokenCredentials(
self.get_user_token('https://management.core.windows.net/'))
print('creating storage account %s' % self.config.storage_account_name)
storage_mgmt_client = StorageManagementClient(
user_token_creds, self.config.subscription_id)
# create the storage account
sa_params = StorageAccountCreateParameters(
sku=Sku(SkuName.standard_ragrs),
kind=Kind.storage,
location=self.config.location)
sa = storage_mgmt_client.storage_accounts.create(
resource_group_name=self.config.group_name,
account_name=self.config.storage_account_name,
parameters=sa_params).result()
# the KeyVault service must be given the "Storage Account Key Operator Service Role" on the
# storage account before the storage account can be added to the vault
print(
'granting Azure Key Vault the "Storage Account Key Operator Service Role" on the storage account'
)
# find the role definition for "Storage Account Key Operator Service Role"
filter_str = 'roleName eq \'Storage Account Key Operator Service Role\''
authorization_mgmt_client = AuthorizationManagementClient(
user_token_creds, self.config.subscription_id)
role_id = list(
authorization_mgmt_client.role_definitions.list(
scope='/', filter=filter_str))[0].id
# create a role assignment granting the key vault service principal this role
role_params = RoleAssignmentCreateParameters(
role_definition_id=role_id,
# the Azure Key Vault service id
principal_id='93c27d83-f79b-4cb2-8dd4-4aa716542e74')
authorization_mgmt_client.role_assignments.create(
scope=sa.id,
role_assignment_name=str(uuid.uuid4()),
parameters=role_params)
# since the set_storage_account can only be called by a user account with access to the keys of
# the storage account, we grant the user that created the storage account access to the vault
# and add the storage account to the vault
vault = self.get_sample_vault()
# grant the user access the vault
self.grant_access_to_sample_vault(vault, self._user_oid)
# add the storage account to the vault using the users KeyVaultClient
print('adding storage acount %s to vault %s' %
(self.config.storage_account_name, vault.name))
attributes = StorageAccountAttributes(enabled=True)
self.keyvault_user_client.set_storage_account(
vault_base_url=self.sample_vault_url,
storage_account_name=sa.name,
resource_id=sa.id,
active_key_name='key1',
auto_regenerate_key=True,
regeneration_period='P30D',
storage_account_attributes=attributes)
def desc(route_table):
return "%s (%s)" % (route_table.id, u.get_name(route_table.tags))
def desc(security_group):
return "%s (%s, %s)" % (security_group.id,
u.get_name(security_group.tags),
security_group.group_name)
def compute(shape, fcompute, name=None, dtype=None, attrs=OrderedDict()):
"""Construct a new tensor based on the shape and the compute function.
The API **returns a new tensor**. The shape must be a tuple. The number of
elements in the tuple decides the dimension of the returned tensor. The
second field `fcompute` defines the construction rule of the returned
tensor, which must be callable. The number of arguments should match the
dimension defined by `shape`, which *we do not check*. This, however,
provides users more programming flexibility.
The compute function specifies how we calculate each element of the
returned tensor. It can contain other HeteroCL APIs, even imperative DSL.
Parameters
----------
shape : tuple
The shape of the returned tensor
fcompute : callable
The construction rule for the returned tensor
name : str, optional
The name of the returned tensor
dtype : Type, optional
The data type of the placeholder
Returns
-------
Tensor
Examples
--------
.. code-block:: python
# example 1.1 - anonymous lambda function
A = hcl.compute((10, 10), lambda x, y: x+y)
# equivalent code
for x in range(0, 10):
for y in range(0, 10):
A[x][y] = x + y
# example 1.2 - explicit function
def addition(x, y):
return x+y
A = hcl.compute((10, 10), addition)
# example 1.3 - imperative function definition
@hcl.def_([(), ()])
def addition(x, y):
hcl.return_(x+y)
A = hcl.compute((10, 10), addition)
# example 2 - undetermined arguments
def compute_tanh(X):
return hcl.compute(X.shape, lambda *args: hcl.tanh(X[args]))
A = hcl.placeholder((10, 10))
B = hcl.placeholder((10, 10, 10))
tA = compute_tanh(A)
tB = compute_tanh(B)
# example 3 - mixed-paradigm programming
def return_max(x, y):
with hcl.if_(x > y):
hcl.return_(x)
with hcl.else_:
hcl.return_(y)
A = hcl.compute((10, 10), return_max)
"""
# check API correctness
if not isinstance(shape, tuple):
raise APIError("The shape of compute API must be a tuple")
# properties for the returned tensor
shape = CastRemover().mutate(shape)
name = get_name("compute", name)
# prepare the iteration variables
args, nargs = process_fcompute(fcompute, shape)
lambda_ivs = [_IterVar((0, shape[n]), args[n], 0) for n in range(0, nargs)]
# call the helper function that returns a new tensor
tensor = compute_body(name,
lambda_ivs,
fcompute,
shape,
dtype,
attrs=attrs)
return tensor
def validate_cache(url):
print("Validating Cache of ["+url+"]")
if url not in cachedata["valid_caches"]:
cachedata["valid_caches"].append(url)
cachedata["name_mappings"][url]=util.get_name(url)
