def generate_tags_and_instance_id(key,
config,
prefix="",
suffix="",
dynamic=True):
"""Generate properly formatted Azure tags and instance_id.
Args:
config.id_labels = {(resource_id, node_name): tags} or None
Returns:
tags (list), instance_id (str)
"""
if not config.get("id_labels"):
keys = TAG_KEYS.get(key)
tags = [dicta(key=key, v=generate_name(config)) for key in keys]
instance_id = generate_instance_id(key, config)
else:
id_label_key = random.choice(list(config.id_labels.keys()))
tag_key_list = random.choice(config.id_labels.get(id_label_key))
SEEN_KEYS = set()
tags = []
for key, value in tag_key_list:
if key not in SEEN_KEYS:
tags.append(dicta(key=key, v=value))
SEEN_KEYS.update([key])
_, node_name = id_label_key
instance_id = generate_instance_id(key, config, node_name)
return tags, instance_id
def validate_config(self, config):
"""
Validates that all known parts of a config are the required types
Params:
config : dicta - the configuration to test
Returns:
bool
"""
validator = dicta(
payer_account=str,
start_date=date,
end_date=date,
storage_classes=list,
max_name_words=int,
max_resource_id_length=int,
max_bandwidth_gens=int,
max_sql_gens=int,
max_storage_gens=int,
max_vmachine_gens=int,
max_vnetwork_gens=int,
)
result = [
f"{k} Must be of type {validator[k].__name__}" for k in validator
if k in config and not isinstance(config[k], validator[k])
]
if result:
raise TypeError(os.linesep.join(result))
return True
def validate_config(self, config):
"""
Validate that all known parts of a config are the required types.
Params:
config : dicta - the configuration to test
Returns:
bool
"""
validator = dicta(
start_date=date,
end_date=date,
payer_account=int,
max_account_id_length=int,
max_users=int,
max_name_words=int,
max_resource_id_length=int,
max_data_transfer_gens=int,
max_ebs_gens=int,
max_ec2_gens=int,
max_rds_gens=int,
max_route53_gens=int,
max_s3_gens=int,
max_vpc_gens=int,
)
result = [
f"{k} Must be of type {validator[k].__name__}" for k in validator
if k in config and not isinstance(config[k], validator[k])
]
if result:
raise TypeError(os.linesep.join(result))
return True
def default_config(self):
"""
Generate a config object with all values set to defaults.
Returns:
dicta
"""
default_date = date.today()
last_day_of_month = monthrange(default_date.year,
default_date.month)[1]
return dicta(
start_date=default_date.replace(day=1) - relativedelta(months=1),
end_date=default_date.replace(day=last_day_of_month),
storage_classes=["gp2"],
max_name_words=2,
max_resource_id_length=10,
max_nodes=1,
max_node_cpu_cores=1,
max_node_memory_gig=2,
max_node_namespaces=1,
max_node_namespace_pods=1,
min_node_namespace_pod_seconds=300,
max_node_namespace_pod_seconds=3600,
max_node_namespace_pod_labels=1,
max_node_namespace_volumes=1,
max_node_namespace_volume_request_gig=20,
max_node_namespace_volume_labels=1,
max_node_namespace_volume_volume_claims=1,
max_node_namespace_volume_volume_claim_labels=1,
max_node_namespace_volume_volume_claim_capacity_gig=20,
)
def default_config(self):
"""
Generate a config object with all values set to defaults.
Returns:
dicta
"""
default_date = date.today()
last_day_of_month = monthrange(default_date.year,
default_date.month)[1]
return dicta(
start_date=default_date.replace(day=1) - relativedelta(months=1),
end_date=default_date.replace(day=last_day_of_month),
payer_account=9999999999999,
max_account_id_length=13,
max_users=1,
max_name_words=2,
max_resource_id_length=10,
max_data_transfer_gens=1,
max_ebs_gens=1,
max_ec2_gens=1,
max_rds_gens=1,
max_route53_gens=1,
max_s3_gens=1,
max_vpc_gens=1,
)
def generate_tags(key, config, prefix="", suffix="", dynamic=True):
"""Generate properly formatted AWS tags.
Returns:
list
"""
keys = RESOURCE_TAG_COLS.get(key)
return [dicta(key=key, v=generate_name(config)) for key in keys]
def initialize_dicta(key, config):
"""Return dicta with common attributes."""
resource_id, tags = generate_resource_id_and_tag(config, key)
return dicta(
start_date=str(config.start_date),
end_date=str(config.end_date),
resource_id=resource_id,
product_sku=FAKER.pystr(min_chars=12, max_chars=12).upper(),
tags=tags,
)
def build_data(self, config, _random=False): # noqa: C901
"""
"""
LOG.info("Data build starting")
data = dicta(
payer=config.payer_account,
bandwidth_gens=[],
sql_gens=[],
storage_gens=[],
vmachine_gens=[],
vnetwork_gens=[],
)
max_bandwidth_gens = FAKER.random_int(
0, config.max_bandwidth_gens
) if _random else config.max_bandwidth_gens
max_sql_gens = FAKER.random_int(
0, config.max_sql_gens) if _random else config.max_sql_gens
max_storage_gens = FAKER.random_int(
0, config.max_storage_gens) if _random else config.max_storage_gens
max_vmachine_gens = FAKER.random_int(
0,
config.max_vmachine_gens) if _random else config.max_vmachine_gens
max_vnetwork_gens = FAKER.random_int(
0,
config.max_vnetwork_gens) if _random else config.max_vnetwork_gens
LOG.info(f"Building {max_bandwidth_gens} Bandwidth generators ...")
for _ in range(max_bandwidth_gens):
data.bandwidth_gens.append(
generate_azure_dicta(config, "bandwidth"))
LOG.info(f"Building {max_sql_gens} SQL generators ...")
for _ in range(max_sql_gens):
data.sql_gens.append(generate_azure_dicta(config, "sql"))
LOG.info(f"Building {max_storage_gens} Storage generators ...")
for _ in range(max_storage_gens):
data.storage_gens.append(generate_azure_dicta(config, "storage"))
LOG.info(
f"Building {max_vmachine_gens} Virtual Machine generators ...")
for _ in range(max_vmachine_gens):
data.vmachine_gens.append(generate_azure_dicta(config, "vmachine"))
LOG.info(
f"Building {max_vnetwork_gens} Virtual Network generators ...")
for _ in range(max_vnetwork_gens):
data.vnetwork_gens.append(generate_azure_dicta(config, "vnetwork"))
return data
def generate_azure_dicta(config, key):
"""Return dicta with common attributes."""
tags, instance_id = generate_tags_and_instance_id(key, config)
rate = round(random.uniform(0.1, 0.50), 5)
usage = round(random.uniform(0.01, 1), 5)
return dicta(
start_date=str(config.start_date),
end_date=str(config.end_date),
instance_id=instance_id,
meter_id=str(uuid4()),
resource_location=random.choice(RESOURCE_LOCATIONS),
usage_quantity=usage,
resource_rate=rate,
pre_tax_cost=usage * rate,
tags=tags,
)
def default_config(self):
"""
Generate a config object with all values set to defaults
Returns:
dicta
"""
default_date = date.today()
last_day_of_month = monthrange(default_date.year,
default_date.month)[1]
return dicta(
payer_account="657f539b-7f89-4b2c-8833-f73a4654a3bc",
start_date=default_date.replace(day=1) - relativedelta(months=1),
end_date=default_date.replace(day=last_day_of_month),
max_name_words=2,
max_resource_id_length=10,
max_bandwidth_gens=1,
max_sql_gens=1,
max_storage_gens=1,
max_vmachine_gens=1,
max_vnetwork_gens=1,
)
def validate_config(self, config):
"""
Validate that all known parts of a config are the required types.
Params:
config : dicta - the configuration to test
Returns:
bool
"""
validator = dicta(
start_date=date,
end_date=date,
storage_classes=list,
max_name_words=int,
max_resource_id_length=int,
max_nodes=int,
max_node_cpu_cores=int,
max_node_memory_gig=int,
max_node_namespaces=int,
max_node_namespace_pods=int,
min_node_namespace_pod_seconds=int,
max_node_namespace_pod_seconds=int,
max_node_namespace_pod_labels=int,
max_node_namespace_volumes=int,
max_node_namespace_volume_request_gig=int,
max_node_namespace_volume_labels=int,
max_node_namespace_volume_volume_claims=int,
max_node_namespace_volume_volume_claim_labels=int,
max_node_namespace_volume_volume_claim_capacity_gig=int,
)
result = [
f"{k} Must be of type {validator[k].__name__}" for k in validator
if k in config and not isinstance(config[k], validator[k])
]
if result:
raise TypeError(os.linesep.join(result))
return True
def generate_resource_id_and_tag(config, key):
"""Generate properly formatted AWS tags and resource_id.
Args:
config.id_labels = {(resource_id, node_name): tags} or None
Returns:
resource_id (str), tags (list)
"""
if not config.get("id_labels"):
resource_id = FAKER.ean8()
tags = generate_tags(key, config)
else:
id_label_key = random.choice(list(config.id_labels.keys()))
tag_key_list = random.choice(config.id_labels.get(id_label_key))
SEEN_KEYS = set()
tags = []
for key, value in tag_key_list:
if key not in SEEN_KEYS:
tags.append(dicta(key=f"resourceTags/user:{key}", v=value))
SEEN_KEYS.update([key])
resource_id, _ = id_label_key
return resource_id, tags
def test_dicta(self):
"""
Test dicta class
"""
td = dicta()
self.assertEqual(len(td), 0)
td.test = 1
self.assertEqual(len(td), 1)
self.assertEqual(td.test, td["test"])
td.test = 2
self.assertEqual(td.test, 2)
td2 = td.copy()
self.assertTrue(isinstance(td2, dicta))
self.assertEqual(td2, td)
with self.assertRaises(KeyError):
td.x
del td.test
self.assertEqual(len(td), 0)
def build_data(self, config, _random=False): # noqa: C901
"""
Build a structure to fill out a nise yaml template.
Struture has the form of:
{start_date: date, (config.start_date)
ens_date: date, (config.end_date)
nodes: [ (number of nodes controlled by config.max_nodes)
{node_name: str, (dynamic)
cpu_cores: int, (config.max_node_cpu_cores)
memory_gig: int, (config.max_node_memory_gig)
resource_id: str, (dynamic)
namespaces: [ (number of namespaces controlled by config.max_node_namespaces)
{namespace: str, (dynamic)
pods: [ (number of pods controlled by config.max_node_namespace_pods)
pod_name: str, (dynamic)
cpu_request: int, (config.max_node_namespace_pod_cpu_request)
mem_request_gig: int, (config.max_node_namespace_pod_mem_request_gig)
cpu_limit: int, (config.max_node_namespace_pod_cpu_limit)
mem_limit_gig: int, (config.max_node_namespace_pod_mem_limit_gig)
pod_seconds: int, (config.max_node_namespace_pod_seconds)
labels: str (dynamic)
],
volumes: [
volume_name: str,
storage_class: str,
volume_request_gig: int,
labels: str,
volume_claims: [
volume_claim_name: str,
pod_name: str,
labels: str,
capacity_gig: int
]
]}
]}
]}
Parameters:
config : dicta
Returns:
dicta
"""
LOG.info("Data build starting")
data = dicta(start_date=str(config.start_date),
end_date=str(config.end_date),
nodes=[],
resourceid_labels=None)
resourceid_labels = {}
if _random:
max_nodes = FAKER.random_int(1, config.max_nodes)
else:
max_nodes = config.max_nodes
for node_ix in range(max_nodes):
LOG.info(f"Building node {node_ix + 1}/{max_nodes}...")
if _random:
cores = FAKER.random_int(1, config.max_node_cpu_cores)
memory = FAKER.random_int(1, config.max_node_memory_gig)
else:
cores = config.max_node_cpu_cores
memory = config.max_node_memory_gig
resource_id = generate_resource_id(config)
node_name = generate_name(config)
id_label_key = (resource_id, node_name)
resourceid_labels[id_label_key] = []
node = dicta(name=node_name,
cpu_cores=cores,
memory_gig=memory,
resource_id=resource_id,
namespaces=[])
data.nodes.append(node)
if _random:
max_namespaces = FAKER.random_int(1,
config.max_node_namespaces)
else:
max_namespaces = config.max_node_namespaces
for namespace_ix in range(max_namespaces):
LOG.info(
f"Building node {node_ix + 1}/{max_nodes}; namespace {namespace_ix + 1}/{max_namespaces}..."
)
namespace = dicta(name=generate_name(config, prefix=node.name),
pods=[],
volumes=[])
node.namespaces.append(namespace)
if _random:
max_pods = FAKER.random_int(1,
config.max_node_namespace_pods)
else:
max_pods = config.max_node_namespace_pods
LOG.info(f"Building {max_pods} pods...")
for pod_ix in range(max_pods):
if _random:
cpu_req = FAKER.random_int(1, node.cpu_cores)
mem_req = FAKER.random_int(1, node.memory_gig)
cpu_lim = FAKER.random_int(1, node.cpu_cores)
mem_lim = FAKER.random_int(1, node.memory_gig)
pod_sec = FAKER.random_int(
config.min_node_namespace_pod_seconds,
config.max_node_namespace_pod_seconds,
step=(config.max_node_namespace_pod_seconds // 10)
or 1800,
)
else:
cpu_lim = cpu_req = node.cpu_cores
mem_lim = mem_req = node.memory_gig
pod_sec = config.max_node_namespace_pod_seconds
pod_labels = generate_labels(
config.max_node_namespace_pod_labels)
resourceid_labels[id_label_key].append(pod_labels)
pod = dicta(
name=generate_name(config,
prefix=namespace.name + "-pod",
suffix=str(pod_ix),
dynamic=False),
cpu_request=cpu_req,
mem_request_gig=mem_req,
cpu_limit=cpu_lim,
mem_limit_gig=mem_lim,
pod_seconds=pod_sec,
labels=pod_labels,
)
namespace.pods.append(pod)
if _random:
max_volumes = FAKER.random_int(
1, config.max_node_namespace_volumes)
else:
max_volumes = config.max_node_namespace_volumes
LOG.info(f"Building {max_volumes} volumes...")
for volume_ix in range(max_volumes):
if _random:
storage_cls = config.storage_classes[FAKER.random_int(
0,
len(config.storage_classes) - 1)]
vol_req = FAKER.random_int(
1, config.max_node_namespace_volume_request_gig)
else:
storage_cls = config.storage_classes[0]
vol_req = config.max_node_namespace_volume_request_gig
volume_labels = generate_labels(
config.max_node_namespace_volume_labels)
resourceid_labels[id_label_key].append(volume_labels)
volume = dicta(
name=generate_name(config,
prefix=namespace.name + "-vol",
suffix=str(volume_ix),
dynamic=False),
storage_class=storage_cls,
volume_request_gig=vol_req,
labels=volume_labels,
volume_claims=[],
)
namespace.volumes.append(volume)
if _random:
max_volume_claims = FAKER.random_int(
1, config.max_node_namespace_volume_volume_claims)
else:
max_volume_claims = config.max_node_namespace_volume_volume_claims
for volume_claim_ix in range(max_volume_claims):
if _random:
cap = FAKER.random_int(
1, config.
max_node_namespace_volume_volume_claim_capacity_gig
)
else:
cap = config.max_node_namespace_volume_volume_claim_capacity_gig
pod_name = namespace.pods[-1 if volume_claim_ix >= len(
namespace.pods) else volume_claim_ix].name
volume_claim_labels = generate_labels(
config.
max_node_namespace_volume_volume_claim_labels)
resourceid_labels[id_label_key].append(
volume_claim_labels)
volume_claim = dicta(
name=generate_name(
config,
prefix=namespace.name + "-vol-claim",
suffix=str(volume_claim_ix),
dynamic=False,
),
pod_name=pod_name,
labels=volume_claim_labels,
capacity_gig=cap,
)
volume.volume_claims.append(volume_claim)
data.resourceid_labels = resourceid_labels
return data
def setUp(self):
"""Setup the test."""
self.dc = dicta(max_resource_id_length=10, max_name_words=2)
def build_data(self, config, _random=False): # noqa: C901
"""Build the data."""
LOG.info("Data build starting")
data = dicta(
payer=config.payer_account,
data_transfer_gens=[],
ebs_gens=[],
ec2_gens=[],
rds_gens=[],
route53_gens=[],
s3_gens=[],
vpc_gens=[],
users=[],
)
max_data_transfer_gens = (FAKER.random_int(
0, config.max_data_transfer_gens) if _random else
config.max_data_transfer_gens)
max_ebs_gens = FAKER.random_int(
0, config.max_ebs_gens) if _random else config.max_ebs_gens
max_ec2_gens = FAKER.random_int(
0, config.max_ec2_gens) if _random else config.max_ec2_gens
max_rds_gens = FAKER.random_int(
0, config.max_rds_gens) if _random else config.max_rds_gens
max_route53_gens = FAKER.random_int(
0, config.max_route53_gens) if _random else config.max_route53_gens
max_s3_gens = FAKER.random_int(
0, config.max_s3_gens) if _random else config.max_s3_gens
max_vpc_gens = FAKER.random_int(
0, config.max_vpc_gens) if _random else config.max_vpc_gens
max_users = FAKER.random_int(
0, config.max_users) if _random else config.max_users
LOG.info(
f"Building {max_data_transfer_gens} data transfer generators ...")
for _ in range(max_data_transfer_gens):
_rate, _amount = RATE_AMT.get("DTG")
data_transfer_gen = initialize_dicta("DTG", config)
data_transfer_gen.update(amount=round(next(_amount), 5),
rate=round(next(_rate), 5))
data.data_transfer_gens.append(data_transfer_gen)
LOG.info(f"Building {max_ebs_gens} EBS generators ...")
for _ in range(max_ebs_gens):
_rate, _amount = RATE_AMT.get("EBS")
ebs_gen = initialize_dicta("EBS", config)
ebs_gen.update(amount=round(next(_amount), 5),
rate=round(next(_rate), 5))
data.ebs_gens.append(ebs_gen)
LOG.info(f"Building {max_ec2_gens} EC2 generators ...")
for _ in range(max_ec2_gens):
instance_type = random.choice(EC2_INSTANCES)
ec2_gen = initialize_dicta("EC2", config)
ec2_gen.update(
processor_arch=instance_type.get("processor_arch"),
region=random.choice(REGIONS),
instance_type=instance_type,
)
data.ec2_gens.append(ec2_gen)
LOG.info(f"Building {max_rds_gens} RDS generators ...")
for _ in range(max_rds_gens):
instance_type = random.choice(RDS_INSTANCES)
rds_gen = initialize_dicta("RDS", config)
rds_gen.update(
processor_arch=instance_type.get("processor_arch"),
region=random.choice(REGIONS),
instance_type=instance_type,
)
data.rds_gens.append(rds_gen)
LOG.info(f"Building {max_route53_gens} Route 53 generators ...")
for _ in range(max_route53_gens):
route53_gen = initialize_dicta("R53", config)
route53_gen.update(product_family=random.choices(
("DNS Zone", "DNS Query"), weights=[1, 10])[0])
data.route53_gens.append(route53_gen)
LOG.info(f"Building {max_s3_gens} S3 generators ...")
for _ in range(max_s3_gens):
_rate, _amount = RATE_AMT.get("S3")
s3_gen = initialize_dicta("S3", config)
s3_gen.update(amount=round(next(_amount), 5),
rate=round(next(_rate), 5))
data.s3_gens.append(s3_gen)
LOG.info(f"Building {max_vpc_gens} VPC generators ...")
for _ in range(max_vpc_gens):
vpc_gen = initialize_dicta("VPC", config)
data.vpc_gens.append(vpc_gen)
LOG.info(f"Adding {max_users} users.")
for _ in range(max_users):
data.users.append(generate_account_id(config))
return data