def generate_tags_and_instance_id(key, config, prefix="", suffix="", dynamic=True): """Generate properly formatted Azure tags and instance_id. Args: config.id_labels = {(resource_id, node_name): tags} or None Returns: tags (list), instance_id (str) """ if not config.get("id_labels"): keys = TAG_KEYS.get(key) tags = [dicta(key=key, v=generate_name(config)) for key in keys] instance_id = generate_instance_id(key, config) else: id_label_key = random.choice(list(config.id_labels.keys())) tag_key_list = random.choice(config.id_labels.get(id_label_key)) SEEN_KEYS = set() tags = [] for key, value in tag_key_list: if key not in SEEN_KEYS: tags.append(dicta(key=key, v=value)) SEEN_KEYS.update([key]) _, node_name = id_label_key instance_id = generate_instance_id(key, config, node_name) return tags, instance_id
def validate_config(self, config): """ Validates that all known parts of a config are the required types Params: config : dicta - the configuration to test Returns: bool """ validator = dicta( payer_account=str, start_date=date, end_date=date, storage_classes=list, max_name_words=int, max_resource_id_length=int, max_bandwidth_gens=int, max_sql_gens=int, max_storage_gens=int, max_vmachine_gens=int, max_vnetwork_gens=int, ) result = [ f"{k} Must be of type {validator[k].__name__}" for k in validator if k in config and not isinstance(config[k], validator[k]) ] if result: raise TypeError(os.linesep.join(result)) return True
def validate_config(self, config): """ Validate that all known parts of a config are the required types. Params: config : dicta - the configuration to test Returns: bool """ validator = dicta( start_date=date, end_date=date, payer_account=int, max_account_id_length=int, max_users=int, max_name_words=int, max_resource_id_length=int, max_data_transfer_gens=int, max_ebs_gens=int, max_ec2_gens=int, max_rds_gens=int, max_route53_gens=int, max_s3_gens=int, max_vpc_gens=int, ) result = [ f"{k} Must be of type {validator[k].__name__}" for k in validator if k in config and not isinstance(config[k], validator[k]) ] if result: raise TypeError(os.linesep.join(result)) return True
def default_config(self): """ Generate a config object with all values set to defaults. Returns: dicta """ default_date = date.today() last_day_of_month = monthrange(default_date.year, default_date.month)[1] return dicta( start_date=default_date.replace(day=1) - relativedelta(months=1), end_date=default_date.replace(day=last_day_of_month), storage_classes=["gp2"], max_name_words=2, max_resource_id_length=10, max_nodes=1, max_node_cpu_cores=1, max_node_memory_gig=2, max_node_namespaces=1, max_node_namespace_pods=1, min_node_namespace_pod_seconds=300, max_node_namespace_pod_seconds=3600, max_node_namespace_pod_labels=1, max_node_namespace_volumes=1, max_node_namespace_volume_request_gig=20, max_node_namespace_volume_labels=1, max_node_namespace_volume_volume_claims=1, max_node_namespace_volume_volume_claim_labels=1, max_node_namespace_volume_volume_claim_capacity_gig=20, )
def default_config(self): """ Generate a config object with all values set to defaults. Returns: dicta """ default_date = date.today() last_day_of_month = monthrange(default_date.year, default_date.month)[1] return dicta( start_date=default_date.replace(day=1) - relativedelta(months=1), end_date=default_date.replace(day=last_day_of_month), payer_account=9999999999999, max_account_id_length=13, max_users=1, max_name_words=2, max_resource_id_length=10, max_data_transfer_gens=1, max_ebs_gens=1, max_ec2_gens=1, max_rds_gens=1, max_route53_gens=1, max_s3_gens=1, max_vpc_gens=1, )
def generate_tags(key, config, prefix="", suffix="", dynamic=True): """Generate properly formatted AWS tags. Returns: list """ keys = RESOURCE_TAG_COLS.get(key) return [dicta(key=key, v=generate_name(config)) for key in keys]
def initialize_dicta(key, config): """Return dicta with common attributes.""" resource_id, tags = generate_resource_id_and_tag(config, key) return dicta( start_date=str(config.start_date), end_date=str(config.end_date), resource_id=resource_id, product_sku=FAKER.pystr(min_chars=12, max_chars=12).upper(), tags=tags, )
def build_data(self, config, _random=False): # noqa: C901 """ """ LOG.info("Data build starting") data = dicta( payer=config.payer_account, bandwidth_gens=[], sql_gens=[], storage_gens=[], vmachine_gens=[], vnetwork_gens=[], ) max_bandwidth_gens = FAKER.random_int( 0, config.max_bandwidth_gens ) if _random else config.max_bandwidth_gens max_sql_gens = FAKER.random_int( 0, config.max_sql_gens) if _random else config.max_sql_gens max_storage_gens = FAKER.random_int( 0, config.max_storage_gens) if _random else config.max_storage_gens max_vmachine_gens = FAKER.random_int( 0, config.max_vmachine_gens) if _random else config.max_vmachine_gens max_vnetwork_gens = FAKER.random_int( 0, config.max_vnetwork_gens) if _random else config.max_vnetwork_gens LOG.info(f"Building {max_bandwidth_gens} Bandwidth generators ...") for _ in range(max_bandwidth_gens): data.bandwidth_gens.append( generate_azure_dicta(config, "bandwidth")) LOG.info(f"Building {max_sql_gens} SQL generators ...") for _ in range(max_sql_gens): data.sql_gens.append(generate_azure_dicta(config, "sql")) LOG.info(f"Building {max_storage_gens} Storage generators ...") for _ in range(max_storage_gens): data.storage_gens.append(generate_azure_dicta(config, "storage")) LOG.info( f"Building {max_vmachine_gens} Virtual Machine generators ...") for _ in range(max_vmachine_gens): data.vmachine_gens.append(generate_azure_dicta(config, "vmachine")) LOG.info( f"Building {max_vnetwork_gens} Virtual Network generators ...") for _ in range(max_vnetwork_gens): data.vnetwork_gens.append(generate_azure_dicta(config, "vnetwork")) return data
def generate_azure_dicta(config, key): """Return dicta with common attributes.""" tags, instance_id = generate_tags_and_instance_id(key, config) rate = round(random.uniform(0.1, 0.50), 5) usage = round(random.uniform(0.01, 1), 5) return dicta( start_date=str(config.start_date), end_date=str(config.end_date), instance_id=instance_id, meter_id=str(uuid4()), resource_location=random.choice(RESOURCE_LOCATIONS), usage_quantity=usage, resource_rate=rate, pre_tax_cost=usage * rate, tags=tags, )
def default_config(self): """ Generate a config object with all values set to defaults Returns: dicta """ default_date = date.today() last_day_of_month = monthrange(default_date.year, default_date.month)[1] return dicta( payer_account="657f539b-7f89-4b2c-8833-f73a4654a3bc", start_date=default_date.replace(day=1) - relativedelta(months=1), end_date=default_date.replace(day=last_day_of_month), max_name_words=2, max_resource_id_length=10, max_bandwidth_gens=1, max_sql_gens=1, max_storage_gens=1, max_vmachine_gens=1, max_vnetwork_gens=1, )
def validate_config(self, config): """ Validate that all known parts of a config are the required types. Params: config : dicta - the configuration to test Returns: bool """ validator = dicta( start_date=date, end_date=date, storage_classes=list, max_name_words=int, max_resource_id_length=int, max_nodes=int, max_node_cpu_cores=int, max_node_memory_gig=int, max_node_namespaces=int, max_node_namespace_pods=int, min_node_namespace_pod_seconds=int, max_node_namespace_pod_seconds=int, max_node_namespace_pod_labels=int, max_node_namespace_volumes=int, max_node_namespace_volume_request_gig=int, max_node_namespace_volume_labels=int, max_node_namespace_volume_volume_claims=int, max_node_namespace_volume_volume_claim_labels=int, max_node_namespace_volume_volume_claim_capacity_gig=int, ) result = [ f"{k} Must be of type {validator[k].__name__}" for k in validator if k in config and not isinstance(config[k], validator[k]) ] if result: raise TypeError(os.linesep.join(result)) return True
def generate_resource_id_and_tag(config, key): """Generate properly formatted AWS tags and resource_id. Args: config.id_labels = {(resource_id, node_name): tags} or None Returns: resource_id (str), tags (list) """ if not config.get("id_labels"): resource_id = FAKER.ean8() tags = generate_tags(key, config) else: id_label_key = random.choice(list(config.id_labels.keys())) tag_key_list = random.choice(config.id_labels.get(id_label_key)) SEEN_KEYS = set() tags = [] for key, value in tag_key_list: if key not in SEEN_KEYS: tags.append(dicta(key=f"resourceTags/user:{key}", v=value)) SEEN_KEYS.update([key]) resource_id, _ = id_label_key return resource_id, tags
def test_dicta(self): """ Test dicta class """ td = dicta() self.assertEqual(len(td), 0) td.test = 1 self.assertEqual(len(td), 1) self.assertEqual(td.test, td["test"]) td.test = 2 self.assertEqual(td.test, 2) td2 = td.copy() self.assertTrue(isinstance(td2, dicta)) self.assertEqual(td2, td) with self.assertRaises(KeyError): td.x del td.test self.assertEqual(len(td), 0)
def build_data(self, config, _random=False): # noqa: C901 """ Build a structure to fill out a nise yaml template. Struture has the form of: {start_date: date, (config.start_date) ens_date: date, (config.end_date) nodes: [ (number of nodes controlled by config.max_nodes) {node_name: str, (dynamic) cpu_cores: int, (config.max_node_cpu_cores) memory_gig: int, (config.max_node_memory_gig) resource_id: str, (dynamic) namespaces: [ (number of namespaces controlled by config.max_node_namespaces) {namespace: str, (dynamic) pods: [ (number of pods controlled by config.max_node_namespace_pods) pod_name: str, (dynamic) cpu_request: int, (config.max_node_namespace_pod_cpu_request) mem_request_gig: int, (config.max_node_namespace_pod_mem_request_gig) cpu_limit: int, (config.max_node_namespace_pod_cpu_limit) mem_limit_gig: int, (config.max_node_namespace_pod_mem_limit_gig) pod_seconds: int, (config.max_node_namespace_pod_seconds) labels: str (dynamic) ], volumes: [ volume_name: str, storage_class: str, volume_request_gig: int, labels: str, volume_claims: [ volume_claim_name: str, pod_name: str, labels: str, capacity_gig: int ] ]} ]} ]} Parameters: config : dicta Returns: dicta """ LOG.info("Data build starting") data = dicta(start_date=str(config.start_date), end_date=str(config.end_date), nodes=[], resourceid_labels=None) resourceid_labels = {} if _random: max_nodes = FAKER.random_int(1, config.max_nodes) else: max_nodes = config.max_nodes for node_ix in range(max_nodes): LOG.info(f"Building node {node_ix + 1}/{max_nodes}...") if _random: cores = FAKER.random_int(1, config.max_node_cpu_cores) memory = FAKER.random_int(1, config.max_node_memory_gig) else: cores = config.max_node_cpu_cores memory = config.max_node_memory_gig resource_id = generate_resource_id(config) node_name = generate_name(config) id_label_key = (resource_id, node_name) resourceid_labels[id_label_key] = [] node = dicta(name=node_name, cpu_cores=cores, memory_gig=memory, resource_id=resource_id, namespaces=[]) data.nodes.append(node) if _random: max_namespaces = FAKER.random_int(1, config.max_node_namespaces) else: max_namespaces = config.max_node_namespaces for namespace_ix in range(max_namespaces): LOG.info( f"Building node {node_ix + 1}/{max_nodes}; namespace {namespace_ix + 1}/{max_namespaces}..." ) namespace = dicta(name=generate_name(config, prefix=node.name), pods=[], volumes=[]) node.namespaces.append(namespace) if _random: max_pods = FAKER.random_int(1, config.max_node_namespace_pods) else: max_pods = config.max_node_namespace_pods LOG.info(f"Building {max_pods} pods...") for pod_ix in range(max_pods): if _random: cpu_req = FAKER.random_int(1, node.cpu_cores) mem_req = FAKER.random_int(1, node.memory_gig) cpu_lim = FAKER.random_int(1, node.cpu_cores) mem_lim = FAKER.random_int(1, node.memory_gig) pod_sec = FAKER.random_int( config.min_node_namespace_pod_seconds, config.max_node_namespace_pod_seconds, step=(config.max_node_namespace_pod_seconds // 10) or 1800, ) else: cpu_lim = cpu_req = node.cpu_cores mem_lim = mem_req = node.memory_gig pod_sec = config.max_node_namespace_pod_seconds pod_labels = generate_labels( config.max_node_namespace_pod_labels) resourceid_labels[id_label_key].append(pod_labels) pod = dicta( name=generate_name(config, prefix=namespace.name + "-pod", suffix=str(pod_ix), dynamic=False), cpu_request=cpu_req, mem_request_gig=mem_req, cpu_limit=cpu_lim, mem_limit_gig=mem_lim, pod_seconds=pod_sec, labels=pod_labels, ) namespace.pods.append(pod) if _random: max_volumes = FAKER.random_int( 1, config.max_node_namespace_volumes) else: max_volumes = config.max_node_namespace_volumes LOG.info(f"Building {max_volumes} volumes...") for volume_ix in range(max_volumes): if _random: storage_cls = config.storage_classes[FAKER.random_int( 0, len(config.storage_classes) - 1)] vol_req = FAKER.random_int( 1, config.max_node_namespace_volume_request_gig) else: storage_cls = config.storage_classes[0] vol_req = config.max_node_namespace_volume_request_gig volume_labels = generate_labels( config.max_node_namespace_volume_labels) resourceid_labels[id_label_key].append(volume_labels) volume = dicta( name=generate_name(config, prefix=namespace.name + "-vol", suffix=str(volume_ix), dynamic=False), storage_class=storage_cls, volume_request_gig=vol_req, labels=volume_labels, volume_claims=[], ) namespace.volumes.append(volume) if _random: max_volume_claims = FAKER.random_int( 1, config.max_node_namespace_volume_volume_claims) else: max_volume_claims = config.max_node_namespace_volume_volume_claims for volume_claim_ix in range(max_volume_claims): if _random: cap = FAKER.random_int( 1, config. max_node_namespace_volume_volume_claim_capacity_gig ) else: cap = config.max_node_namespace_volume_volume_claim_capacity_gig pod_name = namespace.pods[-1 if volume_claim_ix >= len( namespace.pods) else volume_claim_ix].name volume_claim_labels = generate_labels( config. max_node_namespace_volume_volume_claim_labels) resourceid_labels[id_label_key].append( volume_claim_labels) volume_claim = dicta( name=generate_name( config, prefix=namespace.name + "-vol-claim", suffix=str(volume_claim_ix), dynamic=False, ), pod_name=pod_name, labels=volume_claim_labels, capacity_gig=cap, ) volume.volume_claims.append(volume_claim) data.resourceid_labels = resourceid_labels return data
def setUp(self): """Setup the test.""" self.dc = dicta(max_resource_id_length=10, max_name_words=2)
def build_data(self, config, _random=False): # noqa: C901 """Build the data.""" LOG.info("Data build starting") data = dicta( payer=config.payer_account, data_transfer_gens=[], ebs_gens=[], ec2_gens=[], rds_gens=[], route53_gens=[], s3_gens=[], vpc_gens=[], users=[], ) max_data_transfer_gens = (FAKER.random_int( 0, config.max_data_transfer_gens) if _random else config.max_data_transfer_gens) max_ebs_gens = FAKER.random_int( 0, config.max_ebs_gens) if _random else config.max_ebs_gens max_ec2_gens = FAKER.random_int( 0, config.max_ec2_gens) if _random else config.max_ec2_gens max_rds_gens = FAKER.random_int( 0, config.max_rds_gens) if _random else config.max_rds_gens max_route53_gens = FAKER.random_int( 0, config.max_route53_gens) if _random else config.max_route53_gens max_s3_gens = FAKER.random_int( 0, config.max_s3_gens) if _random else config.max_s3_gens max_vpc_gens = FAKER.random_int( 0, config.max_vpc_gens) if _random else config.max_vpc_gens max_users = FAKER.random_int( 0, config.max_users) if _random else config.max_users LOG.info( f"Building {max_data_transfer_gens} data transfer generators ...") for _ in range(max_data_transfer_gens): _rate, _amount = RATE_AMT.get("DTG") data_transfer_gen = initialize_dicta("DTG", config) data_transfer_gen.update(amount=round(next(_amount), 5), rate=round(next(_rate), 5)) data.data_transfer_gens.append(data_transfer_gen) LOG.info(f"Building {max_ebs_gens} EBS generators ...") for _ in range(max_ebs_gens): _rate, _amount = RATE_AMT.get("EBS") ebs_gen = initialize_dicta("EBS", config) ebs_gen.update(amount=round(next(_amount), 5), rate=round(next(_rate), 5)) data.ebs_gens.append(ebs_gen) LOG.info(f"Building {max_ec2_gens} EC2 generators ...") for _ in range(max_ec2_gens): instance_type = random.choice(EC2_INSTANCES) ec2_gen = initialize_dicta("EC2", config) ec2_gen.update( processor_arch=instance_type.get("processor_arch"), region=random.choice(REGIONS), instance_type=instance_type, ) data.ec2_gens.append(ec2_gen) LOG.info(f"Building {max_rds_gens} RDS generators ...") for _ in range(max_rds_gens): instance_type = random.choice(RDS_INSTANCES) rds_gen = initialize_dicta("RDS", config) rds_gen.update( processor_arch=instance_type.get("processor_arch"), region=random.choice(REGIONS), instance_type=instance_type, ) data.rds_gens.append(rds_gen) LOG.info(f"Building {max_route53_gens} Route 53 generators ...") for _ in range(max_route53_gens): route53_gen = initialize_dicta("R53", config) route53_gen.update(product_family=random.choices( ("DNS Zone", "DNS Query"), weights=[1, 10])[0]) data.route53_gens.append(route53_gen) LOG.info(f"Building {max_s3_gens} S3 generators ...") for _ in range(max_s3_gens): _rate, _amount = RATE_AMT.get("S3") s3_gen = initialize_dicta("S3", config) s3_gen.update(amount=round(next(_amount), 5), rate=round(next(_rate), 5)) data.s3_gens.append(s3_gen) LOG.info(f"Building {max_vpc_gens} VPC generators ...") for _ in range(max_vpc_gens): vpc_gen = initialize_dicta("VPC", config) data.vpc_gens.append(vpc_gen) LOG.info(f"Adding {max_users} users.") for _ in range(max_users): data.users.append(generate_account_id(config)) return data