def handle_exception(error):
message = [str(x) for x in error.args]
status_code = 500
success = False
response = {
'success': success,
'error': {
'type': error.__class__.__name__,
'message': message
}
}
logger.exception(error)
logJson(response)
return jsonify(response), status_code
def ociRegion():
query_string = request.query_string
parsed_query_string = urllib.parse.unquote(query_string.decode())
query_json = json.loads(parsed_query_string)
logJson(query_json)
config_profile = query_json.get('config_profile', 'DEFAULT')
logger.info('Using Profile : {0!s:s}'.format(config_profile))
oci_regions = OCIRegions(profile=config_profile)
regions = oci_regions.list()
logger.debug(">>>>>>>>> Regions: {0!s:s}".format(regions))
return json.dumps(regions,
sort_keys=False,
indent=2,
separators=(',', ': '))
def list(self, filter={}):
services = oci.pagination.list_call_get_all_results(self.client.list_services, compartment_id=self.compartment_ocid).data
# Convert to Json object
services_json = self.toJson(services)
logger.debug(str(services_json))
# Filter results
self.services_json = self.filterJsonObjectList(services_json, filter)
logger.debug(str(self.services_json))
logJson(self.services_json)
# Build List of Service Objects that have methods for getting VCN / Security Lists / Route Tables etc
for service in self.services_json:
service['display_name'] = service['name']
return sorted(self.services_json, key=lambda k: k['display_name'])
def list(self, compartment_id=None, filter=None):
if compartment_id is None and self.compartment_id is None:
compartment_id = self.config['tenancy']
elif compartment_id is None:
compartment_id = self.compartment_id
# Add filter
if filter is None:
filter = {}
images = oci.pagination.list_call_get_all_results(
self.client.list_images, compartment_id=compartment_id).data
logger.debug(
'============================== Images Raw =============================='
)
logger.debug(str(images))
# Convert to Json object
images_json = self.toJson(images)
logJson(images_json)
# De-Duplicate
seen = []
deduplicated = []
for image in images_json:
image['sort_key'] = "{0:s} {1:s}".format(
image['operating_system'], image['operating_system_version'])
image['shapes'] = []
if image['sort_key'] not in seen:
deduplicated.append(image)
seen.append(image['sort_key'])
logger.debug(
'============================== Images De-Duplicate =============================='
)
logJson(deduplicated)
#images_json = deduplicated
# Add Shape Compatibility
# TODO: Upgade oci sdk
#shape_capabilities = OCIImageShapeCompatibility()
#for image in images_json:
# image['shapes'] = [s['shape'] for s in shape_capabilities.list(image['id'])]
# Filter results
self.images_json = self.filterJsonObjectList(images_json, filter)
logger.debug(
'============================== Images =============================='
)
logger.debug(str(self.images_json))
return self.images_json
def validateJson():
logger.debug('JSON : {0:s}'.format(str(request.json)))
if request.method == 'POST':
logJson(request.json)
# Validate input json
validator = OCIJsonValidator(request.json)
result = {
"valid": validator.validate(),
"results": validator.getResults()
}
return json.dumps(result,
sort_keys=False,
indent=2,
separators=(',', ': '))
else:
return '404'
def list(self, cluster_option_id='all', filter=None):
kubernetes_versions = self.client.get_cluster_options(
cluster_option_id).data.kubernetes_versions
logger.debug(
'============================== Kubernetes Versions Raw =============================='
)
logger.debug(str(kubernetes_versions))
# Convert to Json object
self.kubernetes_versions_json = []
for version in kubernetes_versions:
self.kubernetes_versions_json.append({
"name": version,
"version": version
})
logJson(self.kubernetes_versions_json)
return self.kubernetes_versions_json
def parseCceJson():
#logger.debug('JSON : {0:s}'.format(str(request.json)))
if request.method == 'GET':
query_string = request.query_string
parsed_query_string = urllib.parse.unquote(query_string.decode())
query_json = json.loads(parsed_query_string)
logJson(query_json)
# Import CCE
parser = OkitCceJsonParser()
response_json = parser.parse(query_json)
logJson(response_json)
return json.dumps(response_json,
sort_keys=False,
indent=2,
separators=(',', ': '))
else:
return '404'
def list(self, compartment_id=None, filter=None):
if compartment_id is None and self.compartment_id is None:
compartment_id = self.getTenancy()
elif compartment_id is None:
compartment_id = self.compartment_id
# Add filter
if filter is None:
filter = {}
shapes = oci.pagination.list_call_get_all_results(
self.client.list_shapes, compartment_id=compartment_id).data
logger.debug(
'============================== Shapes Raw =============================='
)
logger.debug(str(shapes))
# Convert to Json object
shapes_json = self.toJson(shapes)
logJson(shapes_json)
# De-Duplicate
seen = []
deduplicated = []
for shape in shapes_json:
if shape['shape'] not in seen:
shape['sort_key'] = shape['shape']
if 'ocpus' in shape:
split_shape = shape['shape'].split('.')
shape['sort_key'] = "{0:s}-{1:s}-{2:03n}-{3:03n}".format(
split_shape[0], split_shape[1], shape['ocpus'],
shape['memory_in_gbs'])
deduplicated.append(shape)
seen.append(shape['shape'])
logger.debug(
'============================== Shapes De-Duplicate =============================='
)
logJson(deduplicated)
shapes_json = deduplicated
# Filter results
self.shapes_json = self.filterJsonObjectList(shapes_json, filter)
logger.debug(
'============================== Shapes =============================='
)
logger.debug(str(self.shapes_json))
return self.shapes_json
def list(self, provider_service_id=None, filter=None):
if provider_service_id is None:
provider_service_id = self.provider_service_id
# Add filter
if filter is None:
filter = {}
virtual_circuit_bandwidth_shapes = oci.pagination.list_call_get_all_results(
self.client.
list_fast_connect_provider_virtual_circuit_bandwidth_shapes,
provider_service_id=provider_service_id).data
logger.debug(
'============================== VirtualCircuitBandwidthShapes Raw =============================='
)
logger.debug(str(virtual_circuit_bandwidth_shapes))
# Convert to Json object
virtual_circuit_bandwidth_shapes_json = self.toJson(
virtual_circuit_bandwidth_shapes)
logJson(virtual_circuit_bandwidth_shapes_json)
# De-Duplicate
#seen = []
#deduplicated = []
#for virtual_circuit_bandwidth_shape in virtual_circuit_bandwidth_shapes_json:
# if virtual_circuit_bandwidth_shape['virtual_circuit_bandwidth_shape'] not in seen:
# virtual_circuit_bandwidth_shape['sort_key'] = virtual_circuit_bandwidth_shape['virtual_circuit_bandwidth_shape']
# if 'ocpus' in virtual_circuit_bandwidth_shape:
# split_virtual_circuit_bandwidth_shape = virtual_circuit_bandwidth_shape['virtual_circuit_bandwidth_shape'].split('.')
# virtual_circuit_bandwidth_shape['sort_key'] = "{0:s}-{1:s}-{2:03n}-{3:03n}".format(split_virtual_circuit_bandwidth_shape[0], split_virtual_circuit_bandwidth_shape[1], virtual_circuit_bandwidth_shape['ocpus'], virtual_circuit_bandwidth_shape['memory_in_gbs'])
# deduplicated.append(virtual_circuit_bandwidth_shape)
# seen.append(virtual_circuit_bandwidth_shape['virtual_circuit_bandwidth_shape'])
#logger.debug('============================== VirtualCircuitBandwidthShapes De-Duplicate ==============================')
#logJson(deduplicated)
#virtual_circuit_bandwidth_shapes_json = deduplicated
# Filter results
self.virtual_circuit_bandwidth_shapes_json = self.filterJsonObjectList(
virtual_circuit_bandwidth_shapes_json, filter)
logger.debug(
'============================== VirtualCircuitBandwidthShapes =============================='
)
logger.debug(str(self.virtual_circuit_bandwidth_shapes_json))
return self.virtual_circuit_bandwidth_shapes_json
def list(self, compartment_id=None, filter=None):
if compartment_id is None:
compartment_id = self.compartment_id
resource_managers = oci.pagination.list_call_get_all_results(self.client.list_stacks, compartment_id=compartment_id).data
logger.info('Stack Count : {0:02d}'.format(len(resource_managers)))
# Convert to Json object
resource_managers_json = self.toJson(resource_managers)
logJson(resource_managers_json)
# Filter results
self.resource_managers_json = self.filterJsonObjectList(resource_managers_json, filter)
logger.debug(str(self.resource_managers_json))
# Build List of ResourceManager Objects
self.resource_managers_obj = []
for resource_manager in self.resource_managers_json:
self.resource_managers_obj.append(OCIResourceManager(self.config, self.configfile, self.profile, resource_manager))
return self.resource_managers_json
def list(self, filter={}):
regions = oci.pagination.list_call_get_all_results(self.client.list_regions).data
# Convert to Json object
regions_json = self.toJson(regions)
logger.debug(str(regions_json))
# Filter results
self.regions_json = self.filterJsonObjectList(regions_json, filter)
logger.debug(str(self.regions_json))
logJson(self.regions_json)
# Build List of Region Objects that have methods for getting VCN / Security Lists / Route Tables etc
self.regions_obj = []
for region in self.regions_json:
name_parts = region['name'].split('-')
region['display_name'] = '{0!s:s} {1!s:s}'.format(name_parts[0].upper(), name_parts[1].capitalize())
region['id'] = region['name']
self.regions_obj.append(OCIRegion(self.config, self.configfile, self.profile, region))
return sorted(self.regions_json, key=lambda k: k['display_name'], reverse=True)
def list(self,
compartment_id=None,
instance_id=None,
vnic_id=None,
filter=None):
if compartment_id is None:
compartment_id = self.compartment_id
if instance_id is None:
instance_id = self.instance_id
if vnic_id is None:
vnic_id = self.vnic_id
vnic_attachments = oci.pagination.list_call_get_all_results(
self.client.list_vnic_attachments,
compartment_id=compartment_id,
instance_id=instance_id,
vnic_id=vnic_id).data
# Convert to Json object
vnic_attachments_json = self.toJson(vnic_attachments)
logger.debug(str(vnic_attachments_json))
# Filter results
self.vnic_attachments_json = self.filterJsonObjectList(
vnic_attachments_json, filter)
logger.debug('--------------------- Vnics ----------------------')
logJson(self.vnic_attachments_json)
# Update VNic information
oci_vnics = OCIVnics(config=self.config,
configfile=self.configfile,
profile=self.profile)
for attachment in self.vnic_attachments_json:
attachment.update(oci_vnics.get(vnic_id=attachment['vnic_id']))
# Build List of Subnet Objects
self.vnic_attachments_obj = []
for vnic_attachment in self.vnic_attachments_json:
self.vnic_attachments_obj.append(
OCIVnicAttachment(self.config, self.configfile, self.profile,
vnic_attachment))
return self.vnic_attachments_json
def list(self, compartment_id=None, filter=None):
if compartment_id is None:
compartment_id = self.compartment_id
# Get Namespace
namespace = str(self.client.get_namespace().data)
logger.debug('Namespace : {0!s:s}'.format(namespace))
# Add filter to only return AVAILABLE Compartments
if filter is None:
filter = {}
object_storage_buckets_summary = oci.pagination.list_call_get_all_results(
self.client.list_buckets,
namespace_name=namespace,
compartment_id=compartment_id).data
# Convert to Json object
object_storage_buckets_summary_json = self.toJson(
object_storage_buckets_summary)
logger.debug(str(object_storage_buckets_summary_json))
# Filter results
self.object_storage_buckets_json = self.filterJsonObjectList(
object_storage_buckets_summary_json, filter)
logger.debug(str(self.object_storage_buckets_json))
# Convert Bucket Summary to Details
object_storage_buckets_json = []
for bucket_summary in self.object_storage_buckets_json:
bucket = self.client.get_bucket(bucket_summary['namespace'],
bucket_summary['name']).data
object_storage_buckets_json.append(self.toJson(bucket))
object_storage_buckets_json[-1][
'id'] = object_storage_buckets_json[-1].get(
'id', '{0!s:s}-{1!s:s}'.format(bucket_summary['namespace'],
bucket_summary['name']))
self.object_storage_buckets_json = object_storage_buckets_json
logger.debug(str(self.object_storage_buckets_json))
logJson(self.object_storage_buckets_json)
return self.object_storage_buckets_json
def list(self, compartment_id=None, filter=None):
if compartment_id is None and self.compartment_id is None:
compartment_id = self.getTenancy()
elif compartment_id is None:
compartment_id = self.compartment_id
# Add filter
if filter is None:
filter = {}
database_system_shapes = oci.pagination.list_call_get_all_results(
self.client.list_db_system_shapes,
compartment_id=compartment_id).data
logger.debug(
'============================== DatabaseSystemShapes Raw =============================='
)
logger.debug(str(database_system_shapes))
# Convert to Json object
database_system_shapes_json = self.toJson(database_system_shapes)
logJson(database_system_shapes_json)
# De-Duplicate
seen = []
deduplicated = []
for shape in database_system_shapes_json:
if shape['shape'] not in seen:
deduplicated.append(shape)
seen.append(shape['shape'])
logger.debug(
'============================== Shapes De-Duplicate =============================='
)
logJson(deduplicated)
database_system_shapes_json = deduplicated
# Filter results
self.database_system_shapes_json = self.filterJsonObjectList(
database_system_shapes_json, filter)
logger.debug(
'============================== DatabaseSystemShapes =============================='
)
logger.debug(str(self.database_system_shapes_json))
return self.database_system_shapes_json
def ociQuery():
if request.method == 'GET':
query_string = request.query_string
parsed_query_string = urllib.parse.unquote(query_string.decode())
query_json = standardiseIds(json.loads(parsed_query_string),
from_char='-',
to_char='.')
logger.debug(
'===================================== Query Json ====================================='
)
logJson(query_json)
logger.debug(
'======================================================================================'
)
config_profile = query_json.get('config_profile', 'DEFAULT')
logger.info('Using Profile : {0!s:s}'.format(config_profile))
response_json = {}
config = {'region': query_json['region']}
else:
return '404'
def list(self, image_id=None, filter=None):
if image_id is None and self.image_id is None:
image_id = self.config['tenancy']
elif image_id is None:
image_id = self.image_id
# Add filter
if filter is None:
filter = {}
compatibilities = oci.pagination.list_call_get_all_results(
self.client.list_image_shape_compatibility_entries,
image_id=image_id).data
logger.debug(
'============================== Compatibilities Raw =============================='
)
logger.debug(str(compatibilities))
# Convert to Json object
compatibilities_json = self.toJson(compatibilities)
logJson(compatibilities_json)
# De-Duplicate
#seen = []
#deduplicated = []
#for compatibility in compatibilities_json:
# compatibility['sort_key'] = "{0:s} {1:s}".format(compatibility['operating_system'], compatibility['operating_system_version'])
# if compatibility['sort_key'] not in seen:
# deduplicated.append(compatibility)
# seen.append(compatibility['sort_key'])
#logger.debug('============================== Compatibilities De-Duplicate ==============================')
#logJson(deduplicated)
#compatibilitys_json = deduplicated
# Filter results
self.compatibilities_json = self.filterJsonObjectList(
compatibilities_json, filter)
logger.debug(
'============================== Compatibilities =============================='
)
logger.debug(str(self.compatibilities_json))
return self.compatibilities_json
def parseCd3Xlsx():
if request.method == 'POST':
if 'file' in request.files:
file = request.files['file']
if file and file.filename != '':
filename = os.path.join('/okit/workspace',
secure_filename(file.filename))
logger.info("Saving Xlsx File {0!s:s}".format(filename))
file.save(filename)
# Import CD3
parser = OkitCd3ExcelParser()
response_json = parser.parse(filename)
logJson(response_json)
else:
response_json = {}
return json.dumps(response_json,
sort_keys=False,
indent=2,
separators=(',', ': '))
else:
return '404'
def list(self, compartment_id=None, filter=None):
if compartment_id is None and self.compartment_id is None:
compartment_id = self.getTenancy()
elif compartment_id is None:
compartment_id = self.compartment_id
# Add filter
if filter is None:
filter = {}
if 'lifecycle_state' not in filter:
filter['lifecycle_state'] = [
"CREATING", "ACTIVE", "INACTIVE", "UPDATING"
]
mysql_database_systems = oci.pagination.list_call_get_all_results(
self.client.list_db_systems, compartment_id=compartment_id).data
logger.debug(
'============================== MySQLDatabaseSystems Raw =============================='
)
logger.debug(str(mysql_database_systems))
# Convert to Json object
mysql_database_systems_json = self.toJson(mysql_database_systems)
logJson(mysql_database_systems_json)
# Retrieve Full data
for db_system in mysql_database_systems_json:
db_system.update(self.get(db_system["id"]))
# Trim version to just the number
db_system["mysql_version"] = db_system["mysql_version"].split(
'-')[0]
# Filter results
self.mysql_database_systems_json = self.filterJsonObjectList(
mysql_database_systems_json, filter)
logger.debug(
'============================== MySQLDatabaseSystems =============================='
)
logger.debug(str(self.mysql_database_systems_json))
return self.mysql_database_systems_json
def list(self, compartment_id=None, filter=None):
if compartment_id is None and self.compartment_id is None:
compartment_id = self.getTenancy()
elif compartment_id is None:
compartment_id = self.compartment_id
# Add filter
if filter is None:
filter = {}
mysql_versions = oci.pagination.list_call_get_all_results(self.client.list_versions, compartment_id=compartment_id).data
logger.debug('============================== MySQLVersions Raw ==============================')
logger.debug(str(mysql_versions))
# Convert to Json object
mysql_versions_json = self.toJson(mysql_versions)
logJson(mysql_versions_json)
# Filter results
self.mysql_versions_json = self.filterJsonObjectList(mysql_versions_json, filter)
logger.debug('============================== MySQLVersions ==============================')
logger.debug(str(self.mysql_versions_json))
return self.mysql_versions_json
def list(self, compartment_id=None, filter=None):
if compartment_id is None:
compartment_id = self.compartment_id
# Get Namespace
namespace = str(self.client.get_namespace().data)
logger.debug('Namespace : {0!s:s}'.format(namespace))
# Add filter to only return AVAILABLE Compartments
if filter is None:
filter = {}
object_storage_buckets = oci.pagination.list_call_get_all_results(
self.client.list_buckets,
namespace_name=namespace,
compartment_id=compartment_id).data
# Convert to Json object
object_storage_buckets_json = self.toJson(object_storage_buckets)
logger.debug(str(object_storage_buckets_json))
# Convert Bucket Summary to Details
for bucket in object_storage_buckets_json:
bucket.update(self.get(bucket["namespace"], bucket["name"]))
bucket['display_name'] = bucket.get('display_name', bucket['name'])
bucket['id'] = bucket.get(
'id', '{0!s:s}-{1!s:s}'.format(bucket['namespace'],
bucket['name']))
logger.debug(str(object_storage_buckets_json))
# Filter results
self.object_storage_buckets_json = self.filterJsonObjectList(
object_storage_buckets_json, filter)
logger.debug(str(self.object_storage_buckets_json))
logJson(self.object_storage_buckets_json)
return self.object_storage_buckets_json
def list(self, compartment_id=None, filter=None):
if compartment_id is None and self.compartment_id is None:
compartment_id = self.getTenancy()
elif compartment_id is None:
compartment_id = self.compartment_id
# Add filter
if filter is None:
filter = {}
if 'lifecycle_state' not in filter:
filter['lifecycle_state'] = [
"PROVISIONING", "AVAILABLE", "UPDATING"
]
database_systems = oci.pagination.list_call_get_all_results(
self.client.list_db_systems, compartment_id=compartment_id).data
logger.debug(
'============================== DatabaseSystems Raw =============================='
)
logger.debug(str(database_systems))
# Convert to Json object
database_systems_json = self.toJson(database_systems)
logJson(database_systems_json)
# Retrieve Full data
for db_system in database_systems_json:
db_system.update(self.get(db_system["id"]))
# Filter results
self.database_systems_json = self.filterJsonObjectList(
database_systems_json, filter)
logger.info(
'============================== DatabaseSystems =============================='
)
logger.info(str(self.database_systems_json))
return self.database_systems_json
def handle_oci_service_error(error):
status_code = 500
success = False
message = ''
code = ''
for x in error.args:
logger.info(x)
if 'opc-request-id' in x:
code = x['code']
message = x['message']
status_code = x['status']
break
logger.info(message)
response = {
'success': success,
'error': {
'type': error.__class__.__name__,
'code': code,
'message': message
}
}
logger.exception(error)
logJson(response)
return jsonify(response), status_code
def executeQuery(request_json={}, **kwargs):
response_json = {}
logger.info('Request JSON : {0:s}'.format(str(request_json)))
compartment_id = request_json['compartment_id']
filter = request_json.get('virtual_cloud_network_filter', None)
if filter == '':
filter = None
oci_compartments = OCICompartments()
compartment_json = oci_compartments.get(compartment_id=compartment_id)
oci_compartment = oci_compartments.compartments_obj[0]
# Build OKIT Response json add compartment information
response_json['compartments'] = [compartment_json]
logger.info('Compartment: {0!s:s}'.format(oci_compartment.data['name']))
# Query all Virtual Cloud Networks
oci_virtual_cloud_networks = oci_compartment.getVirtualCloudNetworkClients(
)
response_json["virtual_cloud_networks"] = oci_virtual_cloud_networks.list(
filter=filter)
# Loop through resulting json
for oci_virtual_cloud_network in oci_virtual_cloud_networks.virtual_cloud_networks_obj:
logger.info('\tVirtual Cloud Network : {0!s:s}'.format(
oci_virtual_cloud_network.data['display_name']))
# Internet Gateways
oci_internet_gateways = oci_virtual_cloud_network.getInternetGatewayClients(
)
response_json['internet_gateways'] = oci_internet_gateways.list()
for oci_internet_gateway in oci_internet_gateways.internet_gateways_obj:
logger.info('\t\tInternet Gateway : {0!s:s}'.format(
oci_internet_gateway.data['display_name']))
# Route Tables
oci_route_tables = oci_virtual_cloud_network.getRouteTableClients()
response_json['route_tables'] = oci_route_tables.list()
for oci_route_table in oci_route_tables.route_tables_obj:
logger.info('\t\tRoute Table : {0!s:s}'.format(
oci_route_table.data['display_name']))
# Security Lists
security_lists = oci_virtual_cloud_network.getSecurityListClients()
response_json['security_lists'] = security_lists.list()
for security_list in security_lists.security_lists_obj:
logger.info('\t\tSecurity List : {0!s:s}'.format(
security_list.data['display_name']))
# Subnets
subnets = oci_virtual_cloud_network.getSubnetClients()
response_json['subnets'] = subnets.list()
for subnet in subnets.subnets_obj:
logger.info('\t\tSubnet : {0!s:s}'.format(
subnet.data['display_name']))
# Query all Instances
oci_instances = oci_compartment.getInstanceClients()
response_json['instances'] = oci_instances.list(filter=filter)
oci_instance_vnics = oci_compartment.getInstanceVnicClients()
for instance in response_json['instances']:
instance['vnics'] = oci_instance_vnics.list(instance_id=instance['id'])
instance['subnet_id'] = instance['vnics'][0]['subnet_id']
# Query all Load Balancers
oci_load_balancers = oci_compartment.getLoadBalancerClients()
response_json['load_balancers'] = oci_load_balancers.list(filter=filter)
logger.debug('Response : {0:s}'.format(str(response_json)))
logJson(response_json)
response_json = standardiseJson(response_json)
logJson(response_json)
return response_json
def list(self, compartment_id=None, filter=None, exclude_oke=True):
if compartment_id is None:
compartment_id = self.compartment_id
# Add filter to only return "RUNNING", "STARTING", "STOPPING", "STOPPED" Compartments
if filter is None:
filter = {}
if 'lifecycle_state' not in filter:
filter['lifecycle_state'] = ["RUNNING", "STARTING", "STOPPING", "STOPPED"]
instances = oci.pagination.list_call_get_all_results(self.client.list_instances, compartment_id=compartment_id).data
# Convert to Json object
instances_json = self.toJson(instances)
# Filter results
self.instances_json = self.filterJsonObjectList(instances_json, filter)
# Get Volume Attachments as a single call and loop through them to see if they are associated with the instance.
volume_attachments = OCIVolumeAttachments(config=self.config, configfile=self.configfile, profile=self.profile, compartment_id=compartment_id).list()
# Get VNic Attachments as a single call and loop through them to see if they are associated with the instance.
vnic_attachments = OCIVnicAttachments(config=self.config, configfile=self.configfile, profile=self.profile, compartment_id=compartment_id).list()
# Filter out OKE Created Instances
logger.debug('Filtering out OKE Instances from list ({0!s:s}).'.format(len(self.instances_json)))
if exclude_oke:
self.instances_json = [i for i in self.instances_json if 'oke-cluster-id' not in i['metadata']]
logger.debug('Filtered OKE Instances from list ({0!s:s}).'.format(len(self.instances_json)))
for instance in self.instances_json:
# Get OS Details
if ('source_details' in instance and 'image_id' in instance['source_details']):
image = OCIImages(config=self.config, configfile=self.configfile, profile=self.profile, compartment_id=compartment_id).get(instance['source_details']['image_id'])
instance['source_details']['os'] = image['operating_system']
instance['source_details']['version'] = image['operating_system_version']
else:
instance['source_details']['os'] = ''
instance['source_details']['version'] = ''
# Decode Cloud Init Yaml
if 'metadata' in instance and 'user_data' in instance['metadata']:
# instance['metadata']['user_data'] = base64.b64decode(instance['metadata']['user_data']).decode('utf-8')
instance['metadata']['user_data'] = userDataDecode(instance['metadata']['user_data'])
# Add Attached Block Storage Volumes
instance['block_storage_volume_ids'] = [va['volume_id'] for va in volume_attachments if va['instance_id'] == instance['id']]
# Add Vnic Attachments
instance['vnics'] = [va for va in vnic_attachments if va['instance_id'] == instance['id']]
if len(instance['vnics']) > 0:
instance['primary_vnic'] = instance['vnics']
# Add first vnic as the default subnet
instance['subnet_id'] = instance['vnics'][0]['subnet_id'] if len(instance['vnics']) > 0 else ''
# Get Volume Attachments as a single call and loop through them to see if they are associated with the instance.
boot_volume_attachments = OCIBootVolumeAttachments(config=self.config, configfile=self.configfile, profile=self.profile, compartment_id=compartment_id, availability_domain=instance['availability_domain'], instance_id=instance['id']).list()
instance['boot_volume_size_in_gbs'] = boot_volume_attachments[0]['boot_volume']['size_in_gbs']
instance['is_pv_encryption_in_transit_enabled'] = boot_volume_attachments[0]['is_pv_encryption_in_transit_enabled']
# Build object list
# self.instances_obj.append(OCIInstance(self.config, self.configfile, self.profile, instance))
logJson(self.instances_json)
#logger.debug(str(self.instances_json))
return self.instances_json
def ociArtifacts(artifact):
logger.info('Artifact : {0:s}'.format(str(artifact)))
query_string = request.query_string
parsed_query_string = urllib.parse.unquote(query_string.decode())
query_json = standardiseIds(json.loads(parsed_query_string),
from_char='-',
to_char='.')
logger.debug(
'===================================== Query Json ====================================='
)
logJson(query_json)
logger.debug(
'======================================================================================'
)
config_profile = query_json.get('config_profile', 'DEFAULT')
logger.info('Using Profile : {0!s:s}'.format(config_profile))
response_json = {}
config = {'region': query_json['region']}
if artifact == 'Compartment':
logger.info('---- Processing Compartment')
oci_compartments = OCICompartments(config=config,
profile=config_profile)
response_json = oci_compartments.get(
compartment_id=query_json['compartment_id'])
elif artifact == 'AutonomousDatabase':
logger.info('---- Processing Autonomous Databases')
oci_autonomous_databases = OCIAutonomousDatabases(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_autonomous_databases.list(
filter=query_json.get('autonomous_database_filter', None))
elif artifact == 'BlockStorageVolume':
logger.info('---- Processing Block Storage Volumes')
oci_block_storage_volumes = OCIBlockStorageVolumes(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_block_storage_volumes.list(
filter=query_json.get('block_storage_volume_filter', None))
elif artifact == 'Compartments':
logger.info('---- Processing Compartments')
oci_compartments = OCICompartments(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_compartments.list(
filter=query_json.get('compartment_filter', None))
elif artifact == 'CustomerPremiseEquipment':
logger.info('---- Processing Customer Premise Equipment')
oci_cpes = OCICustomerPremiseEquipments(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_cpes.list(
filter=query_json.get('cpe_filter', None))
elif artifact == 'DatabaseSystem':
logger.info('---- Processing Database Systems')
oci_database_systems = OCIDatabaseSystems(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_database_systems.list(
filter=query_json.get('database_system_filter', None))
elif artifact == 'DynamicRoutingGateway':
logger.info('---- Processing Dynamic Routing Gateways')
oci_dynamic_routing_gateways = OCIDynamicRoutingGateways(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_dynamic_routing_gateways.list(
filter=query_json.get('dynamic_routing_gateway_filter', None))
elif artifact == 'FastConnect':
logger.info('---- Processing FastConnects')
oci_fast_connects = OCIFastConnects(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_fast_connects.list(
filter=query_json.get('fast_connect_filter', None))
elif artifact == 'FileStorageSystem':
logger.info('---- Processing File Storage Systems')
oci_file_storage_systems = OCIFileStorageSystems(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_file_storage_systems.list(
filter=query_json.get('file_storage_system_filter', None))
elif artifact == 'Instance':
logger.info('---- Processing Instances')
oci_instances = OCIInstances(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_instances.list(
filter=query_json.get('instance_filter', None))
elif artifact == 'InstancePool':
logger.info('---- Processing Instance Pools')
oci_instance_pools = OCIInstancePools(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_instance_pools.list(
filter=query_json.get('instance_filter', None))
elif artifact == 'InternetGateway':
logger.info('---- Processing Internet Gateways')
oci_internet_gateways = OCIInternetGateways(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'],
vcn_id=query_json['vcn_id'])
response_json = oci_internet_gateways.list(
filter=query_json.get('internet_gateway_filter', None))
elif artifact == 'IPSecConnection':
logger.info('---- Processing IPSec Connections')
oci_ipsec_connections = OCIIPSecConnections(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_ipsec_connections.list(
filter=query_json.get('ipsec_connection_filter', None))
elif artifact == 'LoadBalancer':
logger.info('---- Processing Load Balancers')
oci_load_balancers = OCILoadBalancers(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_load_balancers.list(
filter=query_json.get('load_balancer_filter', None))
response_json = [
lb for lb in response_json
if query_json['subnet_id'] in lb['subnet_ids']
]
elif artifact == 'LocalPeeringGateway':
logger.info('---- Processing LocalPeeringGateways')
oci_local_peering_gateways = OCILocalPeeringGateways(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'],
vcn_id=query_json['vcn_id'])
response_json = oci_local_peering_gateways.list(
filter=query_json.get('local_peering_gateway_filter', None))
elif artifact == 'MySQLDatabaseSystem':
logger.info('---- Processing MySQL Database Systems')
oci_mysql_database_systems = OCIMySQLDatabaseSystems(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_mysql_database_systems.list(
filter=query_json.get('mysql_database_system_filter', None))
elif artifact == 'NATGateway':
logger.info('---- Processing NAT Gateways')
oci_nat_gateways = OCINATGateways(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'],
vcn_id=query_json['vcn_id'])
response_json = oci_nat_gateways.list(
filter=query_json.get('nat_gateway_filter', None))
elif artifact == 'NetworkSecurityGroup':
logger.info('---- Processing Network Security Groups')
oci_network_security_groups = OCINetworkSecurityGroups(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'],
vcn_id=query_json['vcn_id'])
response_json = oci_network_security_groups.list(
filter=query_json.get('network_security_group_filter', None))
elif artifact == 'ObjectStorageBucket':
logger.info('---- Processing Object Storage Buckets')
oci_object_storage_buckets = OCIObjectStorageBuckets(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_object_storage_buckets.list(
filter=query_json.get('object_storage_bucket_filter', None))
elif artifact == 'OkeCluster':
logger.info('---- Processing OKE Clusters')
oke_clusters = OCIContainers(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oke_clusters.list(
filter=query_json.get('oke_cluster_filter', None))
elif artifact == 'RemotePeeringConnection':
logger.info('---- Processing Remote Peering Connections')
oci_remote_peering_connections = OCIRemotePeeringConnections(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_remote_peering_connections.list(
filter=query_json.get('remote_peering_connection_filter', None))
elif artifact == 'RouteTable':
logger.info('---- Processing Route Tables')
oci_route_tables = OCIRouteTables(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'],
vcn_id=query_json['vcn_id'])
response_json = oci_route_tables.list(
filter=query_json.get('route_table_filter', None))
elif artifact == 'SecurityList':
logger.info('---- Processing Security Lists')
oci_security_lists = OCISecurityLists(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'],
vcn_id=query_json['vcn_id'])
response_json = oci_security_lists.list(
filter=query_json.get('security_list_filter', None))
elif artifact == 'ServiceGateway':
logger.info('---- Processing Service Gateways')
oci_service_gateways = OCIServiceGateways(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'],
vcn_id=query_json['vcn_id'])
response_json = oci_service_gateways.list(
filter=query_json.get('service_gateway_filter', None))
elif artifact == 'Subnet':
logger.info('---- Processing Subnets')
oci_subnets = OCISubnets(config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'],
vcn_id=query_json['vcn_id'])
response_json = oci_subnets.list(
filter=query_json.get('subnet_filter', None))
elif artifact == 'VirtualCloudNetwork':
logger.info('---- Processing Virtual Cloud Networks')
oci_virtual_cloud_networks = OCIVirtualCloudNetworks(
config=config,
profile=config_profile,
compartment_id=query_json['compartment_id'])
response_json = oci_virtual_cloud_networks.list(
filter=query_json.get('virtual_cloud_network_filter', None))
else:
logger.warn('---- Unknown Artifact : {0:s}'.format(str(artifact)))
return '404'
logger.debug(
json.dumps(response_json,
sort_keys=True,
indent=2,
separators=(',', ': ')))
return json.dumps(standardiseIds(response_json), sort_keys=True)
def designer():
# Test if developer mode
developer_mode = (request.args.get('developer', default='false') == 'true')
if developer_mode:
logger.info(
"<<<<<<<<<<<<<<<<<<<<<<<<<< Developer Mode >>>>>>>>>>>>>>>>>>>>>>>>>>"
)
# Test if experimental mode
experimental_mode = (request.args.get('experimental',
default='false') == 'true')
if experimental_mode:
logger.info(
"<<<<<<<<<<<<<<<<<<<<<<<<<< Experimental Mode >>>>>>>>>>>>>>>>>>>>>>>>>>"
)
# Read Artifact Model Specific JavaScript Files
artefact_model_js_files = sorted(
os.listdir(os.path.join(bp.static_folder, 'model', 'js', 'artefacts')))
# Read Artifact View Specific JavaScript Files
if os.path.exists(os.path.join(
bp.static_folder, 'view', 'js', 'artefacts')) and os.path.isdir(
os.path.join(bp.static_folder, 'view', 'js', 'artefacts')):
artefact_view_js_files = sorted(
os.listdir(
os.path.join(bp.static_folder, 'view', 'js', 'artefacts')))
else:
artefact_view_js_files = []
artefact_view_js_files.extend(
sorted(
os.listdir(
os.path.join(bp.static_folder, 'view', 'designer', 'js',
'artefacts'))))
# Get Palette Icon Groups / Icons
svg_files = []
svg_icon_groups = {}
# Read Files
for (dirpath, dirnames,
filenames) in os.walk(os.path.join(bp.static_folder, 'palette')):
logger.debug('dirpath : {0!s:s}'.format(dirpath))
logger.debug('dirnames : {0!s:s}'.format(dirnames))
logger.debug('filenames : {0!s:s}'.format(filenames))
if os.path.basename(dirpath) != 'palette':
svg_files.extend([
os.path.join(os.path.basename(dirpath), f) for f in filenames
if f.endswith(".svg")
])
svg_icon_groups[os.path.basename(dirpath)] = [
f for f in filenames if f.endswith(".svg")
]
else:
svg_files.extend([f for f in filenames if f.endswith(".svg")])
logger.debug('Files Walk : {0!s:s}'.format(svg_files))
logger.debug('SVG Icon Groups {0!s:s}'.format(svg_icon_groups))
palette_icon_groups = []
for key in sorted(svg_icon_groups.keys()):
palette_icon_group = {'name': str(key).title(), 'icons': []}
for palette_svg in sorted(svg_icon_groups[key]):
palette_icon = {
'svg':
os.path.join(key, palette_svg),
'title':
os.path.basename(palette_svg).split('.')[0].replace('_', ' ')
}
palette_icon_group['icons'].append(palette_icon)
palette_icon_groups.append(palette_icon_group)
logger.debug('Palette Icon Groups : {0!s:s}'.format(palette_icon_groups))
logJson(palette_icon_groups)
# Read Fragment Files
fragment_files = os.listdir(
os.path.join(bp.static_folder, 'fragments', 'svg'))
fragment_icons = []
for fragment_svg in sorted(fragment_files):
logger.debug('Fragment : {0!s:s}'.format(fragment_svg))
logger.debug('Fragment full : {0!s:s}'.format(
os.path.join(bp.static_folder, 'fragments', 'svg', fragment_svg)))
fragment_icon = {
'svg':
fragment_svg,
'title':
os.path.basename(fragment_svg).split('.')[0].replace('_',
' ').title()
}
logger.debug('Icon : {0!s:s}'.format(fragment_icon))
fragment_icons.append(fragment_icon)
# Walk Template directory Structure
template_files = []
template_dirs = {}
logger.debug('Walking the template directories')
rootdir = os.path.join(bp.static_folder, 'templates')
for (dirpath, dirnames, filenames) in os.walk(rootdir, followlinks=True):
logger.debug('dirpath : {0!s:s}'.format(dirpath))
logger.debug('dirnames : {0!s:s}'.format(dirnames))
logger.debug('filenames : {0!s:s}'.format(filenames))
relpath = os.path.relpath(dirpath, rootdir)
logger.debug('Relative Path : {0!s:s}'.format(relpath))
template_files.extend([
os.path.join(relpath, f) for f in filenames if f.endswith(".json")
])
template_dirs[relpath] = [f for f in filenames if f.endswith(".json")]
logger.debug('Files Walk : {0!s:s}'.format(template_files))
logger.debug('Template Dirs {0!s:s}'.format(template_dirs))
template_groups = []
for key in sorted(template_dirs.keys()):
template_group = {
'name': str(key).replace('_', ' ').title(),
'templates': [],
'directories': {}
}
for template_file in sorted(template_dirs[key]):
try:
okit_template = {
'json': os.path.join(key, template_file),
'id': template_file.replace('.', '_')
}
filename = os.path.join(bp.static_folder, 'templates', key,
template_file)
template_json = readJsonFile(filename)
logger.debug('Template Json : {0!s:s}'.format(template_json))
okit_template['title'] = template_json['title']
okit_template['description'] = template_json.get(
'description', template_json['title'])
template_group['templates'].append(okit_template)
except Exception as e:
logger.debug(e)
template_groups.append(template_group)
logger.debug('Template Groups {0!s:s}'.format(template_groups))
logJson(template_groups)
template_categories = {}
for key in sorted(template_dirs.keys()):
name = str(key.split('/')[0]).replace('_', ' ').title()
path = key
category = template_categories.get(name, {
'path': path,
'name': '',
'templates': [],
'children': {}
})
template_categories[name] = build_categories(
path, key, category, sorted(template_dirs[key]))
logger.debug('Categories {0!s:s}'.format(template_categories))
logJson(template_categories)
config_sections = {"sections": readConfigFileSections()}
logger.debug('Config Sections {0!s:s}'.format(config_sections))
#Render The Template
return render_template('okit/okit_designer.html',
artefact_model_js_files=artefact_model_js_files,
artefact_view_js_files=artefact_view_js_files,
palette_icon_groups=palette_icon_groups,
fragment_icons=fragment_icons,
okit_templates_groups=template_groups,
okit_template_categories=template_categories,
developer_mode=developer_mode,
experimental_mode=experimental_mode)
def ociResourceManger():
if request.method == 'GET':
query_string = request.query_string
parsed_query_string = urllib.parse.unquote(query_string.decode())
query_json = json.loads(parsed_query_string)
logJson(query_json)
config_profile = query_json.get('location',
{}).get('config_profile', 'DEFAULT')
compartment_id = query_json.get('location',
{}).get('compartment_id', None)
region = query_json.get('location', {}).get('region', None)
try:
config = {'region': region}
oci_resourcemanager = OCIResourceManagers(
config=config,
profile=config_profile,
compartment_id=compartment_id)
stacks = oci_resourcemanager.list()
return json.dumps(stacks,
sort_keys=False,
indent=2,
separators=(',', ': '))
except Exception as e:
logger.exception(e)
return str(e), 500
elif request.method == 'POST':
logger.debug('JSON : {0:s}'.format(str(request.json)))
config_profile = request.json.get('location',
{}).get('config_profile', 'DEFAULT')
compartment_id = request.json.get('location',
{}).get('compartment_id', None)
region = request.json.get('location', {}).get('region', None)
plan_or_apply = request.json.get('location',
{}).get('plan_or_apply', 'PLAN')
create_or_update = request.json.get('location',
{}).get('create_or_update',
'CREATE')
stack_id = request.json.get('location', {}).get('stack_id', '')
stack_name = request.json.get('location', {}).get(
'stack_name',
'okit-stack-{0!s:s}'.format(time.strftime('%Y%m%d%H%M%S')))
logger.info('Using Profile : {0!s:s}'.format(config_profile))
try:
config = {'region': region}
destination_dir = tempfile.mkdtemp()
logger.debug(">>>>>>>>>>>>> {0!s:s}".format(destination_dir))
stack = {}
stack['display_name'] = stack_name
oci_compartments = OCICompartments(config=config,
profile=config_profile)
# Generate Resource Manager Terraform zip
generator = OCIResourceManagerGenerator(
template_root,
destination_dir,
request.json,
tenancy_ocid=oci_compartments.getTenancy(),
region=region,
compartment_ocid=compartment_id)
generator.generate()
generator.writeFiles()
zipname = generator.createZipArchive(
os.path.join(destination_dir, 'resource-manager'),
"/tmp/okit-resource-manager")
logger.info('Zipfile : {0:s}'.format(str(zipname)))
# Upload to Resource manager
stack['compartment_id'] = compartment_id
stack['zipfile'] = zipname
stack['variables'] = generator.getVariables()
resource_manager = OCIResourceManagers(
config=config,
profile=config_profile,
compartment_id=compartment_id)
if create_or_update == 'UPDATE':
stack['id'] = stack_id
stack_json = resource_manager.updateStack(stack)
else:
stack_json = resource_manager.createStack(stack)
resource_manager.createJob(stack_json, plan_or_apply)
return_code = 200
resource_manager.list()
shutil.rmtree(destination_dir)
return stack['display_name'], return_code
except Exception as e:
logger.exception(e)
return str(e), 500
return
def designer():
# Read Artifact Model Specific JavaScript Files
artefact_model_js_files = sorted(
os.listdir(os.path.join(bp.static_folder, 'model', 'js', 'artefacts')))
# Read Artifact View Specific JavaScript Files
artefact_view_js_files = sorted(
os.listdir(
os.path.join(bp.static_folder, 'view', 'designer', 'js',
'artefacts')))
# Get Palette Icon Groups / Icons
svg_files = []
svg_icon_groups = {}
# Read Files
for (dirpath, dirnames,
filenames) in os.walk(os.path.join(bp.static_folder, 'palette')):
logger.debug('dirpath : {0!s:s}'.format(dirpath))
logger.debug('dirnames : {0!s:s}'.format(dirnames))
logger.debug('filenames : {0!s:s}'.format(filenames))
if os.path.basename(dirpath) != 'palette':
svg_files.extend([
os.path.join(os.path.basename(dirpath), f) for f in filenames
])
svg_icon_groups[os.path.basename(dirpath)] = filenames
else:
svg_files.extend(filenames)
logger.debug('Files Walk : {0!s:s}'.format(svg_files))
logger.debug('SVG Icon Groups {0!s:s}'.format(svg_icon_groups))
palette_icon_groups = []
for key in sorted(svg_icon_groups.keys()):
palette_icon_group = {'name': str(key).title(), 'icons': []}
for palette_svg in sorted(svg_icon_groups[key]):
palette_icon = {
'svg':
os.path.join(key, palette_svg),
'title':
os.path.basename(palette_svg).split('.')[0].replace('_', ' ')
}
palette_icon_group['icons'].append(palette_icon)
palette_icon_groups.append(palette_icon_group)
logger.debug('Palette Icon Groups : {0!s:s}'.format(palette_icon_groups))
logJson(palette_icon_groups)
# Read Fragment Files
fragment_files = os.listdir(
os.path.join(bp.static_folder, 'fragments', 'svg'))
fragment_icons = []
for fragment_svg in sorted(fragment_files):
logger.info('Fragment : {0!s:s}'.format(fragment_svg))
logger.info('Fragment full : {0!s:s}'.format(
os.path.join(bp.static_folder, 'fragments', 'svg', fragment_svg)))
fragment_icon = {
'svg':
fragment_svg,
'title':
os.path.basename(fragment_svg).split('.')[0].replace('_',
' ').title()
}
logger.info('Icon : {0!s:s}'.format(fragment_icon))
fragment_icons.append(fragment_icon)
# Walk Template directory Structure
template_files = []
template_dirs = {}
logger.debug('Walking the template directories')
rootdir = os.path.join(bp.static_folder, 'templates')
for (dirpath, dirnames, filenames) in os.walk(rootdir, followlinks=True):
logger.debug('dirpath : {0!s:s}'.format(dirpath))
logger.debug('dirnames : {0!s:s}'.format(dirnames))
logger.debug('filenames : {0!s:s}'.format(filenames))
relpath = os.path.relpath(dirpath, rootdir)
logger.debug('Relative Path : {0!s:s}'.format(relpath))
template_files.extend([os.path.join(relpath, f) for f in filenames])
template_dirs[relpath] = filenames
logger.debug('Files Walk : {0!s:s}'.format(template_files))
logger.debug('Template Dirs {0!s:s}'.format(template_dirs))
template_groups = []
for key in sorted(template_dirs.keys()):
template_group = {
'name': str(key).replace('_', ' ').title(),
'templates': []
}
for template_file in sorted(template_dirs[key]):
try:
okit_template = {
'json': os.path.join(key, template_file),
'id': template_file.replace('.', '_')
}
filename = os.path.join(bp.static_folder, 'templates', key,
template_file)
template_json = readJsonFile(filename)
logger.debug('Template Json : {0!s:s}'.format(template_json))
okit_template['title'] = template_json['title']
okit_template['description'] = template_json.get(
'description', template_json['title'])
template_group['templates'].append(okit_template)
except Exception as e:
logger.debug(e)
template_groups.append(template_group)
logger.debug('Template Groups {0!s:s}'.format(template_groups))
logJson(template_groups)
config_sections = {"sections": readConfigFileSections()}
logger.info('Config Sections {0!s:s}'.format(config_sections))
#Render The Template
return render_template('okit/okit_designer.html',
artefact_model_js_files=artefact_model_js_files,
artefact_view_js_files=artefact_view_js_files,
palette_icon_groups=palette_icon_groups,
fragment_icons=fragment_icons,
okit_templates_groups=template_groups)
def standardiseJson(json_data={}, **kwargs):
logJson(json_data)
json_data = standardiseIds(json_data)
logJson(json_data)
return json_data