def delete_network(workout_id):
try:
# First delete any routes specific to the workout
result = compute.routes().list(project=project, filter='name = {}*'.format(workout_id)).execute()
if 'items' in result:
for route in result['items']:
response = compute.routes().delete(project=project, route=route["name"]).execute()
try:
compute.zoneOperations().wait(project=project, zone=zone, operation=response["id"]).execute()
except:
pass
# Now it is safe to delete the networks.
result = compute.networks().list(project=project, filter='name = {}*'.format(workout_id)).execute()
if 'items' in result:
for network in result['items']:
# Networks are not being deleted because the operation occurs too fast.
response = compute.networks().delete(project=project, network=network["name"]).execute()
compute.globalOperations().wait(project=project, operation=response["id"]).execute()
response = compute.globalOperations().get(project=project, operation=response["id"]).execute()
if 'error' in response:
if not long_delete_network(network["name"], response):
return False
return True
except():
print("Error in deleting network for %s" % workout_id)
return False
def _delete_orphaned_networks():
"""
This function iterates through all of the networks and deletes any networks which do not have corresponding
subnetworks. This operation is safe because all active workouts will have a subnetwork.
:returns: True if the operations was successful. Otherwise, this returns false.
"""
networks = compute.networks().list(project=project).execute()
if 'items' in networks:
for network in networks['items']:
if 'subnetworks' not in network:
cloud_log("cybergym-app",
f"Deleting orphaned network {network['name']}",
LOG_LEVELS.INFO)
try:
response = compute.networks().delete(
project=project,
network=network["name"]).execute()
except HttpError:
cloud_log("cybergym-app",
f"Error deleting network {network['name']}",
LOG_LEVELS.ERROR)
if not gcp_operation_wait(service=compute,
response=response,
wait_type="global"):
cloud_log(
"cybergym-app",
f"Timeout waiting for network {network['name']} to delete",
LOG_LEVELS.WARNING)
pass
else:
return False
return True
def _delete_network(self):
i = 0
# Now it is safe to delete the networks.
success = True
result = compute.networks().list(
project=project, filter=f'name = {self.build_id}*').execute()
if 'items' in result:
for network in result['items']:
# Networks are not being deleted because the operation occurs too fast.
response = compute.networks().delete(
project=project, network=network["name"]).execute()
if not gcp_operation_wait(service=compute,
response=response,
wait_type="global"):
cloud_log(
"cybergym-app",
f"Timeout waiting for network {network['name']} to delete",
LOG_LEVELS.WARNING)
success = False
else:
# For workouts with multiple networks, avoid setting success to true if at least one
# network has an error.
if not success:
success = True
else:
# Return true since the network has already been deleted
return True
if success:
self._process_workout_deletion()
return True
else:
return False
def create_network(networks, build_id):
"""
Build the network for the given build specification
:param networks: A specification of networks and subnetwork to build
:param build_id: The ID of the build. This will be used for referencing the build by name in the future.
:return:
"""
for network in networks:
network_body = {
"name": "%s-%s" % (build_id, network['name']),
"autoCreateSubnetworks": False,
"region": region
}
response = compute.networks().insert(project=project,
body=network_body).execute()
compute.globalOperations().wait(project=project,
operation=response["id"]).execute()
time.sleep(10)
for subnet in network['subnets']:
subnetwork_body = {
"name":
"%s" % (subnet['name']),
"network":
"projects/%s/global/networks/%s" %
(project, network_body['name']),
"ipCidrRange":
subnet['ip_subnet']
}
response = compute.subnetworks().insert(
project=project, region=region,
body=subnetwork_body).execute()
compute.regionOperations().wait(
project=project, region=region,
operation=response["id"]).execute()
def long_delete_network(network, response):
"""
This is necessary because the network does not always delete. The routes take a while to clear from the GCP.
:param network: The network name to delete
:param response: The last response from globalOperations.get()
:return: Boolean on whether the network delete was successful.
"""
max_tries = 3
i = 0
while 'error' in response and i < max_tries:
response = compute.networks().delete(project=project, network=network).execute()
compute.globalOperations().wait(project=project, operation=response["id"]).execute()
response = compute.globalOperations().get(project=project, operation=response["id"]).execute()
i += 1
if i < max_tries:
return True
else:
return False
def _wait_for_deletion(self, wait_type=ArenaWorkoutDeleteType.SERVER):
"""
For asynchronous deletion, wait until all jobs have completed.
@param build_id: The id of the build to use in searching for resources
@type build_id: String
@param wait_type: designated type of resource
@type wait_type: String
@return: Status
@rtype: Boolean
"""
i = 0
all_deleted = False
while not all_deleted and i < 10:
if wait_type == ArenaWorkoutDeleteType.SERVER:
result = compute.instances().list(
project=project,
zone=zone,
filter=f"name = {self.build_id}*").execute()
elif wait_type == ArenaWorkoutDeleteType.ROUTES:
result = compute.routes().list(
project=project,
filter=f"name = {self.build_id}*").execute()
elif wait_type == ArenaWorkoutDeleteType.FIREWALL_RULES:
result = compute.firewalls().list(
project=project,
filter=f"name = {self.build_id}*").execute()
elif wait_type == ArenaWorkoutDeleteType.NETWORK:
result = compute.networks().list(
project=project,
filter=f"name = {self.build_id}*").execute()
elif wait_type == ArenaWorkoutDeleteType.SUBNETWORK:
result = compute.subnetworks().list(
project=project,
region=region,
filter=f"name = {self.build_id}*").execute()
if 'items' not in result:
all_deleted = True
else:
i += 1
time.sleep(10)
return all_deleted
def build_workout(workout_id):
"""
Builds a workout compute environment according to the specification referenced in the datastore with key workout_id
:param workout_id: The workout_id key in the datastore holding the build specification
:return: None
"""
key = ds_client.key('cybergym-workout', workout_id)
workout = ds_client.get(key)
# This can sometimes happen when debugging a workout ID and the Datastore record no longer exists.
if not workout:
cloud_log(workout_id,
f"The datastore record for {workout_id} no longer exists!",
LOG_LEVELS.ERROR)
raise LookupError
if 'state' not in workout or not workout['state']:
state_transition(entity=workout, new_state=BUILD_STATES.START)
# Create the networks and subnets
if check_ordered_workout_state(workout, BUILD_STATES.BUILDING_NETWORKS):
state_transition(entity=workout,
new_state=BUILD_STATES.BUILDING_NETWORKS)
for network in workout['networks']:
cloud_log(workout_id,
f"Building network {workout_id}-{network['name']}",
LOG_LEVELS.INFO)
network_body = {
"name": f"{workout_id}-{network['name']}",
"autoCreateSubnetworks": False,
"region": region
}
try:
response = compute.networks().insert(
project=project, body=network_body).execute()
compute.globalOperations().wait(
project=project, operation=response["id"]).execute()
time.sleep(10)
except HttpError as err:
# If the network already exists, then this may be a rebuild and ignore the error
if err.resp.status in [409]:
pass
for subnet in network['subnets']:
cloud_log(
workout_id,
f"Building the subnetwork {network_body['name']}-{subnet['name']}",
LOG_LEVELS.INFO)
subnetwork_body = {
"name":
f"{network_body['name']}-{subnet['name']}",
"network":
"projects/%s/global/networks/%s" %
(project, network_body['name']),
"ipCidrRange":
subnet['ip_subnet']
}
try:
response = compute.subnetworks().insert(
project=project, region=region,
body=subnetwork_body).execute()
compute.regionOperations().wait(
project=project,
region=region,
operation=response["id"]).execute()
except HttpError as err:
# If the subnetwork already exists, then this may be a rebuild and ignore the error
if err.resp.status in [409]:
pass
state_transition(entity=workout,
new_state=BUILD_STATES.COMPLETED_NETWORKS)
# Now create the server configurations
if check_ordered_workout_state(workout, BUILD_STATES.BUILDING_SERVERS):
state_transition(entity=workout,
new_state=BUILD_STATES.BUILDING_SERVERS)
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
for server in workout['servers']:
server_name = f"{workout_id}-{server['name']}"
cloud_log(workout_id,
f"Sending pubsub message to build {server_name}",
LOG_LEVELS.INFO)
publisher.publish(topic_path,
data=b'Server Build',
server_name=server_name,
action=SERVER_ACTIONS.BUILD)
# Also build the student entry server for the workout
publisher.publish(topic_path,
data=b'Server Build',
server_name=f"{workout_id}-student-guacamole",
action=SERVER_ACTIONS.BUILD)
state_transition(entity=workout,
new_state=BUILD_STATES.COMPLETED_SERVERS)
# Create all of the network routes and firewall rules
if check_ordered_workout_state(workout, BUILD_STATES.BUILDING_ROUTES):
state_transition(entity=workout,
new_state=BUILD_STATES.BUILDING_ROUTES)
cloud_log(
workout_id,
f"Creating network routes and firewall rules for {workout_id}",
LOG_LEVELS.INFO)
if 'routes' in workout and workout['routes']:
workout_route_setup(workout_id)
if check_ordered_workout_state(workout, BUILD_STATES.BUILDING_FIREWALL):
state_transition(entity=workout,
new_state=BUILD_STATES.BUILDING_FIREWALL)
firewall_rules = []
for rule in workout['firewall_rules']:
firewall_rules.append({
"name":
"%s-%s" % (workout_id, rule['name']),
"network":
"%s-%s" % (workout_id, rule['network']),
"targetTags":
rule['target_tags'],
"protocol":
rule['protocol'],
"ports":
rule['ports'],
"sourceRanges":
rule['source_ranges']
})
create_firewall_rules(firewall_rules)
state_transition(entity=workout,
new_state=BUILD_STATES.COMPLETED_FIREWALL)
cloud_log(
workout_id,
f"Finished the build process with a final state: {workout['state']}",
LOG_LEVELS.INFO)
def build_workout(workout_id):
"""
Builds a workout compute environment according to the specification referenced in the datastore with key workout_id
:param workout_id: The workout_id key in the datastore holding the build specification
:return: None
"""
key = ds_client.key('cybergym-workout', workout_id)
workout = ds_client.get(key)
# This can sometimes happen when debugging a workout ID and the Datastore record no longer exists.
if not workout:
print('No workout for %s exists in the data store' % workout_id)
return
startup_scripts = None
# Parse the assessment specification to obtain any startup scripts for the workout.
if 'state' not in workout or not workout['state']:
state_transition(entity=workout, new_state=BUILD_STATES.START)
if workout['assessment']:
startup_scripts = get_startup_scripts(workout_id=workout_id,
assessment=workout['assessment'])
# Create the networks and subnets
if check_ordered_workout_state(workout, BUILD_STATES.BUILDING_NETWORKS):
state_transition(entity=workout,
new_state=BUILD_STATES.BUILDING_NETWORKS)
print('Creating networks')
for network in workout['networks']:
network_body = {
"name": "%s-%s" % (workout_id, network['name']),
"autoCreateSubnetworks": False,
"region": region
}
response = compute.networks().insert(project=project,
body=network_body).execute()
compute.globalOperations().wait(
project=project, operation=response["id"]).execute()
time.sleep(10)
for subnet in network['subnets']:
subnetwork_body = {
"name":
"%s-%s" % (network_body['name'], subnet['name']),
"network":
"projects/%s/global/networks/%s" %
(project, network_body['name']),
"ipCidrRange":
subnet['ip_subnet']
}
response = compute.subnetworks().insert(
project=project, region=region,
body=subnetwork_body).execute()
compute.regionOperations().wait(
project=project, region=region,
operation=response["id"]).execute()
state_transition(entity=workout,
new_state=BUILD_STATES.COMPLETED_NETWORKS)
# Now create the server configurations
if check_ordered_workout_state(workout, BUILD_STATES.BUILDING_SERVERS):
state_transition(entity=workout,
new_state=BUILD_STATES.BUILDING_SERVERS)
print('Creating servers')
for server in workout['servers']:
server_name = "%s-%s" % (workout_id, server['name'])
sshkey = server["sshkey"]
tags = server['tags']
machine_type = server["machine_type"]
network_routing = server["network_routing"]
min_cpu_platform = server[
"minCpuPlatform"] if "minCpuPlatform" in server else None
nics = []
for n in server['nics']:
nic = {
"network": f"{workout_id}-{n['network']}",
"internal_IP": n['internal_IP'],
"subnet": f"{workout_id}-{n['network']}-{n['subnet']}",
"external_NAT": n['external_NAT']
}
# Nested VMs are sometimes used for vulnerable servers. This adds those specified IP addresses as
# aliases to the NIC
if 'IP_aliases' in n and n['IP_aliases']:
alias_ip_ranges = []
for ipaddr in n['IP_aliases']:
alias_ip_ranges.append({"ipCidrRange": ipaddr})
nic['aliasIpRanges'] = alias_ip_ranges
nics.append(nic)
# Add the startup script for assessment as metadata if it exists
meta_data = None
if startup_scripts and server['name'] in startup_scripts:
meta_data = startup_scripts[server['name']]
create_instance_custom_image(compute=compute,
workout=workout_id,
name=server_name,
custom_image=server['image'],
machine_type=machine_type,
networkRouting=network_routing,
networks=nics,
tags=tags,
meta_data=meta_data,
sshkey=sshkey,
minCpuPlatform=min_cpu_platform)
state_transition(entity=workout,
new_state=BUILD_STATES.COMPLETED_SERVERS)
# Create the student entry guacamole server
if check_ordered_workout_state(workout,
BUILD_STATES.BUILDING_STUDENT_ENTRY):
state_transition(entity=workout,
new_state=BUILD_STATES.BUILDING_STUDENT_ENTRY)
if workout['student_entry']:
network_name = f"{workout_id}-{workout['student_entry']['network']}"
student_entry_username = workout['student_entry'][
'username'] if 'username' in workout['student_entry'] else None
security_mode = workout['student_entry'][
'security-mode'] if 'security-mode' in workout[
'student_entry'] else 'nla'
guac_connection = [{
'workout_id': workout_id,
'entry_type': workout['student_entry']['type'],
'ip': workout['student_entry']['ip'],
'username': student_entry_username,
'password': workout['student_entry']['password'],
'security-mode': security_mode
}]
build_guacamole_server(build=workout,
network=network_name,
guacamole_connections=guac_connection)
# Get the workout key again or the state transition will overwrite it
workout = ds_client.get(
ds_client.key('cybergym-workout', workout_id))
else:
state_transition(entity=workout, new_state=BUILD_STATES.BROKEN)
return
state_transition(entity=workout,
new_state=BUILD_STATES.COMPLETED_STUDENT_ENTRY)
# Create all of the network routes and firewall rules
if check_ordered_workout_state(workout, BUILD_STATES.BUILDING_ROUTES):
state_transition(entity=workout,
new_state=BUILD_STATES.BUILDING_ROUTES)
print('Creating network routes and firewall rules')
if 'routes' in workout and workout['routes']:
for route in workout['routes']:
response = compute.instances().get(
project=project,
zone=zone,
instance=f"{workout_id}-{route['next_hop_instance']}")
r = {
"name":
"%s-%s" % (workout_id, route['name']),
"network":
"%s-%s" % (workout_id, route['network']),
"destRange":
route['dest_range'],
"nextHopInstance":
"%s-%s" % (workout_id, route['next_hop_instance'])
}
create_route(r)
if check_ordered_workout_state(workout, BUILD_STATES.BUILDING_FIREWALL):
state_transition(entity=workout,
new_state=BUILD_STATES.BUILDING_FIREWALL)
firewall_rules = []
for rule in workout['firewall_rules']:
firewall_rules.append({
"name":
"%s-%s" % (workout_id, rule['name']),
"network":
"%s-%s" % (workout_id, rule['network']),
"targetTags":
rule['target_tags'],
"protocol":
rule['protocol'],
"ports":
rule['ports'],
"sourceRanges":
rule['source_ranges']
})
create_firewall_rules(firewall_rules)
# build_workout('isirdhzjqk')