def delete_network(workout_id):
try:
# First delete any routes specific to the workout
result = compute.routes().list(project=project, filter='name = {}*'.format(workout_id)).execute()
if 'items' in result:
for route in result['items']:
response = compute.routes().delete(project=project, route=route["name"]).execute()
try:
compute.zoneOperations().wait(project=project, zone=zone, operation=response["id"]).execute()
except:
pass
# Now it is safe to delete the networks.
result = compute.networks().list(project=project, filter='name = {}*'.format(workout_id)).execute()
if 'items' in result:
for network in result['items']:
# Networks are not being deleted because the operation occurs too fast.
response = compute.networks().delete(project=project, network=network["name"]).execute()
compute.globalOperations().wait(project=project, operation=response["id"]).execute()
response = compute.globalOperations().get(project=project, operation=response["id"]).execute()
if 'error' in response:
if not long_delete_network(network["name"], response):
return False
return True
except():
print("Error in deleting network for %s" % workout_id)
return False
def server_delete(server_name):
server = ds_client.get(ds_client.key('cybergym-server', server_name))
state_transition(entity=server, new_state=SERVER_STATES.DELETING)
workout_globals.refresh_api()
try:
response = compute.instances().delete(project=project, zone=zone, instance=server_name).execute()
except HttpError as exception:
# If the server is already deleted or no longer exists,
state_transition(entity=server, new_state=SERVER_STATES.DELETED)
print(f"Finished deleting {server_name}")
# If all servers in the workout have been deleted, then set the workout state to True
build_id = server['workout']
check_build_state_change(build_id=build_id, check_server_state=SERVER_STATES.DELETED,
change_build_state=BUILD_STATES.COMPLETED_DELETING_SERVERS)
return True
print(f'Sent delete request to {server_name}, and waiting for response')
i = 0
success = False
while not success and i < 5:
try:
print(f"Begin waiting for delete response from operation {response['id']}")
compute.zoneOperations().wait(project=project, zone=zone, operation=response["id"]).execute()
success = True
except timeout:
i += 1
print('Response timeout for deleting server. Trying again')
pass
if not success:
print(f'Timeout in trying to delete server {server_name}')
state_transition(entity=server, new_state=SERVER_STATES.BROKEN)
return False
# If this is a student entry server, delete the DNS
if 'student_entry' in server and server['student_entry']:
print(f'Deleting DNS record for {server_name}')
ip_address = server['external_ip']
delete_dns(server['workout'], ip_address)
state_transition(entity=server, new_state=SERVER_STATES.DELETED)
print(f"Finished deleting {server_name}")
# If all servers in the workout have been deleted, then set the workout state to True
build_id = server['workout']
check_build_state_change(build_id=build_id, check_server_state=SERVER_STATES.DELETED,
change_build_state=BUILD_STATES.COMPLETED_DELETING_SERVERS)
return True
# server_start('hxckdwxwld-nested')
# server_delete('oztfvquhhi-cybergym-publicprivate')
def delete_firewall_rules(workout_id):
try:
result = compute.firewalls().list(project=project, filter='name = {}*'.format(workout_id)).execute()
if 'items' in result:
for fw_rule in result['items']:
response = compute.firewalls().delete(project=project, firewall=fw_rule["name"]).execute()
try:
compute.zoneOperations().wait(project=project, zone=zone, operation=response["id"]).execute()
except:
pass
return True
except():
print("Error in deleting firewall rules for %s" % workout_id)
return False
def delete_subnetworks(workout_id):
try:
result = compute.subnetworks().list(project=project, region=region,
filter='name = {}*'.format(workout_id)).execute()
if 'items' in result:
for subnetwork in result['items']:
response = compute.subnetworks().delete(project=project, region=region,
subnetwork=subnetwork["name"]).execute()
try:
compute.zoneOperations().wait(project=project, zone=zone, operation=response["id"]).execute()
except:
pass
return True
except():
print("Error in deleting subnetworks for %s" % workout_id)
return True
def server_start(server_name):
"""
Starts a server based on the specification in the Datastore entity with name server_name. A guacamole server
is also registered with DNS.
:param server_name: The Datastore entity name of the server to start
:return: A boolean status on the success of the start
"""
server = ds_client.get(ds_client.key('cybergym-server', server_name))
state_transition(entity=server, new_state=SERVER_STATES.STARTING)
workout_globals.refresh_api()
response = compute.instances().start(project=project, zone=zone, instance=server_name).execute()
print(f'Sent start request to {server_name}, and waiting for response')
i = 0
success = False
while not success and i < 5:
try:
print(f"Begin waiting for start response from operation {response['id']}")
compute.zoneOperations().wait(project=project, zone=zone, operation=response["id"]).execute()
success = True
except timeout:
i += 1
print('Response timeout for starting server. Trying again')
pass
if not success:
print(f'Timeout in trying to start server {server_name}')
state_transition(entity=server, new_state=SERVER_STATES.BROKEN)
return False
# If this is the guacamole server for student entry, then register the new DNS
if 'student_entry' in server and server['student_entry']:
print(f'Setting DNS record for {server_name}')
ip_address = register_student_entry(server['workout'], server_name)
server['external_ip'] = ip_address
state_transition(entity=server, new_state=SERVER_STATES.RUNNING)
print(f"Finished starting {server_name}")
# If all servers have started, then change the build state
build_id = server['workout']
check_build_state_change(build_id=build_id, check_server_state=SERVER_STATES.RUNNING,
change_build_state=BUILD_STATES.RUNNING)
return True
def server_build(server_name):
"""
Builds an individual server based on the specification in the Datastore entity with name server_name.
:param server_name: The Datastore entity name of the server to build
:return: A boolean status on the success of the build
"""
server = ds_client.get(ds_client.key('cybergym-server', server_name))
build_id = server['workout']
g_logger = log_client.logger(str(server_name))
state_transition(entity=server, new_state=SERVER_STATES.BUILDING)
config = server['config'].copy()
"""
Currently, we need a workaround to insert the guacamole startup script because of a 1500 character limit on
indexed fields. The exclude_from_index does not work on embedded datastore fields
"""
if 'student_entry' in server and server['student_entry']:
config['metadata'] = {
'items': [{
"key": "startup-script",
"value": server['guacamole_startup_script']
}]
}
# Begin the server build and keep trying for a bounded number of additional 30-second cycles
i = 0
build_success = False
while not build_success and i < 5:
workout_globals.refresh_api()
try:
if server['add_disk']:
try:
image_config = {
"name":
server_name + "-disk",
"sizeGb":
server['add_disk'],
"type":
"projects/" + project + "/zones/" + zone +
"/diskTypes/pd-ssd"
}
response = compute.disks().insert(
project=project, zone=zone,
body=image_config).execute()
compute.zoneOperations().wait(
project=project, zone=zone,
operation=response["id"]).execute()
except HttpError as err:
# If the disk already exists (i.e. a nuke), then ignore
if err.resp.status in [409]:
pass
if server['build_type'] == BUILD_TYPES.MACHINE_IMAGE:
source_machine_image = f"projects/{project}/global/machineImages/{server['machine_image']}"
compute_beta = discovery.build('compute', 'beta')
response = compute_beta.instances().insert(
project=project,
zone=zone,
body=config,
sourceMachineImage=source_machine_image).execute()
else:
if "delayed_start" in server and server["delayed_start"]:
time.sleep(30)
response = compute.instances().insert(project=project,
zone=zone,
body=config).execute()
build_success = True
g_logger.log_text(
f'Sent job to build {server_name}, and waiting for response')
except BrokenPipeError:
i += 1
except HttpError as exception:
cloud_log(
build_id,
f"Error when trying to build {server_name}: {exception.reason}",
LOG_LEVELS.ERROR)
return False
i = 0
success = False
while not success and i < 5:
try:
g_logger.log_text(
f"Begin waiting for build operation {response['id']}")
compute.zoneOperations().wait(project=project,
zone=zone,
operation=response["id"]).execute()
success = True
except timeout:
i += 1
g_logger.log_text('Response timeout for build. Trying again')
pass
if success:
g_logger.log_text(f'Successfully built server {server_name}')
state_transition(entity=server,
new_state=SERVER_STATES.RUNNING,
existing_state=SERVER_STATES.BUILDING)
else:
g_logger.log_text(f'Timeout in trying to build server {server_name}')
state_transition(entity=server, new_state=SERVER_STATES.BROKEN)
return False
# If this is a student entry server, register the DNS
if 'student_entry' in server and server['student_entry']:
g_logger.log_text(f'Setting DNS record for {server_name}')
ip_address = register_student_entry(server['workout'], server_name)
server['external_ip'] = ip_address
ds_client.put(server)
server = ds_client.get(ds_client.key('cybergym-server', server_name))
# Now stop the server before completing
g_logger.log_text(f'Stopping {server_name}')
compute.instances().stop(project=project, zone=zone,
instance=server_name).execute()
state_transition(entity=server, new_state=SERVER_STATES.STOPPED)
# If no other servers are building, then set the workout to the state of READY.
check_build_state_change(build_id=build_id,
check_server_state=SERVER_STATES.STOPPED,
change_build_state=BUILD_STATES.READY)
def server_delete(server_name):
g_logger = log_client.logger(str(server_name))
server_list = list(
ds_client.query(kind='cybergym-server').add_filter(
'name', '=', str(server_name)).fetch())
server_is_deleted = list(
ds_client.query(kind='cybergym-server').add_filter(
'name', '=', str(server_name)).add_filter('state', '=',
'DELETED').fetch())
if server_is_deleted and server_list:
g_logger.log_text(f'Server "' + server_name +
'" has already been deleted.')
return True
elif not server_list:
g_logger.log_text(f'Server of name "' + server_name +
'" does not exist in datastore, unable to Delete.')
return True
else:
server = ds_client.get(ds_client.key('cybergym-server', server_name))
state_transition(entity=server, new_state=SERVER_STATES.DELETING)
# If there are snapshots associated with this server, then delete the snapshots.
if 'snapshot' in server and server['snapshot']:
Snapshot.delete_snapshot(server_name)
workout_globals.refresh_api()
try:
response = compute.instances().delete(project=project,
zone=zone,
instance=server_name).execute()
except HttpError as exception:
# If the server is already deleted or no longer exists,
state_transition(entity=server, new_state=SERVER_STATES.DELETED)
g_logger.log_text(f"Finished deleting {server_name}")
# If all servers in the workout have been deleted, then set the workout state to True
build_id = server['workout']
check_build_state_change(
build_id=build_id,
check_server_state=SERVER_STATES.DELETED,
change_build_state=BUILD_STATES.COMPLETED_DELETING_SERVERS)
return True
g_logger.log_text(
f'Sent delete request to {server_name}, and waiting for response')
i = 0
success = False
while not success and i < 5:
try:
g_logger.log_text(
f"Begin waiting for delete response from operation {response['id']}"
)
compute.zoneOperations().wait(project=project,
zone=zone,
operation=response["id"]).execute()
success = True
except timeout:
i += 1
g_logger.log_text(
'Response timeout for deleting server. Trying again')
pass
if not success:
g_logger.log_text(f'Timeout in trying to delete server {server_name}')
state_transition(entity=server, new_state=SERVER_STATES.BROKEN)
return False
# If this is a student entry server, delete the DNS
if 'student_entry' in server and server['student_entry']:
g_logger.log_text(f'Deleting DNS record for {server_name}')
ip_address = server['external_ip']
delete_dns(server['workout'], ip_address)
state_transition(entity=server, new_state=SERVER_STATES.DELETED)
g_logger.log_text(f"Finished deleting {server_name}")
# If all servers in the workout have been deleted, then set the workout state to True
build_id = server['workout']
check_build_state_change(
build_id=build_id,
check_server_state=SERVER_STATES.DELETED,
change_build_state=BUILD_STATES.COMPLETED_DELETING_SERVERS)
return True
def server_build(server_name):
"""
Builds an individual server based on the specification in the Datastore entity with name server_name.
:param server_name: The Datastore entity name of the server to build
:return: A boolean status on the success of the build
"""
print(f'Building server {server_name}')
server = ds_client.get(ds_client.key('cybergym-server', server_name))
state_transition(entity=server, new_state=SERVER_STATES.BUILDING)
# Commented because this is only for Fortinet right now.
# if 'canIPForward' in server and server['config']['canIpForward']:
# image_config = {"name": server_name + "-disk", "sizeGb": 30,
# "type": "projects/" + project + "/zones/" + zone + "/diskTypes/pd-ssd"}
# response = compute.disks().insert(project=project, zone=zone, body=image_config).execute()
# compute.zoneOperations().wait(project=project, zone=zone, operation=response["id"]).execute()
# Begin the server build and keep trying for a bounded number of additional 30-second cycles
i = 0
build_success = False
while not build_success and i < 5:
workout_globals.refresh_api()
try:
response = compute.instances().insert(project=project, zone=zone, body=server['config']).execute()
build_success = True
print(f'Sent job to build {server_name}, and waiting for response')
except BrokenPipeError:
i += 1
i = 0
success = False
while not success and i < 5:
try:
print(f"Begin waiting for build operation {response['id']}")
compute.zoneOperations().wait(project=project, zone=zone, operation=response["id"]).execute()
success = True
except timeout:
i += 1
print('Response timeout for build. Trying again')
pass
if success:
print(f'Successfully built server {server_name}')
state_transition(entity=server, new_state=SERVER_STATES.RUNNING, existing_state=SERVER_STATES.BUILDING)
else:
print(f'Timeout in trying to build server {server_name}')
state_transition(entity=server, new_state=SERVER_STATES.BROKEN)
return False
# If this is a student entry server, register the DNS
if 'student_entry' in server and server['student_entry']:
print(f'Setting DNS record for {server_name}')
ip_address = register_student_entry(server['workout'], server_name)
server['external_ip'] = ip_address
ds_client.put(server)
server = ds_client.get(ds_client.key('cybergym-server', server_name))
# Now stop the server before completing
print(f'Stopping {server_name}')
compute.instances().stop(project=project, zone=zone, instance=server_name).execute()
state_transition(entity=server, new_state=SERVER_STATES.STOPPED)
# If no other servers are building, then set the workout to the state of READY.
build_id = server['workout']
check_build_state_change(build_id=build_id, check_server_state=SERVER_STATES.STOPPED,
change_build_state=BUILD_STATES.READY)