def state_transition(entity, new_state, existing_state=None):
"""
Consistently changes a datastore entity with the necessary state checks.
:param entity: A datastore entity
:param new_state: The new state for the server
:param existing_state: A check for the existing state of the server. This defaults to None.
:return: Boolean on success. If the expected existing state is not the same as the actual, this returns False.
"""
if 'state' not in entity:
entity['state'] = None
if existing_state and entity['state'] != existing_state:
g_logger = log_client.logger(entity.key.name)
g_logger.log_struct(
{
"message": "Error in state transition: Expected entity to be in {} state. Instead, it was in state {}".format(existing_state, entity['state'])
}, severity=LOG_LEVELS.WARNING
)
# print(f"Error in state transition: Expected entity to be in {existing_state} state. Instead, it was in the state"
# f" {entity['state']}")
return False
ts = str(calendar.timegm(time.gmtime()))
entity['state'] = new_state
entity['state-timestamp'] = ts
if new_state == BUILD_STATES.DELETED:
entity['active'] = False
elif new_state == BUILD_STATES.READY:
entity['active'] = True
g_logger = log_client.logger(entity.key.name)
g_logger.log_text(str('State Transition {}: Transitioning to {} at {}'.format(entity.key.name, new_state, ts)))
ds_safe_put(entity)
return True
def stop_arena(unit_id):
"""
Arenas have server builds for the unit as well as individual workouts. This function
stops all of these servers
:param unit_id: The build ID of the arena
:return: None
"""
# First stop the unit's servers
result = compute.instances().list(
project=project, zone=zone,
filter='name = {}*'.format(unit_id)).execute()
unit = ds_client.get(ds_client.key('cybergym-unit', unit_id))
unit['arena']['running'] = False
ds_client.put(unit)
g_logger = log_client.logger('arena-actions')
if 'items' in result:
for vm_instance in result['items']:
response = compute.instances().stop(
project=project, zone=zone,
instance=vm_instance["name"]).execute()
g_logger.log_struct(
{"message": "Stopped servers for arena {}".format(unit_id)},
severity=LOG_LEVELS)
else:
g_logger.log_struct(
{"message": "No servers in arena {} to stop".format(unit_id)},
severity=LOG_LEVELS.WARNING)
for workout_id in unit['workouts']:
g_logger = log_client.logger(str(workout_id))
result = compute.instances().list(
project=project, zone=zone,
filter='name = {}*'.format(workout_id)).execute()
workout = ds_client.get(ds_client.key('cybergym-workout', workout_id))
workout['running'] = False
ds_client.put(workout)
if 'items' in result:
for vm_instance in result['items']:
response = compute.instances().stop(
project=project, zone=zone,
instance=vm_instance["name"]).execute()
g_logger.log_struct(
{
"message":
"Workout servers stopped for workout {}".format(workout_id)
},
severity=LOG_LEVELS.INFO)
else:
g_logger.log_struct(
{
"message":
"No servers to stop for workout {}".format(workout_id)
},
severity=LOG_LEVELS.WARNING)
def workout_route_setup(workout_id):
key = ds_client.key('cybergym-workout', workout_id)
workout = ds_client.get(key)
g_logger = log_client.logger(workout_id)
if 'routes' in workout and workout['routes']:
for route in workout['routes']:
i = 0
while not test_server_existence(
workout_id, route['next_hop_instance']) and i < 50:
time.sleep(10)
i += 1
if i >= 50:
g_logger.log_text(
f"Timeout waiting to add routes for {route['next_hop_instance']}"
)
return False
r = {
"name":
"%s-%s" % (workout_id, route['name']),
"network":
"%s-%s" % (workout_id, route['network']),
"destRange":
route['dest_range'],
"nextHopInstance":
"%s-%s" % (workout_id, route['next_hop_instance'])
}
create_route(r)
def start_arena(unit_id):
g_logger = log_client.logger('arena-actions')
g_logger.log_struct({"message": "Starting arena {}".format(unit_id)},
severity=LOG_LEVELS.INFO)
unit = ds_client.get(ds_client.key('cybergym-unit', unit_id))
state_transition(entity=unit, new_state=BUILD_STATES.STARTING)
unit['arena']['running'] = True
unit['arena']['gm_start_time'] = str(calendar.timegm(time.gmtime()))
ds_client.put(unit)
# Start the central servers
g_logger.log_struct(
{"message": "Starting central servers for arena {}".format(unit_id)},
severity=LOG_LEVELS.INFO)
query_central_arena_servers = ds_client.query(kind='cybergym-server')
query_central_arena_servers.add_filter("workout", "=", unit_id)
for server in list(query_central_arena_servers.fetch()):
# Publish to a server management topic
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path,
data=b'Server Build',
server_name=server['name'],
action=SERVER_ACTIONS.START)
print(future.result())
# Now start all of the student workouts for this arena
for workout_id in unit['workouts']:
start_vm(workout_id)
def stop_lapsed_workouts():
# Get the current time to compare with the start time to see if a workout needs to stop
ts = calendar.timegm(time.gmtime())
# Query all workouts which have not been deleted
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter("state", "=", BUILD_STATES.RUNNING)
for workout in list(query_workouts.fetch()):
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if "start_time" in workout and "run_hours" in workout and workout.get(
'type', 'arena') != 'arena':
workout_id = workout.key.name
start_time = int(workout.get('start_time', 0))
run_hours = int(workout.get('run_hours', 0))
# Stop the workout servers if the run time has exceeded the request
if ts - start_time >= run_hours * 3600:
g_logger = log_client.logger(str(workout_id))
g_logger.log_struct(
{
"message":
"The workout {} has exceeded its run time and will be stopped"
.format(workout_id)
},
severity=LOG_LEVELS.INFO)
stop_workout(workout_id)
def stop_lapsed_arenas():
# Get the current time to compare with the start time to see if a workout needs to stop
ts = calendar.timegm(time.gmtime())
# Query all workouts which have not been deleted
query_units = ds_client.query(kind='cybergym-unit')
query_units.add_filter("arena.running", "=", True)
for unit in list(query_units.fetch()):
if 'arena' in unit and "gm_start_time" in unit[
'arena'] and "run_hours" in unit['arena']:
unit_id = unit.key.name
start_time = int(unit['arena'].get('gm_start_time', 0))
run_hours = int(unit['arena'].get('run_hours', 0))
# Stop the workout servers if the run time has exceeded the request
if ts - start_time >= run_hours * 3600:
g_logger = log_client.logger('arena-actions')
g_logger.log_struct(
{
"message":
"The arena {} has exceeded its run time and will be stopped"
.format(unit_id)
},
severity=LOG_LEVELS.INFO)
stop_arena(unit_id)
def delete_dns(build_id, ip_address):
"""
Deletes a DNS record based on the build_id host name and IP address.
:param build_id: The Datastore entity build_id, which is also the record host name.
:param ip_address: The IP address of of the record to delete.
:return: None
"""
# Use the parent project when modifying the DNS otherwise, if this IS the parent, then use the current project
target_project = parent_project if parent_project else project
try:
service = googleapiclient.discovery.build('dns', 'v1')
change_body = {"deletions": [
{
"kind": "dns#resourceRecordSet",
"name": build_id + dns_suffix + ".",
"rrdatas": [ip_address],
"type": "A",
"ttl": 30
},
]}
service.changes().create(project=target_project, managedZone=dnszone, body=change_body).execute()
except():
g_logger = log_client.logger(str(build_id))
g_logger.log_text("Error in deleting DNS record for workout {}".format(build_id), severity=LOG_LEVELS.ERROR)
return False
return True
def server_rebuild(server_name):
server = ds_client.get(ds_client.key('cybergym-server', server_name))
g_logger = log_client.logger(str(server_name))
g_logger.log_text(f"Rebuilding server {server_name}")
if 'state' in server:
if server['state'] == 'RUNNING':
server_stop(server_name)
server_delete(server_name)
server_build(server_name)
def add_active_directory_dns(build_id, ip_address, network):
"""
Add a DNS forwarding rule to support an Active Directory server acting as the default DNS server
@param build_id: The ID of the workout or arena with the DNS server
@type build_id: String
@param ip_address: IP address of the active directory server
@type ip_address: String
@param network: A list of network names to use for DNS forwarding
@type network: List
@return: Status
@rtype: Boolean
"""
service = googleapiclient.discovery.build('dns', 'v1')
g_logger = log_client.logger(build_id)
managed_zone_body = {
"kind": "dns#managedZone",
"name": build_id,
"dnsName": "cybergym.local.",
"visibility": "private",
"description": "",
"privateVisibilityConfig": {
"kind": "dns#managedZonePrivateVisibilityConfig",
"networks": []
},
"forwardingConfig": {
"kind": "dns#managedZoneForwardingConfig",
"targetNameServers": [
{
"kind": "dns#managedZoneForwardingConfigNameServerTarget",
"ipv4Address": ip_address,
"forwardingPath": "default"
}
],
}
}
forwarding_network = {
"kind": "dns#managedZonePrivateVisibilityConfigNetwork",
"networkUrl": f"https://www.googleapis.com/compute/v1/projects/{project}/global/networks/{network}"
}
managed_zone_body["privateVisibilityConfig"]["networks"].append(forwarding_network)
# Try first to perform the DNS change, but in case the DNS did not previously exist, try again without the deletion change.
try:
request = service.managedZones().create(project=project, body=managed_zone_body)
response = request.execute()
except HttpError as err:
error_text = ast.literal_eval(err.content.decode("UTF-8"))['error']['errors'][0]['message']
g_logger.log_text(f"Error while adding private DNS zone for Active Directory: {error_text}")
return False
return True
def stop_everything():
g_logger = log_client.logger('workout-actions')
result = compute.instances().list(project=project, zone=zone).execute()
if 'items' in result:
for vm_instance in result['items']:
response = compute.instances().stop(
project=project, zone=zone,
instance=vm_instance["name"]).execute()
g_logger.log_struct(
{
"message": "All machines stopped (daily cleanup)",
},
severity=LOG_LEVELS.INFO)
else:
g_logger.log_struct({"message": "No workouts to stop (daily cleanup)"},
severity=LOG_LEVELS.WARNING)
def get_server_ext_address(server_name):
"""
Provides the IP address of a given server name. Right now, this is used for managing DNS entries.
:param server_name: The server name in the cloud project
:return: The IP address of the server or throws an error
"""
g_logger = log_client.logger(str(server_name))
try:
new_instance = compute.instances().get(project=project,
zone=zone,
instance=server_name).execute()
ip_address = new_instance['networkInterfaces'][0]['accessConfigs'][0][
'natIP']
except KeyError:
g_logger.log_text('Server %s does not have an external IP address' %
server_name)
return False
return ip_address
def delete_active_directory_dns(managed_zone):
"""
Deletes the given managed zone
@param managed_zone: Managed zone for deleting
@type managed_zone: str
@return: Status
@rtype: bool
"""
g_logger = log_client.logger(managed_zone)
service = googleapiclient.discovery.build('dns', 'v1')
try:
request = service.managedZones().delete(project=project, managedZone=managed_zone)
request.execute()
except HttpError as err:
error_text = ast.literal_eval(err.content.decode("UTF-8"))['error']['errors'][0]['message']
g_logger.log_text(f"Error while adding private DNS zone for Active Directory: {error_text}")
return False
return True
def nuke_workout(workout_id):
"""
:param workout_id: The ID of the workout specification in the Datastore
:returns: None
"""
g_logger = log_client.logger(str(workout_id))
workout = ds_client.get(ds_client.key('cybergym-workout', workout_id))
g_logger.log_text("Nuke Operation: Delete")
state_transition(entity=workout, new_state=BUILD_STATES.NUKING)
DeletionManager(deletion_type=DeletionManager.DeletionType.SPECIFIC,
build_id=workout_id,
build_type=WORKOUT_TYPES.WORKOUT).run()
time.sleep(60)
g_logger.log_text("Nuke Operation: Rebuild")
state_transition(entity=workout, new_state=BUILD_STATES.START)
build_workout(workout_id)
return
def test_guacamole(ip_address, build_id=None):
max_attempts = 15
attempts = 0
g_logger = log_client.logger(str(build_id))
success = False
while not success and attempts < max_attempts:
try:
requests.get(f"http://{ip_address}:8080/guacamole/#", timeout=40)
return True
except requests.exceptions.Timeout:
g_logger.log_text(
f"Build {build_id}: Timeout {attempts} waiting for guacamole server connection."
)
attempts += 1
except ConnectionError:
g_logger.log_text(
f"HTTP Connection Error for Guacamole Server {ip_address}")
attempts += 1
return False
def stop_workout(workout_id):
result = compute.instances().list(
project=project, zone=zone,
filter='name = {}*'.format(workout_id)).execute()
workout = ds_client.get(ds_client.key('cybergym-workout', workout_id))
state_transition(entity=workout,
new_state=BUILD_STATES.READY,
existing_state=BUILD_STATES.RUNNING)
start_time = None
if 'start_time' in workout:
start_time = workout['start_time']
stop_time = calendar.timegm(time.gmtime())
runtime = int(stop_time) - int(start_time)
if 'runtime_counter' in workout:
accumulator = workout['runtime_counter']
new_runtime = int(accumulator) + runtime
workout['runtime_counter'] = new_runtime
else:
workout['runtime_counter'] = runtime
ds_client.put(workout)
query_workout_servers = ds_client.query(kind='cybergym-server')
query_workout_servers.add_filter("workout", "=", workout_id)
for server in list(query_workout_servers.fetch()):
# Publish to a server management topic
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path,
data=b'Server Build',
server_name=server['name'],
action=SERVER_ACTIONS.STOP)
print(future.result())
g_logger = log_client.logger(str(workout_id))
if 'items' in result:
for vm_instance in result['items']:
response = compute.instances().stop(
project=project, zone=zone,
instance=vm_instance["name"]).execute()
g_logger.log_struct({"message": "Workout stopped"},
severity=LOG_LEVELS.INFO)
else:
g_logger.log_struct({"message": "No workouts to stop"},
severity=LOG_LEVELS.WARNING)
def server_stop(server_name):
server = ds_client.get(ds_client.key('cybergym-server', server_name))
g_logger = log_client.logger(str(server_name))
state_transition(entity=server, new_state=SERVER_STATES.STOPPING)
i = 0
stop_success = False
while not stop_success and i < 5:
workout_globals.refresh_api()
try:
response = compute.instances().stop(
project=project, zone=zone, instance=server_name).execute()
stop_success = True
g_logger.log_text(
f'Sent job to start {server_name}, and waiting for response')
return True
except BrokenPipeError:
i += 1
return False
def cloud_fn_build_workout(event, context):
""" Responds to a pub/sub event in which the user has included
Args:
event (dict): The dictionary with data specific to this type of
event. The `data` field contains the PubsubMessage message. The
`attributes` field will contain custom attributes if there are any.
context (google.cloud.functions.Context): The Cloud Functions event
metadata. The `event_id` field contains the Pub/Sub message ID. The
`timestamp` field contains the publish time.
Returns:
A success status
"""
if not bm.check_budget():
cloud_log(
LogIDs.BUDGET_MANAGEMENT,
"BUDGET ALERT: Cannot run cloud_fn_build_workout because the budget "
"exceeded variable is set for the project", LOG_LEVELS.ERROR)
return
workout_id = event['attributes']['workout_id'] if 'workout_id' in event[
'attributes'] else None
action = event['attributes']['action'] if 'action' in event[
'attributes'] else None
if not workout_id:
print(
f'No workout ID provided in cloud_fn_build_workout for published message'
)
return
g_logger = log_client.logger(workout_id)
if action == WORKOUT_ACTIONS.NUKE:
g_logger.log_text(f"Nuking and rebuilding workout")
nuke_workout(workout_id)
else:
g_logger.log_text(f"Building workout")
build_workout(workout_id)
if context:
g_logger.log_text(f"Workout action has completed.")
def start_vm(workout_id):
g_logger = log_client.logger(str(workout_id))
g_logger.log_struct({"message": "Starting workout {}".format(workout_id)},
severity=LOG_LEVELS.INFO)
# print("Starting workout %s" % workout_id)
workout = ds_client.get(ds_client.key('cybergym-workout', workout_id))
state_transition(entity=workout, new_state=BUILD_STATES.STARTING)
workout['start_time'] = str(calendar.timegm(time.gmtime()))
ds_client.put(workout)
query_workout_servers = ds_client.query(kind='cybergym-server')
query_workout_servers.add_filter("workout", "=", workout_id)
for server in list(query_workout_servers.fetch()):
# Publish to a server management topic
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path,
data=b'Server Build',
server_name=server['name'],
action=SERVER_ACTIONS.START)
print(future.result())
def register_student_entry(build_id, server_name):
build = ds_client.get(ds_client.key('cybergym-workout', build_id))
if not build:
build = ds_client.get(ds_client.key('cybergym-unit', build_id))
g_logger = log_client.logger(str(build_id))
# Add the external_IP address for the workout. This allows easy deletion of the DNS record when deleting the arena
ip_address = get_server_ext_address(server_name)
add_dns_record(build_id, ip_address)
# Make sure the guacamole server actually comes up successfully before setting the workout state to ready
g_logger.log_text(
f"DNS record set for {server_name}. Now Testing guacamole connection. This may take a few minutes."
)
if test_guacamole(ip_address, build_id):
# Now, since this is the guacamole server, update the state of the workout to READY
g_logger.log_text(f"Setting the build {build_id} to ready")
state_transition(entity=build, new_state=BUILD_STATES.READY)
else:
state_transition(entity=build,
new_state=BUILD_STATES.GUACAMOLE_SERVER_LOAD_TIMEOUT)
# Return the IP address used for the server build function to set the server datastore element
return ip_address
def server_build(server_name):
"""
Builds an individual server based on the specification in the Datastore entity with name server_name.
:param server_name: The Datastore entity name of the server to build
:return: A boolean status on the success of the build
"""
server = ds_client.get(ds_client.key('cybergym-server', server_name))
build_id = server['workout']
g_logger = log_client.logger(str(server_name))
state_transition(entity=server, new_state=SERVER_STATES.BUILDING)
config = server['config'].copy()
"""
Currently, we need a workaround to insert the guacamole startup script because of a 1500 character limit on
indexed fields. The exclude_from_index does not work on embedded datastore fields
"""
if 'student_entry' in server and server['student_entry']:
config['metadata'] = {
'items': [{
"key": "startup-script",
"value": server['guacamole_startup_script']
}]
}
# Begin the server build and keep trying for a bounded number of additional 30-second cycles
i = 0
build_success = False
while not build_success and i < 5:
workout_globals.refresh_api()
try:
if server['add_disk']:
try:
image_config = {
"name":
server_name + "-disk",
"sizeGb":
server['add_disk'],
"type":
"projects/" + project + "/zones/" + zone +
"/diskTypes/pd-ssd"
}
response = compute.disks().insert(
project=project, zone=zone,
body=image_config).execute()
compute.zoneOperations().wait(
project=project, zone=zone,
operation=response["id"]).execute()
except HttpError as err:
# If the disk already exists (i.e. a nuke), then ignore
if err.resp.status in [409]:
pass
if server['build_type'] == BUILD_TYPES.MACHINE_IMAGE:
source_machine_image = f"projects/{project}/global/machineImages/{server['machine_image']}"
compute_beta = discovery.build('compute', 'beta')
response = compute_beta.instances().insert(
project=project,
zone=zone,
body=config,
sourceMachineImage=source_machine_image).execute()
else:
if "delayed_start" in server and server["delayed_start"]:
time.sleep(30)
response = compute.instances().insert(project=project,
zone=zone,
body=config).execute()
build_success = True
g_logger.log_text(
f'Sent job to build {server_name}, and waiting for response')
except BrokenPipeError:
i += 1
except HttpError as exception:
cloud_log(
build_id,
f"Error when trying to build {server_name}: {exception.reason}",
LOG_LEVELS.ERROR)
return False
i = 0
success = False
while not success and i < 5:
try:
g_logger.log_text(
f"Begin waiting for build operation {response['id']}")
compute.zoneOperations().wait(project=project,
zone=zone,
operation=response["id"]).execute()
success = True
except timeout:
i += 1
g_logger.log_text('Response timeout for build. Trying again')
pass
if success:
g_logger.log_text(f'Successfully built server {server_name}')
state_transition(entity=server,
new_state=SERVER_STATES.RUNNING,
existing_state=SERVER_STATES.BUILDING)
else:
g_logger.log_text(f'Timeout in trying to build server {server_name}')
state_transition(entity=server, new_state=SERVER_STATES.BROKEN)
return False
# If this is a student entry server, register the DNS
if 'student_entry' in server and server['student_entry']:
g_logger.log_text(f'Setting DNS record for {server_name}')
ip_address = register_student_entry(server['workout'], server_name)
server['external_ip'] = ip_address
ds_client.put(server)
server = ds_client.get(ds_client.key('cybergym-server', server_name))
# Now stop the server before completing
g_logger.log_text(f'Stopping {server_name}')
compute.instances().stop(project=project, zone=zone,
instance=server_name).execute()
state_transition(entity=server, new_state=SERVER_STATES.STOPPED)
# If no other servers are building, then set the workout to the state of READY.
check_build_state_change(build_id=build_id,
check_server_state=SERVER_STATES.STOPPED,
change_build_state=BUILD_STATES.READY)
def nvd_update():
"""
Instantiated by cloud function to keep the CVEs current in the cloud SQL database
"""
g_logger = log_client.logger("nvd_update")
runtimeconfig_client = runtimeconfig.Client()
myconfig = runtimeconfig_client.config('cybergym')
mysql_password = myconfig.get_variable('sql_password').value.decode(
"utf-8")
mysql_ip = myconfig.get_variable('sql_ip').value.decode("utf-8")
dbcon = pymysql.connect(host=mysql_ip,
user="******",
password=mysql_password,
db='cybergym',
charset='utf8mb4')
dbcur = dbcon.cursor()
setup_required = test_and_create_nvd_table(dbcon)
sql_insert_vuln = """
INSERT IGNORE INTO nvd_data (cve_id, vendor, product, attack_vector, complexity,
priv, ui, confidentiality, integrity, availability, description)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
if setup_required:
g_logger.log_text(
f"Initializing NVD data with current year of vulnerabilities")
today = datetime.today()
url = f"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{today.year}.json.gz"
else:
g_logger.log_text(f"Adding new recent vulnerabilities")
url = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-recent.json.gz"
ssl._create_default_https_context = ssl._create_unverified_context
local_filename = urllib.request.urlretrieve(url)
json_feed = json.loads(gzip.open(local_filename[0]).read())
print("Processing ", local_filename)
for cve in json_feed["CVE_Items"]:
try:
if len(cve['configurations']['nodes']) > 0:
if len(cve['configurations']['nodes'][0]['cpe_match']) > 0:
cpe = cve['configurations']['nodes'][0]['cpe_match'][0]
cpe_parts = cpe['cpe23Uri'].split(':')
cpe_vendor = cpe_parts[3]
cpe_product = cpe_parts[4]
cve_id = cve["cve"]["CVE_data_meta"]["ID"]
cwe = cve['cve']['problemtype']['problemtype_data'][0][
'description'][0]['value']
cvss = cve["impact"]["baseMetricV3"]["cvssV3"]
attack_vector = cvss["attackVector"]
complexity = cvss["attackComplexity"]
priv = cvss["privilegesRequired"]
ui = cvss["userInteraction"]
confidentiality = cvss["confidentialityImpact"]
integrity = cvss["integrityImpact"]
availability = cvss["availabilityImpact"]
vuln_description = dbcon.escape(
cve["cve"]["description"]["description_data"][0]
["value"])
vuln_args = (cve_id, cpe_vendor, cpe_product,
attack_vector, complexity, priv, ui,
confidentiality, integrity, availability,
vuln_description)
dbcur.execute(sql_insert_vuln, vuln_args)
except KeyError:
pass
dbcon.commit()
def medic():
"""
Reviews the state of all active workouts in the project and attempts to correct any which may have an invalid
state. Invalid states often occur due to timeouts in processing the Google Cloud Functions.
:returns: None
"""
g_logger = log_client.logger('workout-actions')
g_logger.log_text("MEDIC: Running Medic function")
#
# Fixing build timeout issues
#
# The add_filter does not have a != operator. This provides an equivalent results for active workouts.
query_current_workouts = ds_client.query(kind='cybergym-workout')
results = list(
query_current_workouts.add_filter('active', '=', True).fetch())
for workout in results:
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
if 'state' in workout:
build_state = workout['state']
# if the workout state has not completed, then attempt to continue rebuilding the workout from where
# it left off.
if build_state in ordered_workout_build_states:
g_logger.log_text(
"MEDIC: Workout {} is in a build state of {}. Attempting to fix..."
.format(workout.key.name, build_state))
build_workout(workout_id=workout.key.name)
elif type(workout) is datastore.entity.Entity:
# If there is no state, then this is not a valid workout, and we can delete the Datastore entity.
g_logger.log_text(
"Invalid workout specification in the datastore for workout ID: {}. Deleting the record."
.format(workout.key.name))
ds_client.delete(workout.key)
#
# Fixing workouts in state COMPLETED_FIREWALL. This may occur when the firewall gets built after the guacamole server
#
query_completed_firewalls = ds_client.query(kind='cybergym-workout')
results = list(
query_completed_firewalls.add_filter(
"state", "=", BUILD_STATES.COMPLETED_FIREWALL).fetch())
for workout in results:
# Only transition the state if the last state change occurred over 5 minutes ago.
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
if workout['state-timestamp'] < str(
calendar.timegm(time.gmtime()) - 300):
g_logger.log_text(
"MEDIC: Workout {} stuck in firewall completion. Changing state to READY"
.format(workout.key.name))
state_transition(workout, new_state=BUILD_STATES.RUNNING)
stop_workout(workout.key.name)
#
# Fixing workouts in state GUACAMOLE_SERVER_TIMEOUT. This may occur waiting for the guacamole server to come up
#
query_student_entry_timeouts = ds_client.query(kind='cybergym-workout')
results = list(
query_student_entry_timeouts.add_filter(
"state", "=", BUILD_STATES.GUACAMOLE_SERVER_LOAD_TIMEOUT).fetch())
for workout in results:
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
# Change this to RUNNING unless the state change occurred over 15 minutes ago
if workout['state-timestamp'] < str(
calendar.timegm(time.gmtime()) - 900):
g_logger.log_text(
"MEDIC: Workout {} stuck in guacamole timeout. Changing state to READY"
.format(workout.key.name))
state_transition(workout, new_state=BUILD_STATES.RUNNING)
stop_workout(workout.key.name)
else:
g_logger.log_text(
"MEDIC: Workout {} stuck in guacamole timeout. Changing state to READY"
.format(workout.key.name))
print(
f"Workout {workout.key.name} stuck in guacamole timeout. Changing state to READY"
)
state_transition(workout, new_state=BUILD_STATES.READY)
#
# Fixing workouts in the state of STARTING. This may occur after a timeout in starting workouts.
#
query_start_timeouts = ds_client.query(kind='cybergym-workout')
results = list(
query_start_timeouts.add_filter("state", "=",
BUILD_STATES.STARTING).fetch())
for workout in results:
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
# Only transition the state if the last state change occurred over 5 minutes ago.
if workout['state-timestamp'] < str(
calendar.timegm(time.gmtime()) - 300):
g_logger.log_text(
"MEDIC: Workout {} stuck in a STARTING state. Stopping the workout."
.format(workout.key.name))
state_transition(workout, new_state=BUILD_STATES.RUNNING)
stop_workout(workout.key.name)
#
# Fixing workouts in the state of STOPPING. This may occur after a timeout in stopping workouts.
#
query_stop_timeouts = ds_client.query(kind='cybergym-workout')
results = list(
query_stop_timeouts.add_filter("state", "=",
BUILD_STATES.STOPPING).fetch())
for workout in results:
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
# Only transition the state if the last state change occurred over 5 minutes ago.
if workout['state-timestamp'] < str(
calendar.timegm(time.gmtime()) - 300):
g_logger.log_text(
"MEDIC: Workout {} stuck in a STARTING state. Stopping the workout."
.format(workout.key.name))
state_transition(workout, new_state=BUILD_STATES.RUNNING)
stop_workout(workout.key.name)
#
# Fixing workouts in the state of NUKING. This may occur after a timeout in deleting the workouts.
#
query_nuking_timeouts = ds_client.query(kind='cybergym-workout')
results = list(
query_nuking_timeouts.add_filter("state", "=",
BUILD_STATES.NUKING).fetch())
for workout in results:
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
# Only transition the state if the last state change occurred over 5 minutes ago.
if workout['state-timestamp'] < str(
calendar.timegm(time.gmtime()) - 300):
g_logger.log_text(
"MEDIC: Workout {} stuck in a NUKING state. Attempting to nuke again."
.format(workout.key.name))
nuke_workout(workout.key.name)
#
#Fixing machines that did not get built
#
query_rebuild = ds_client.query(kind='cybergym-workout')
query_rebuild.add_filter('state', '=', BUILD_STATES.READY)
query_rebuild.add_filter('build_project_location', '=', project)
running_machines = list(query_rebuild.fetch())
current_machines = compute.instances().list(project=project,
zone=zone).execute()
list_current = []
list_running = []
list_missing = []
current_machines_items = current_machines.get('items', None)
while current_machines_items:
for instance in current_machines_items:
list_current.append(instance['name'])
if 'nextPageToken' in current_machines:
current_machines = compute.instances().list(
project=project,
zone=zone,
pageToken=current_machines['nextPageToken']).execute()
current_machines_items = current_machines.get('items', None)
else:
break
for i in running_machines:
unit = ds_client.get(ds_client.key('cybergym-unit', i['unit_id']))
if unit['build_type'] == 'arena':
for server in i['student_servers']:
datastore_server_name = i.key.name + '-' + server['name']
list_running.append(datastore_server_name)
if datastore_server_name not in list_current:
list_missing.append(datastore_server_name)
if unit['build_type'] == 'compute':
for server in i['servers']:
datastore_server_name = i.key.name + '-' + server['name']
list_running.append(datastore_server_name)
if datastore_server_name not in list_current:
list_missing.append(datastore_server_name)
cloud_log('Medic', f'Missing servers{list_missing}', LOG_LEVELS.INFO)
for server in list_missing:
cloud_log('Medic', 'Rebuilding server{}'.format(server),
LOG_LEVELS.INFO)
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path,
data=b'Server Build',
server_name=server,
action=SERVER_ACTIONS.BUILD)
return
def server_start(server_name):
"""
Starts a server based on the specification in the Datastore entity with name server_name. A guacamole server
is also registered with DNS.
:param server_name: The Datastore entity name of the server to start
:return: A boolean status on the success of the start
"""
server = ds_client.get(ds_client.key('cybergym-server', server_name))
state_transition(entity=server, new_state=SERVER_STATES.STARTING)
g_logger = log_client.logger(str(server_name))
# Begin the server start and keep trying for a bounded number of cycles
i = 0
start_success = False
while not start_success and i < 5:
workout_globals.refresh_api()
try:
if "delayed_start" in server and server["delayed_start"]:
time.sleep(30)
response = compute.instances().start(
project=project, zone=zone, instance=server_name).execute()
start_success = True
g_logger.log_text(
f'Sent job to start {server_name}, and waiting for response')
except BrokenPipeError:
i += 1
g_logger.log_text(
f'Sent start request to {server_name}, and waiting for response')
i = 0
success = False
while not success and i < 5:
try:
g_logger.log_text(
f"Begin waiting for start response from operation {response['id']}"
)
compute.zoneOperations().wait(project=project,
zone=zone,
operation=response["id"]).execute()
success = True
except timeout:
i += 1
g_logger.log_text(
'Response timeout for starting server. Trying again')
pass
if not success:
g_logger.log_text(f'Timeout in trying to start server {server_name}')
state_transition(entity=server, new_state=SERVER_STATES.BROKEN)
return False
# If this is the guacamole server for student entry, then register the new DNS
if 'student_entry' in server and server['student_entry']:
g_logger.log_text(f'Setting DNS record for {server_name}')
ip_address = register_student_entry(server['workout'], server_name)
server['external_ip'] = ip_address
state_transition(entity=server, new_state=SERVER_STATES.RUNNING)
g_logger.log_text(f"Finished starting {server_name}")
# If all servers have started, then change the build state
build_id = server['workout']
check_build_state_change(build_id=build_id,
check_server_state=SERVER_STATES.RUNNING,
change_build_state=BUILD_STATES.RUNNING)
return True
def server_delete(server_name):
g_logger = log_client.logger(str(server_name))
server_list = list(
ds_client.query(kind='cybergym-server').add_filter(
'name', '=', str(server_name)).fetch())
server_is_deleted = list(
ds_client.query(kind='cybergym-server').add_filter(
'name', '=', str(server_name)).add_filter('state', '=',
'DELETED').fetch())
if server_is_deleted and server_list:
g_logger.log_text(f'Server "' + server_name +
'" has already been deleted.')
return True
elif not server_list:
g_logger.log_text(f'Server of name "' + server_name +
'" does not exist in datastore, unable to Delete.')
return True
else:
server = ds_client.get(ds_client.key('cybergym-server', server_name))
state_transition(entity=server, new_state=SERVER_STATES.DELETING)
# If there are snapshots associated with this server, then delete the snapshots.
if 'snapshot' in server and server['snapshot']:
Snapshot.delete_snapshot(server_name)
workout_globals.refresh_api()
try:
response = compute.instances().delete(project=project,
zone=zone,
instance=server_name).execute()
except HttpError as exception:
# If the server is already deleted or no longer exists,
state_transition(entity=server, new_state=SERVER_STATES.DELETED)
g_logger.log_text(f"Finished deleting {server_name}")
# If all servers in the workout have been deleted, then set the workout state to True
build_id = server['workout']
check_build_state_change(
build_id=build_id,
check_server_state=SERVER_STATES.DELETED,
change_build_state=BUILD_STATES.COMPLETED_DELETING_SERVERS)
return True
g_logger.log_text(
f'Sent delete request to {server_name}, and waiting for response')
i = 0
success = False
while not success and i < 5:
try:
g_logger.log_text(
f"Begin waiting for delete response from operation {response['id']}"
)
compute.zoneOperations().wait(project=project,
zone=zone,
operation=response["id"]).execute()
success = True
except timeout:
i += 1
g_logger.log_text(
'Response timeout for deleting server. Trying again')
pass
if not success:
g_logger.log_text(f'Timeout in trying to delete server {server_name}')
state_transition(entity=server, new_state=SERVER_STATES.BROKEN)
return False
# If this is a student entry server, delete the DNS
if 'student_entry' in server and server['student_entry']:
g_logger.log_text(f'Deleting DNS record for {server_name}')
ip_address = server['external_ip']
delete_dns(server['workout'], ip_address)
state_transition(entity=server, new_state=SERVER_STATES.DELETED)
g_logger.log_text(f"Finished deleting {server_name}")
# If all servers in the workout have been deleted, then set the workout state to True
build_id = server['workout']
check_build_state_change(
build_id=build_id,
check_server_state=SERVER_STATES.DELETED,
change_build_state=BUILD_STATES.COMPLETED_DELETING_SERVERS)
return True
