def snapshot_all():
"""
Query all workouts over the past 4 months that have not been deleted and take a snapshot of each server where
indicated.
@return:
@rtype:
"""
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter('active', '=', True)
for workout in list(query_workouts.fetch()):
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if 'state' in workout and workout[
'state'] != BUILD_STATES.DELETED:
query_workout_servers = ds_client.query(
kind='cybergym-server')
query_workout_servers.add_filter("workout", "=",
workout.key.name)
for server in list(query_workout_servers.fetch()):
snapshot = server.get('snapshot', None)
if snapshot:
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(
project, pubsub_topic)
publisher.publish(topic_path,
data=b'Server Snapshot',
server_name=server['name'],
action=SERVER_ACTIONS.SNAPSHOT)
def delete_arenas():
"""
It is common for cloud functions to time out when deleting several workouts. This adds an additional work thread
to delete arenas similar to the delete_workouts() function
:return:
"""
query_old_units = ds_client.query(kind='cybergym-unit')
query_old_units.add_filter("timestamp", ">", str(calendar.timegm(time.gmtime()) - 10512000))
for unit in list(query_old_units.fetch()):
arena_type = False
if 'build_type' in unit and unit['build_type'] == 'arena':
arena_type = True
if arena_type:
try:
arena = unit['arena']
if workout_age(arena['timestamp']) >= int(arena['expiration']) \
and unit['state'] != BUILD_STATES.DELETED:
arena_id = unit.key.name
print('Deleting resources from arena %s' % arena_id)
if delete_specific_arena(arena_id, unit):
state_transition(unit, BUILD_STATES.DELETED)
except KeyError:
state_transition(unit, BUILD_STATES.DELETED)
# Delete any misfit arenas
query_misfit_arenas = ds_client.query(kind='cybergym-unit')
query_misfit_arenas.add_filter("misfit", "=", True)
for unit in list(query_misfit_arenas.fetch()):
arena_id = unit.key.name
print('Deleting resources from arena %s' % arena_id)
if delete_specific_arena(arena_id, unit):
ds_client.delete(unit.key)
print("Finished deleting arena %s" % arena_id)
def _delete_expired_arenas(self):
"""
It is common for cloud functions to time out when deleting several workouts. This adds an additional work thread
to delete arenas similar to the delete_workouts() function
:return:
"""
query_old_units = ds_client.query(kind='cybergym-unit')
query_old_units.add_filter(
"timestamp", ">",
str(calendar.timegm(time.gmtime()) - self.lookback_seconds))
for unit in list(query_old_units.fetch()):
arena_type = False
if 'build_type' in unit and unit['build_type'] == 'arena':
arena_type = True
if arena_type:
try:
unit_state = unit.get('state', None)
unit_deleted = True if not unit_state or unit_state == BUILD_STATES.DELETED else False
if self._workout_age(unit['timestamp']) >= int(unit['expiration']) \
and not unit_deleted:
state_transition(unit, BUILD_STATES.READY_DELETE)
pubsub_topic = PUBSUB_TOPICS.DELETE_EXPIRED
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(
project, pubsub_topic)
publisher.publish(topic_path,
data=b'Workout Delete',
workout_type=WORKOUT_TYPES.ARENA,
unit_id=unit.key.name)
except KeyError:
state_transition(unit, BUILD_STATES.DELETED)
cloud_log("delete_arenas", f"Deleted expired arenas", LOG_LEVELS.INFO)
def fix_server_in_unit(build_id, server_name, type, parameters):
"""
Fixes a server when something goes wrong in the unit
:@param unit_id: The unit_id to delete
:@param server: The server in the unit to act on
:@param type: The type of fix. The following types are supported
- ssh-key - Include the new public sshkey as a parameter
- image-correction - Include the new image name for the server as a parameter
:@param parameter: Parameter of the fix based on the type
"""
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter('unit_id', '=', build_id)
for workout in list(query_workouts.fetch()):
for server in workout['servers']:
if server['name'] == server_name:
if type == "ssh-key":
print(f"Begin setting ssh key for {workout.key.name}")
server['sshkey'] = parameters
print(f"Completed setting ssh key for {workout.key.name}")
elif type == "image-correction":
old_server_image = server['image']
print(f"Begin changing the image name in {workout.key.name} from {old_server_image} "
f"to {parameters}")
server['image'] = parameters
print(f"Completed image correction for {workout.key.name}")
ds_client.put(workout)
def stop_lapsed_arenas():
# Get the current time to compare with the start time to see if a workout needs to stop
ts = calendar.timegm(time.gmtime())
# Query all workouts which have not been deleted
query_units = ds_client.query(kind='cybergym-unit')
query_units.add_filter("arena.running", "=", True)
for unit in list(query_units.fetch()):
if 'arena' in unit and "gm_start_time" in unit[
'arena'] and "run_hours" in unit['arena']:
unit_id = unit.key.name
start_time = int(unit['arena'].get('gm_start_time', 0))
run_hours = int(unit['arena'].get('run_hours', 0))
# Stop the workout servers if the run time has exceeded the request
if ts - start_time >= run_hours * 3600:
g_logger = log_client.logger('arena-actions')
g_logger.log_struct(
{
"message":
"The arena {} has exceeded its run time and will be stopped"
.format(unit_id)
},
severity=LOG_LEVELS.INFO)
stop_arena(unit_id)
def stop_lapsed_workouts():
# Get the current time to compare with the start time to see if a workout needs to stop
ts = calendar.timegm(time.gmtime())
# Query all workouts which have not been deleted
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter("state", "=", BUILD_STATES.RUNNING)
for workout in list(query_workouts.fetch()):
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if "start_time" in workout and "run_hours" in workout and workout.get(
'type', 'arena') != 'arena':
workout_id = workout.key.name
start_time = int(workout.get('start_time', 0))
run_hours = int(workout.get('run_hours', 0))
# Stop the workout servers if the run time has exceeded the request
if ts - start_time >= run_hours * 3600:
g_logger = log_client.logger(str(workout_id))
g_logger.log_struct(
{
"message":
"The workout {} has exceeded its run time and will be stopped"
.format(workout_id)
},
severity=LOG_LEVELS.INFO)
stop_workout(workout_id)
def start_arena(unit_id):
g_logger = log_client.logger('arena-actions')
g_logger.log_struct({"message": "Starting arena {}".format(unit_id)},
severity=LOG_LEVELS.INFO)
unit = ds_client.get(ds_client.key('cybergym-unit', unit_id))
state_transition(entity=unit, new_state=BUILD_STATES.STARTING)
unit['arena']['running'] = True
unit['arena']['gm_start_time'] = str(calendar.timegm(time.gmtime()))
ds_client.put(unit)
# Start the central servers
g_logger.log_struct(
{"message": "Starting central servers for arena {}".format(unit_id)},
severity=LOG_LEVELS.INFO)
query_central_arena_servers = ds_client.query(kind='cybergym-server')
query_central_arena_servers.add_filter("workout", "=", unit_id)
for server in list(query_central_arena_servers.fetch()):
# Publish to a server management topic
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path,
data=b'Server Build',
server_name=server['name'],
action=SERVER_ACTIONS.START)
print(future.result())
# Now start all of the student workouts for this arena
for workout_id in unit['workouts']:
start_vm(workout_id)
def delete_all_active_units(keep_data=False):
"""
This will delete ALL active units and should be used only during special functions when no other activity is
occurring
:param keep_data: Specifies whether to mark the active workouts as misfits or change the expiration time to now.
"""
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter('active', '=', True)
for workout in list(query_workouts.fetch()):
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if 'state' in workout and workout['state'] != BUILD_STATES.DELETED:
if keep_data:
workout['expiration'] = 0
else:
workout['misfit'] = True
ds_client.put(workout)
print(
"All active workouts have been processed. Starting to process the delete workouts function"
)
if keep_data:
DeletionManager(
deletion_type=DeletionManager.DeletionType.EXPIRED).run()
else:
DeletionManager(
deletion_type=DeletionManager.DeletionType.MISFIT).run()
print("Sent commands to delete workouts and arenas")
def delete_workouts():
"""
Queries the data store for workouts which have expired. Workout expiration is defined during the build
process based on the number of days an instructor needs the workout to be available. Resources to delete include
servers, networks, routes, firewall-rules and DNS names. The deletion of resources is based on a unique
identifier for the workout. Every built resource uses this for a prefix.
Deletion must occur in a given order with networks being deleted last.
There is also a mechanism to delete what we refer to as misfit workouts. This is simply a boolean in the data store
to indicate when a workout was created by a mistake or some other error in processing.
This function is intended to be consumed through cloud_fn_delete_expired_workout and tied to a pubsub topic
triggered by a cloud scheduler to run every 15 minutes or more.
:return: None
"""
# Only process the workouts from the last 4 months. 10512000 is the number of seconds in a month
query_old_workouts = ds_client.query(kind='cybergym-workout')
query_old_workouts.add_filter("timestamp", ">", str(calendar.timegm(time.gmtime()) - 10512000))
for workout in list(query_old_workouts.fetch()):
if 'state' not in workout:
workout['state'] = BUILD_STATES.DELETED
container_type = False
if 'build_type' in workout and workout['build_type'] == 'container':
container_type = True
arena_type = False
if 'build_type' in workout and workout['build_type'] == 'arena' or \
'type' in workout and workout['type'] == 'arena':
arena_type = True
if workout['state'] != BUILD_STATES.DELETED and not container_type and not arena_type:
if workout_age(workout['timestamp']) >= int(workout['expiration']):
workout_id = workout.key.name
print('Deleting resources from workout %s' % workout_id)
if delete_specific_workout(workout_id, workout):
state_transition(workout, BUILD_STATES.DELETED)
query_misfit_workouts = ds_client.query(kind='cybergym-workout')
query_misfit_workouts.add_filter("misfit", "=", True)
for workout in list(query_misfit_workouts.fetch()):
workout_id = workout.key.name
print('Deleting resources from workout %s' % workout_id)
if delete_specific_workout(workout_id, workout):
ds_client.delete(workout.key)
print("Finished deleting workout %s" % workout_id)
def _process_workout_deletion(self):
"""
Since workouts are deleted asynchronously, this functions is called when the last step of workout deletion
occurs.
@param workout_id: The ID of the workout to query
@type workout_id: String
@return: None
@rtype: None
"""
if self.build_type == WORKOUT_TYPES.ARENA:
unit = ds_client.get(ds_client.key('cybergym-unit', self.build_id))
all_workouts_deleted = True
if unit:
is_misfit = unit['arena'].get('misfit', None)
# validate deletion state for student arena servers
for workout_id in unit['workouts']:
query_workout_servers = ds_client.query(
kind='cybergym-server')
query_workout_servers.add_filter("workout", "=",
workout_id)
server_list = list(query_workout_servers.fetch())
for server in server_list:
workout_state = server['state']
if workout_state != BUILD_STATES.DELETED:
all_workouts_deleted = False
# validate deletion state for student entry server
query_unit_server = ds_client.query(kind='cybergym-server')
query_unit_server.add_filter("workout", "=", self.build_id)
unit_server = list(query_unit_server.fetch())
student_entry_state = unit_server[0]['state']
if student_entry_state != BUILD_STATES.DELETED:
all_workouts_deleted = False
# if all machines have DELETED state, update arena state to DELETED
if all_workouts_deleted:
state_transition(unit, BUILD_STATES.DELETED)
if is_misfit:
ds_client.delete(unit)
elif self.build_type == WORKOUT_TYPES.WORKOUT:
workout = ds_client.get(
ds_client.key('cybergym-workout', self.build_id))
if workout:
state_transition(workout, BUILD_STATES.DELETED)
is_misfit = workout.get('misfit', None)
if is_misfit:
ds_client.delete(workout.key)
def update_registered_email(class_name, curr_email, new_email):
"""
Updates student email for all assigned workouts. Intended for cases where
student was registered in a class under an incorrect email address.
:param class_name: Name of the class we want to update student email in
:param curr_email: Email we want to update
:param new_email: Email we want to update with
"""
# Query target class
cybergym_class = ds_client.query(kind='cybergym-class')
cybergym_class.add_filter('class_name', '=', class_name)
cybergym_class_list = list(cybergym_class.fetch())
# Update class roster
for classes in cybergym_class_list:
for student in classes['roster']:
if student['student_email'] == curr_email:
print(f'[+] Update class roster with {new_email}')
student['student_email'] = new_email
ds_client.put(classes)
break
# Update all workouts for curr_email
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter('student_email', '=', curr_email)
for workout in list(query_workouts.fetch()):
workout['student_email'] = new_email
print(f'[*] Updating workout\'s student_email with {new_email}')
ds_client.put(workout)
# Finally, replace current email with new email in authed students list
query_auth_users = ds_client.query(kind='cybergym-admin-info')
for students in list(query_auth_users.fetch()):
for pos, student in enumerate(students['students']):
if student == curr_email:
students['students'][pos] = new_email
print(
f'[+] Replaced {curr_email} with {students["students"][pos]} in authed users list'
)
ds_client.put(students)
break
print('[+] Update complete!')
def _delete_expired_workouts(self):
"""
Queries the data store for workouts which have expired. Workout expiration is defined during the build
process based on the number of days an instructor needs the workout to be available. Resources to delete include
servers, networks, routes, firewall-rules and DNS names. The deletion of resources is based on a unique
identifier for the workout. Every built resource uses this for a prefix.
Deletion must occur in a given order with networks being deleted last.
This function is intended to be consumed through cloud_fn_delete_expired_workout and tied to a pubsub topic
triggered by a cloud scheduler to run every 15 minutes or more.
:return: None
"""
query_old_workouts = ds_client.query(kind='cybergym-workout')
query_old_workouts.add_filter('active', '=', True)
for workout in list(query_old_workouts.fetch()):
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if 'state' not in workout:
workout['state'] = BUILD_STATES.DELETED
container_type = False
# If the workout is a container, then expire the container so it does not show up in the list of running workouts.
if 'build_type' in workout and workout[
'build_type'] == 'container':
container_type = True
if 'timestamp' not in workout or 'expiration' not in workout and \
workout['state'] != BUILD_STATES.DELETED:
state_transition(workout, BUILD_STATES.DELETED)
elif self._workout_age(workout['timestamp']) >= int(workout['expiration']) and \
workout['state'] != BUILD_STATES.DELETED:
state_transition(workout, BUILD_STATES.DELETED)
arena_type = False
if 'build_type' in workout and workout['build_type'] == 'arena' or \
'type' in workout and workout['type'] == 'arena':
arena_type = True
if not container_type and not arena_type:
if self._workout_age(workout['timestamp']) >= int(
workout['expiration']):
if workout['state'] != BUILD_STATES.DELETED:
state_transition(workout,
BUILD_STATES.READY_DELETE)
workout_id = workout.key.name
# Delete the workouts asynchronously
pubsub_topic = PUBSUB_TOPICS.DELETE_EXPIRED
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(
project, pubsub_topic)
publisher.publish(
topic_path,
data=b'Workout Delete',
workout_type=WORKOUT_TYPES.WORKOUT,
workout_id=workout_id)
def extend_timeout_unit(unit_id, hours):
"""
Extend the number of days before the workout automatically expires for a given unit.
:param unit_id: The unit_id
:param days: Number of days to extend for the unit
"""
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter('unit_id', '=', unit_id)
for workout in list(query_workouts.fetch()):
current_expiration = int(workout['expiration'])
workout['expiration'] = f"{current_expiration + days}"
ds_client.put(workout)
def check_build_state_change(build_id, check_server_state, change_build_state):
query_workout_servers = ds_client.query(kind='cybergym-server')
query_workout_servers.add_filter("workout", "=", build_id)
for check_server in list(query_workout_servers.fetch()):
if check_server['state'] != check_server_state:
return
# If we've made it this far, then all of the servers have changed to the desired state.
# now we can change the entire state.
build = ds_client.get(ds_client.key('cybergym-workout', build_id))
if not build:
build = ds_client.get(ds_client.key('cybergym-unit', build_id))
state_transition(build, change_build_state)
def update_student_instructions_for_unit(unit_id, instructions_file):
"""
Update the student instructions with the specified file.
@param unit_id: The unit to update
@type unit_id: str
@param instructions_file: File to use in the Cloud storage under the globally defined student_instructions_url
@type instructions_file: str
"""
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter('unit_id', '=', unit_id)
for workout in list(query_workouts.fetch()):
workout[
'student_instructions_url'] = f"{student_instructions_url}{instructions_file}"
ds_client.put(workout)
def create_new_server_in_unit(unit_id, build_server_spec):
"""
Use this script when a new server is needed for an existing Unit. This is often helpful for semester long labs
in which you would like to modify the build environment.
@param unit_id: The unit_id to add the server to
@type unit_id: String
@param build_server_spec: The yaml specification file which holds the new server
@type build_server_spec: String
@param spec_folder: Folder where the specs are located
@type spec_folder: String
@return: None
@rtype: None
"""
spec_folder = "..\\build-files\\server-specs"
# Open and read YAML file
server_spec = os.path.join(spec_folder, build_server_spec)
with open(server_spec, "r") as f:
yaml_spec = yaml.load(f, Loader=yaml.SafeLoader)
name = yaml_spec['name']
custom_image = yaml_spec['image']
tags = yaml_spec['tags'] if 'tags' in yaml_spec else None
machine_type = yaml_spec['machine_type'] if 'machine_type' in yaml_spec else 'n1-standard-1'
network_routing = yaml_spec['network_routing'] if 'network_routing' in yaml_spec else False
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter('unit_id', '=', unit_id)
for workout in list(query_workouts.fetch()):
workout_project = workout.get('build_project_location', project)
if workout_project == project:
workout_id = workout.key.name
nics = []
for n in yaml_spec['nics']:
n['external_NAT'] = n['external_NAT'] if 'external_NAT' in n else False
nic = {
"network": f"{workout_id}-{n['network']}",
"internal_IP": n['internal_IP'],
"subnet": f"{workout_id}-{n['network']}-{n['subnet']}",
"external_NAT": n['external_NAT']
}
if 'IP_aliases' in n and n['IP_aliases']:
alias_ip_ranges = []
for ipaddr in n['IP_aliases']:
alias_ip_ranges.append({"ipCidrRange": ipaddr})
nic['aliasIpRanges'] = alias_ip_ranges
nics.append(nic)
create_instance_custom_image(compute=compute, workout=workout_id, name=f"{workout_id}-{name}",
custom_image=custom_image, machine_type=machine_type,
networkRouting=network_routing, networks=nics, tags=tags)
def nuke_rebuild_unit(unit_id):
"""
Nukes a full unit. This can be helpful, for example, if you've run out of quota
:param unit_id: The unit_id to delete
:param delete_key: Boolean on whether to delete the Datastore entity
"""
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter('unit_id', '=', unit_id)
for workout in list(query_workouts.fetch()):
workout_project = workout.get('build_project_location', project)
if workout_project == project:
print(f"Begin nuking and rebuilding workout {workout.key.name}")
nuke_workout(workout.key.name)
print(
f"Completed nuking and rebuilding workout {workout.key.name}")
def delete_vms(build_id):
"""
Send pubsub message for asynchronous deletion of all workout machines.
"""
print("Deleting computing resources for workout %s" % build_id)
query_workout_servers = ds_client.query(kind='cybergym-server')
query_workout_servers.add_filter("workout", "=", build_id)
for server in list(query_workout_servers.fetch()):
# Publish to a server management topic
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path, data=b'Server Delete', server_name=server['name'],
action=SERVER_ACTIONS.DELETE)
print(future.result())
def create_dns_forwarding(build_id, ip_address, network):
"""
Creates DNS forwarding rules for workouts with Active Directory servers. This script is useful
until a new build operation can be tested.
:@param unit_id: The unit_id to delete
:@param ip_address: The IP address of the domain controller.
:@param network: The name of the network to add in the forwarding rule.
"""
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter('unit_id', '=', build_id)
for workout in list(query_workouts.fetch()):
workout_project = workout.get('build_project_location', project)
if workout_project == project:
workout_id = workout.key.name
add_active_directory_dns(build_id=workout_id, ip_address=ip_address, network=f"{workout_id}-{network}")
def stop_lapsed_workouts():
# Get the current time to compare with the start time to see if a workout needs to stop
ts = calendar.timegm(time.gmtime())
# Query all workouts which have not been deleted
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter("running", "=", True)
for workout in list(query_workouts.fetch()):
if "start_time" in workout and "run_hours" in workout and workout.get(
'type', 'arena') != 'arena':
workout_id = workout.key.name
start_time = int(workout.get('start_time', 0))
run_hours = int(workout.get('run_hours', 0))
# Stop the workout servers if the run time has exceeded the request
if ts - start_time >= run_hours * 3600:
stop_workout(workout_id)
def start_vm(workout_id):
print("Starting workout %s" % workout_id)
workout = ds_client.get(ds_client.key('cybergym-workout', workout_id))
state_transition(entity=workout, new_state=BUILD_STATES.STARTING)
workout['start_time'] = str(calendar.timegm(time.gmtime()))
ds_client.put(workout)
query_workout_servers = ds_client.query(kind='cybergym-server')
query_workout_servers.add_filter("workout", "=", workout_id)
for server in list(query_workout_servers.fetch()):
# Publish to a server management topic
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path, data=b'Server Build', server_name=server['name'],
action=SERVER_ACTIONS.START)
print(future.result())
def stop_workout(workout_id):
result = compute.instances().list(
project=project, zone=zone,
filter='name = {}*'.format(workout_id)).execute()
workout = ds_client.get(ds_client.key('cybergym-workout', workout_id))
state_transition(entity=workout,
new_state=BUILD_STATES.READY,
existing_state=BUILD_STATES.RUNNING)
start_time = None
if 'start_time' in workout:
start_time = workout['start_time']
stop_time = calendar.timegm(time.gmtime())
runtime = int(stop_time) - int(start_time)
if 'runtime_counter' in workout:
accumulator = workout['runtime_counter']
new_runtime = int(accumulator) + runtime
workout['runtime_counter'] = new_runtime
else:
workout['runtime_counter'] = runtime
ds_client.put(workout)
query_workout_servers = ds_client.query(kind='cybergym-server')
query_workout_servers.add_filter("workout", "=", workout_id)
for server in list(query_workout_servers.fetch()):
# Publish to a server management topic
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path,
data=b'Server Build',
server_name=server['name'],
action=SERVER_ACTIONS.STOP)
print(future.result())
g_logger = log_client.logger(str(workout_id))
if 'items' in result:
for vm_instance in result['items']:
response = compute.instances().stop(
project=project, zone=zone,
instance=vm_instance["name"]).execute()
g_logger.log_struct({"message": "Workout stopped"},
severity=LOG_LEVELS.INFO)
else:
g_logger.log_struct({"message": "No workouts to stop"},
severity=LOG_LEVELS.WARNING)
def stop_lapsed_arenas():
# Get the current time to compare with the start time to see if a workout needs to stop
ts = calendar.timegm(time.gmtime())
# Query all workouts which have not been deleted
query_units = ds_client.query(kind='cybergym-unit')
query_units.add_filter("arena.running", "=", True)
for unit in list(query_units.fetch()):
if 'arena' in unit and "gm_start_time" in unit[
'arena'] and "run_hours" in unit['arena']:
unit_id = unit.key.name
start_time = int(unit['arena'].get('gm_start_time', 0))
run_hours = int(unit['arena'].get('run_hours', 0))
# Stop the workout servers if the run time has exceeded the request
if ts - start_time >= run_hours * 3600:
stop_arena(unit_id)
# stop_lapsed_arenas()
def _delete_misfits(self, misfit_type):
"""
Periodically a build will fail to build the complete workout. This results in what is defined as a misfit and
is unusable at that point.
@param misfit_type: Either workout or arena
@type misfit_type: String
@return:
@rtype:
"""
pubsub_topic = PUBSUB_TOPICS.DELETE_EXPIRED
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
query_kind = 'cybergym-unit' if misfit_type == WORKOUT_TYPES.ARENA else 'cybergym-workout'
query_misfits = ds_client.query(kind=query_kind)
query_misfits.add_filter('active', '=', True)
for build in list(query_misfits.fetch()):
workout_project = build.get('build_project_location', project)
if workout_project == project:
build_id = build.key.name
is_misfit = build.get('misfit', False)
current_state = build.get('state', None)
if is_misfit and current_state != BUILD_STATES.DELETED:
if misfit_type == WORKOUT_TYPES.ARENA:
state_transition(build, BUILD_STATES.READY_DELETE)
publisher.publish(topic_path,
data=b'Workout Delete',
workout_type=WORKOUT_TYPES.ARENA,
unit_id=build_id)
elif misfit_type == WORKOUT_TYPES.WORKOUT:
state_transition(build, BUILD_STATES.READY_DELETE)
publisher.publish(topic_path,
data=b'Workout Delete',
workout_type=WORKOUT_TYPES.WORKOUT,
workout_id=build_id)
cloud_log(LogIDs.DELETION_MANAGEMENT,
f"PubSub commands sent to delete misfit {misfit_type}",
LOG_LEVELS.INFO)
def get_unit_assessment(unit_id):
workout_list = ds_client.query(kind="cybergym-workout")
workout_list.add_filter('unit_id', '=', unit_id)
for workout in list(workout_list.fetch()):
if 'student_name' in workout and workout['student_name']:
output_str = ""
question_list = ""
if 'submitted_answers' in workout:
questionNum = 0
num_submitted = 0
for question in workout['assessment']['questions']:
questionNum += 1
if question['type'] == 'auto':
if question['complete'] == True:
num_submitted += 1
question_list += f"\n\t{questionNum}: Complete"
else:
question_list += f"\n\t{questionNum}: Not Completed"
for answer in workout['submitted_answers']:
if answer['question'] == question['question']:
if answer['answer']:
num_submitted += 1
question_list += f"\n\t{questionNum}: Submitted {answer['answer']}\tTrue Answer: {question['answer']}"
if answer['answer'].lower(
) == question['answer'].lower():
question_list += "\t\tGraded: Correct"
else:
question_list += "\t\tGraded: Incorrect"
else:
question_list += f"\n\t{questionNum}: Not Submitted"
output_str += f"=====\n{workout['student_name']} Submitted: {num_submitted}/{len(workout['assessment']['questions'])}"
if output_str:
print(output_str)
if question_list:
print(question_list)
def _delete_vms(self):
"""
Send pubsub message for asynchronous deletion of all workout machines.
By default it filters available instances based on self.build_id however for arena builds,
it is necessary to pass in value, workout_id as self.build_id is the unit_id for the arena.
"""
query_workout_servers = ds_client.query(kind='cybergym-server')
cloud_log(self.build_id,
f"Deleting computing resources for build {self.build_id}",
LOG_LEVELS.INFO)
query_workout_servers.add_filter("workout", "=", self.build_id)
for server in list(query_workout_servers.fetch()):
# Publish to a server management topic
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path,
data=b'Server Delete',
server_name=server['name'],
action=SERVER_ACTIONS.DELETE)
print(future.result())
def delete_unit(unit_id, delete_key=False, delete_immediately=False):
"""
Deletes a full unit when it was created on accident
:param unit_id: The unit_id to delete
:param delete_key: Boolean on whether to delete the Datastore entity
:param delete_immediately: Whether to delete immediately or create misfits and let the cloud function delete this.
"""
bm = BudgetManager()
query_workouts = ds_client.query(kind='cybergym-workout')
query_workouts.add_filter('unit_id', '=', unit_id)
for workout in list(query_workouts.fetch()):
workout['misfit'] = True
ds_client.put(workout)
print(
"All workouts marked as misfits. Starting to process the delete workouts function"
)
if bm.check_budget():
DeletionManager(
deletion_type=DeletionManager.DeletionType.MISFIT).run()
print("Completed deleting workouts")
else:
print(
"Cannot delete misfits. Budget exceeded variable is set for this project."
)
def medic():
"""
Reviews the state of all active workouts in the project and attempts to correct any which may have an invalid
state. Invalid states often occur due to timeouts in processing the Google Cloud Functions.
:returns: None
"""
g_logger = log_client.logger('workout-actions')
g_logger.log_text("MEDIC: Running Medic function")
#
# Fixing build timeout issues
#
# The add_filter does not have a != operator. This provides an equivalent results for active workouts.
query_current_workouts = ds_client.query(kind='cybergym-workout')
results = list(
query_current_workouts.add_filter('active', '=', True).fetch())
for workout in results:
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
if 'state' in workout:
build_state = workout['state']
# if the workout state has not completed, then attempt to continue rebuilding the workout from where
# it left off.
if build_state in ordered_workout_build_states:
g_logger.log_text(
"MEDIC: Workout {} is in a build state of {}. Attempting to fix..."
.format(workout.key.name, build_state))
build_workout(workout_id=workout.key.name)
elif type(workout) is datastore.entity.Entity:
# If there is no state, then this is not a valid workout, and we can delete the Datastore entity.
g_logger.log_text(
"Invalid workout specification in the datastore for workout ID: {}. Deleting the record."
.format(workout.key.name))
ds_client.delete(workout.key)
#
# Fixing workouts in state COMPLETED_FIREWALL. This may occur when the firewall gets built after the guacamole server
#
query_completed_firewalls = ds_client.query(kind='cybergym-workout')
results = list(
query_completed_firewalls.add_filter(
"state", "=", BUILD_STATES.COMPLETED_FIREWALL).fetch())
for workout in results:
# Only transition the state if the last state change occurred over 5 minutes ago.
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
if workout['state-timestamp'] < str(
calendar.timegm(time.gmtime()) - 300):
g_logger.log_text(
"MEDIC: Workout {} stuck in firewall completion. Changing state to READY"
.format(workout.key.name))
state_transition(workout, new_state=BUILD_STATES.RUNNING)
stop_workout(workout.key.name)
#
# Fixing workouts in state GUACAMOLE_SERVER_TIMEOUT. This may occur waiting for the guacamole server to come up
#
query_student_entry_timeouts = ds_client.query(kind='cybergym-workout')
results = list(
query_student_entry_timeouts.add_filter(
"state", "=", BUILD_STATES.GUACAMOLE_SERVER_LOAD_TIMEOUT).fetch())
for workout in results:
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
# Change this to RUNNING unless the state change occurred over 15 minutes ago
if workout['state-timestamp'] < str(
calendar.timegm(time.gmtime()) - 900):
g_logger.log_text(
"MEDIC: Workout {} stuck in guacamole timeout. Changing state to READY"
.format(workout.key.name))
state_transition(workout, new_state=BUILD_STATES.RUNNING)
stop_workout(workout.key.name)
else:
g_logger.log_text(
"MEDIC: Workout {} stuck in guacamole timeout. Changing state to READY"
.format(workout.key.name))
print(
f"Workout {workout.key.name} stuck in guacamole timeout. Changing state to READY"
)
state_transition(workout, new_state=BUILD_STATES.READY)
#
# Fixing workouts in the state of STARTING. This may occur after a timeout in starting workouts.
#
query_start_timeouts = ds_client.query(kind='cybergym-workout')
results = list(
query_start_timeouts.add_filter("state", "=",
BUILD_STATES.STARTING).fetch())
for workout in results:
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
# Only transition the state if the last state change occurred over 5 minutes ago.
if workout['state-timestamp'] < str(
calendar.timegm(time.gmtime()) - 300):
g_logger.log_text(
"MEDIC: Workout {} stuck in a STARTING state. Stopping the workout."
.format(workout.key.name))
state_transition(workout, new_state=BUILD_STATES.RUNNING)
stop_workout(workout.key.name)
#
# Fixing workouts in the state of STOPPING. This may occur after a timeout in stopping workouts.
#
query_stop_timeouts = ds_client.query(kind='cybergym-workout')
results = list(
query_stop_timeouts.add_filter("state", "=",
BUILD_STATES.STOPPING).fetch())
for workout in results:
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
# Only transition the state if the last state change occurred over 5 minutes ago.
if workout['state-timestamp'] < str(
calendar.timegm(time.gmtime()) - 300):
g_logger.log_text(
"MEDIC: Workout {} stuck in a STARTING state. Stopping the workout."
.format(workout.key.name))
state_transition(workout, new_state=BUILD_STATES.RUNNING)
stop_workout(workout.key.name)
#
# Fixing workouts in the state of NUKING. This may occur after a timeout in deleting the workouts.
#
query_nuking_timeouts = ds_client.query(kind='cybergym-workout')
results = list(
query_nuking_timeouts.add_filter("state", "=",
BUILD_STATES.NUKING).fetch())
for workout in results:
workout_project = workout.get('build_project_location', project)
if workout_project == project:
if get_workout_type(workout) == WORKOUT_TYPES.WORKOUT:
# Only transition the state if the last state change occurred over 5 minutes ago.
if workout['state-timestamp'] < str(
calendar.timegm(time.gmtime()) - 300):
g_logger.log_text(
"MEDIC: Workout {} stuck in a NUKING state. Attempting to nuke again."
.format(workout.key.name))
nuke_workout(workout.key.name)
#
#Fixing machines that did not get built
#
query_rebuild = ds_client.query(kind='cybergym-workout')
query_rebuild.add_filter('state', '=', BUILD_STATES.READY)
query_rebuild.add_filter('build_project_location', '=', project)
running_machines = list(query_rebuild.fetch())
current_machines = compute.instances().list(project=project,
zone=zone).execute()
list_current = []
list_running = []
list_missing = []
current_machines_items = current_machines.get('items', None)
while current_machines_items:
for instance in current_machines_items:
list_current.append(instance['name'])
if 'nextPageToken' in current_machines:
current_machines = compute.instances().list(
project=project,
zone=zone,
pageToken=current_machines['nextPageToken']).execute()
current_machines_items = current_machines.get('items', None)
else:
break
for i in running_machines:
unit = ds_client.get(ds_client.key('cybergym-unit', i['unit_id']))
if unit['build_type'] == 'arena':
for server in i['student_servers']:
datastore_server_name = i.key.name + '-' + server['name']
list_running.append(datastore_server_name)
if datastore_server_name not in list_current:
list_missing.append(datastore_server_name)
if unit['build_type'] == 'compute':
for server in i['servers']:
datastore_server_name = i.key.name + '-' + server['name']
list_running.append(datastore_server_name)
if datastore_server_name not in list_current:
list_missing.append(datastore_server_name)
cloud_log('Medic', f'Missing servers{list_missing}', LOG_LEVELS.INFO)
for server in list_missing:
cloud_log('Medic', 'Rebuilding server{}'.format(server),
LOG_LEVELS.INFO)
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path,
data=b'Server Build',
server_name=server,
action=SERVER_ACTIONS.BUILD)
return
def create_instance_custom_image(compute,
workout,
name,
custom_image,
machine_type,
networkRouting,
networks,
tags,
meta_data,
sshkey=None,
student_entry=False,
minCpuPlatform=None):
"""
Core function to create a new server according to the input specification. This gets called through
a cloud function during the automatic build
:param compute: A compute object to build the server
:param workout: The ID of the build
:param name: Name of the server
:param custom_image: Cloud image to use for the build
:param machine_type: The cloud machine type
:param networkRouting: True or False whether this is a firewall which routes traffic
:param networks: The NIC specification for this server
:param tags: Tags are key and value pairs which sometimes define the firewall rules
:param meta_data: This includes startup scripts
:param sshkey: If the server is running an SSH service, then this adds the public ssh key used for connections
:param student_entry: If this is a student_entry image, then add that to the configuration.
:return: None
"""
# First check to see if the server configuration already exists. If so, then return without error
exists_check = ds_client.query(kind='cybergym-server')
exists_check.add_filter("name", "=", name)
if exists_check.fetch().num_results > 0:
print(f'Server {name} already exists. Skipping configuration')
return
image_response = compute.images().get(project=project,
image=custom_image).execute()
source_disk_image = image_response['selfLink']
# Configure the machine
machine = "zones/%s/machineTypes/%s" % (zone, machine_type)
networkInterfaces = []
for network in networks:
if network["external_NAT"]:
accessConfigs = {'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT'}
else:
accessConfigs = None
add_network_interface = {
'network':
'projects/%s/global/networks/%s' % (project, network["network"]),
'subnetwork':
'regions/us-central1/subnetworks/' + network["subnet"],
'accessConfigs': [accessConfigs]
}
if 'internal_IP' in network:
add_network_interface['networkIP'] = network["internal_IP"]
if 'aliasIpRanges' in network:
add_network_interface['aliasIpRanges'] = network['aliasIpRanges']
networkInterfaces.append(add_network_interface)
config = {
'name':
name,
'machineType':
machine,
# allow http and https server with tags
'tags':
tags,
# Specify the boot disk and the image to use as a source.
'disks': [{
'boot': True,
'autoDelete': True,
'initializeParams': {
'sourceImage': source_disk_image,
}
}],
'networkInterfaces':
networkInterfaces,
# Allow the instance to access cloud storage and logging.
'serviceAccounts': [{
'email':
'default',
'scopes': [
'https://www.googleapis.com/auth/devstorage.read_write',
'https://www.googleapis.com/auth/logging.write'
]
}],
}
if meta_data:
config['metadata'] = {'items': meta_data}
if sshkey:
if 'items' in meta_data:
config['metadata']['items'].append({
"key": "ssh-keys",
"value": sshkey
})
else:
config['metadata'] = {
'items': {
"key": "ssh-keys",
"value": sshkey
}
}
# For a network routing firewall (i.e. Fortinet) add an additional disk for logging.
if networkRouting:
config["canIpForward"] = True
# Commented out because only Fortinet uses this. Need to create a custom build template instead.
# new_disk = {"mode": "READ_WRITE", "boot": False, "autoDelete": True,
# "source": "projects/" + project + "/zones/" + zone + "/disks/" + name + "-disk"}
# config['disks'].append(new_disk)
if minCpuPlatform:
config['minCpuPlatform'] = minCpuPlatform
new_server = datastore.Entity(ds_client.key('cybergym-server', f'{name}'))
new_server.update({
'name': name,
'workout': workout,
'config': config,
'state': SERVER_STATES.READY,
'state-timestamp': str(calendar.timegm(time.gmtime())),
'student_entry': student_entry
})
ds_client.put(new_server)
# Publish to a server build topic
pubsub_topic = PUBSUB_TOPICS.MANAGE_SERVER
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project, pubsub_topic)
future = publisher.publish(topic_path,
data=b'Server Build',
server_name=name,
action=SERVER_ACTIONS.BUILD)
print(future.result())
def server_delete(server_name):
g_logger = log_client.logger(str(server_name))
server_list = list(
ds_client.query(kind='cybergym-server').add_filter(
'name', '=', str(server_name)).fetch())
server_is_deleted = list(
ds_client.query(kind='cybergym-server').add_filter(
'name', '=', str(server_name)).add_filter('state', '=',
'DELETED').fetch())
if server_is_deleted and server_list:
g_logger.log_text(f'Server "' + server_name +
'" has already been deleted.')
return True
elif not server_list:
g_logger.log_text(f'Server of name "' + server_name +
'" does not exist in datastore, unable to Delete.')
return True
else:
server = ds_client.get(ds_client.key('cybergym-server', server_name))
state_transition(entity=server, new_state=SERVER_STATES.DELETING)
# If there are snapshots associated with this server, then delete the snapshots.
if 'snapshot' in server and server['snapshot']:
Snapshot.delete_snapshot(server_name)
workout_globals.refresh_api()
try:
response = compute.instances().delete(project=project,
zone=zone,
instance=server_name).execute()
except HttpError as exception:
# If the server is already deleted or no longer exists,
state_transition(entity=server, new_state=SERVER_STATES.DELETED)
g_logger.log_text(f"Finished deleting {server_name}")
# If all servers in the workout have been deleted, then set the workout state to True
build_id = server['workout']
check_build_state_change(
build_id=build_id,
check_server_state=SERVER_STATES.DELETED,
change_build_state=BUILD_STATES.COMPLETED_DELETING_SERVERS)
return True
g_logger.log_text(
f'Sent delete request to {server_name}, and waiting for response')
i = 0
success = False
while not success and i < 5:
try:
g_logger.log_text(
f"Begin waiting for delete response from operation {response['id']}"
)
compute.zoneOperations().wait(project=project,
zone=zone,
operation=response["id"]).execute()
success = True
except timeout:
i += 1
g_logger.log_text(
'Response timeout for deleting server. Trying again')
pass
if not success:
g_logger.log_text(f'Timeout in trying to delete server {server_name}')
state_transition(entity=server, new_state=SERVER_STATES.BROKEN)
return False
# If this is a student entry server, delete the DNS
if 'student_entry' in server and server['student_entry']:
g_logger.log_text(f'Deleting DNS record for {server_name}')
ip_address = server['external_ip']
delete_dns(server['workout'], ip_address)
state_transition(entity=server, new_state=SERVER_STATES.DELETED)
g_logger.log_text(f"Finished deleting {server_name}")
# If all servers in the workout have been deleted, then set the workout state to True
build_id = server['workout']
check_build_state_change(
build_id=build_id,
check_server_state=SERVER_STATES.DELETED,
change_build_state=BUILD_STATES.COMPLETED_DELETING_SERVERS)
return True
