def get_user_groups(self, user): """ Get the groups this users is authorized under for this Arena @param user: Email address of authenticated user @type user: str @return: List of groups assigned to the user @rtype: list """ user_groups = [] for group in self.UserGroups.ALL_GROUPS: if user in self.admin_info[group]: user_groups.append(group) if not user_groups and user not in self.admin_info[ self.UserGroups.PENDING]: cloud_log( LogIDs.USER_AUTHORIZATION, f"Unauthorized User: {user}. Adding to pending authorization", LOG_LEVELS.ERROR) self.admin_info[self.UserGroups.PENDING].append(user) ds_client.put(self.admin_info) cloud_log(LogIDs.USER_AUTHORIZATION, f"{user} logged in under groups {user_groups}", LOG_LEVELS.INFO) return user_groups
def _parse_container(self): container_info = self.build_spec['container_info'] assessment = self.build_spec.get('assessment', None) host_name = self.build_spec['workout'].get('host_name', None) if not host_name: error_message = f"Container build does not contain a host name in its specification." cloud_log(LogIDs.MAIN_APP, error_message, LOG_LEVELS.ERROR) raise InvalidBuildSpecification(error_message) container_url = f"https://{host_name}{dns_suffix}" for i in range(self.build_count): student_name = self.student_names[i] if self.student_names else None student_email = self.student_emails[ i] if self.student_emails else None cloud_ready_spec = { 'unit_id': self.unit_id, 'build_type': BuildTypes.CONTAINER, 'type': self.spec_name, 'workout_name': self.workout_name, 'expiration': self.workout_length, 'time_expiry': self.time_expiry, 'container_url': container_url, 'student_instructions_url': self.student_instructions_url, 'time_created': datetime.now().strftime("%Y-%m-%d %H:%M:%S"), 'timestamp': str(calendar.timegm(time.gmtime())), 'complete': False, 'container_info': container_info, 'assessment': assessment, 'state': BUILD_STATES.RUNNING, 'instructor_email': self.email, 'student_name': student_name, 'registration_required': self.registration_required, 'student_email': student_email } self.cloud_ready_specs.append(cloud_ready_spec)
def commit_to_cloud(self): """ Commits the parsed workout specification for multiple student builds to the cloud datastore. This first stores the unit to the datastore and then all the individual builds @return: None """ cloud_log(LogIDs.MAIN_APP, f"Creating unit {self.unit_id}", LOG_LEVELS.INFO) new_unit = datastore.Entity( ds_client.key('cybergym-unit', self.unit_id)) new_unit.update(self.new_unit) workout_ids = [] for cloud_ready_spec in self.cloud_ready_specs: workout_id = ''.join( random.choice(string.ascii_lowercase) for i in range(10)) workout_ids.append(workout_id) new_workout = datastore.Entity( ds_client.key('cybergym-workout', workout_id)) new_workout.update(cloud_ready_spec) ds_client.put(new_workout) # Store the server specifications for compute workouts if self.build_type == BuildTypes.COMPUTE: self._commit_workout_servers(workout_id, new_workout) new_unit['workouts'] = workout_ids ds_client.put(new_unit) # If this is an arena, then store all server configurations at this time. if self.build_type == BuildTypes.ARENA: CompetitionServerSpecToCloud( unit=new_unit, workout_ids=workout_ids, workout_specs=self.cloud_ready_specs).commit_to_cloud() return {'unit_id': self.unit_id, 'build_type': self.build_type}
def validate_machine_images(self): """ Args: self.image_list: List of images used in workout Returns: Dict of status and error message. Status is False if at least one image doesn't exist for that project. """ if self.image_list: images_state = { 'status': True, 'bad_images': [], } for image in self.image_list: try: compute.images().get(project=project, image=image).execute() except HttpError as e: # Image doesn't exist; append bad image to list and log event images_state['bad_images'].append(image) logger.error('%s' % e.error_details[0]["message"]) # Images in image_list doesn't exist in current project if len(images_state['bad_images']) > 0: error_message = f"Compute images not found! The following compute images do not exist in the " \ f"cloud project: {images_state['bad_images']}" cloud_log(LogIDs.MAIN_APP, error_message, LOG_LEVELS.ERROR) raise self.ImageNotFound(error_message) # All images exist in current project return images_state
def login(): if request.method == 'POST': user_data = request.get_json(force=True) if 'user_email' in user_data: arena_auth = ArenaAuthorizer() user_groups = arena_auth.get_user_groups(user_data['user_email']) if user_groups: session['user_email'] = user_data['user_email'] session['user_groups'] = user_groups cloud_log(LogIDs.MAIN_APP, f"User {user_data['user_email']} logged in", LOG_LEVELS.INFO) return json.dumps({"redirect": "/home"}) return json.dumps({'redirect': '/unauthorized'}) return render_template('login.html', auth_config=auth_config, error_resp='403')
def _get_build_allocation(self): """ Identify the build allocation among the parent and all children based on available resources and identified quotas. @param workout_count: Number of workouts to build in this request @type workout_count: Integer @return: The parent/or children projects to use, which may split across multiple projects @rtype: Dict {'<parent>': 80, '<child1>': 300, '<child2>': 300} """ # First, get all available projects. available = {project: self.MAX_BUILDS} admin_info = ds_client.get( ds_client.key(AdminInfoEntity.KIND, 'cybergym')) if AdminInfoEntity.Entities.CHILD_PROJECTS in admin_info: for child_project in admin_info[ AdminInfoEntity.Entities.CHILD_PROJECTS]: available[child_project] = self.MAX_BUILDS # Next find all active workouts for the application query_workouts = ds_client.query(kind='cybergym-workout') query_workouts.add_filter('active', '=', True) # Determine the available build left for each parent and children projects. for workout in list(query_workouts.fetch()): workout_project = workout.get('build_project_location', project) if workout_project not in available: cloud_log( LogIDs.MAIN_APP, f"Error in workout specification. The project {workout_project} is not " f"a valid project for this application.", LOG_LEVELS.ERROR) return False available[workout_project] -= 1 # Set the build allocation for this project based on availability. for _project in available: workouts_available = available[_project] if workouts_available > 0: _project_allocation = min(self.remaining_count, workouts_available) self.allocation[_project] = _project_allocation self.remaining_count -= _project_allocation if self.remaining_count > 0: cloud_log( LogIDs.MAIN_APP, "Error: Not enough available resources to complete the build!", LOG_LEVELS.ERROR) return False else: return True
def build_workout(): """ Initiates a cloud build for the specified workout ID @return: None """ data = request.get_json(force=True) workout_id = data.get('workout_id', None) if not workout_id: return redirect(404) publisher = pubsub_v1.PublisherClient() topic_path = publisher.topic_path(project, workout_globals.ps_build_workout_topic) publisher.publish(topic_path, data=b'Cyber Gym Workout', workout_id=workout_id) cloud_log(workout_id, f"Student initiated cloud build for workout {workout_id}", LOG_LEVELS.INFO) return 'Workout Built'
def __init__(self, unit, workout_ids, workout_specs): self.unit_id = unit.key.name self.unit = unit self.workout_specs = workout_specs self.workout_ids = workout_ids self.arena = self.unit.get('arena', None) if not self.arena: cloud_log( self.unit_id, f"Build operation failed. No unit {self.unit_id} exists in the data store", LOG_LEVELS.ERROR) raise LookupError self.student_entry_type = self.arena['student_entry_type'] self.student_entry_server = self.arena['student_entry'] self.student_entry_username = self.arena['student_entry_username'] self.student_entry_password = self.arena['student_entry_password'] self.network_type = self.arena['student_network_type']
def __init__(self): self.admin_info = ds_client.get( ds_client.key('cybergym-admin-info', 'cybergym')) if self.UserGroups.ADMINS not in self.admin_info: admin_email = myconfig.get_variable.config('admin_email') if not admin_email: cloud_log(LogIDs.MAIN_APP, f"Error: Admin Email is not set for this project!", LOG_LEVELS.ERROR) else: self.admin_info[self.UserGroups.ADMINS] = [admin_email] if self.UserGroups.AUTHORIZED not in self.admin_info: self.admin_info[self.UserGroups.AUTHORIZED] = [] if self.UserGroups.STUDENTS not in self.admin_info: self.admin_info[self.UserGroups.STUDENTS] = [] if self.UserGroups.PENDING not in self.admin_info: self.admin_info[self.UserGroups.PENDING] = [] ds_client.put(self.admin_info)
def commit_to_cloud(self): """ Store the student entry server specification for the given workout. @return: None @rtype: """ self._prepare_guac_connections() guac_startup_script = self._get_guac_startup_script( self.guac_connections) student_entry_ip = self._get_student_entry_ip_address(self.build, self.student_entry_network) \ if not self.student_entry_ip_address else self.student_entry_ip_address if not student_entry_ip: cloud_log( LogIDs.MAIN_APP, "Could not find available IP address for student entry guacamole server", LOG_LEVELS.ERROR) raise LookupError server_spec = { 'name': "student-guacamole", 'image': self.STUDENT_ENTRY_IMAGE, 'tags': { 'items': ['student-entry'] }, 'machine_type': 'n1-standard-1', 'nics': [{ "network": self.student_entry_network, "subnet": "default", "external_NAT": True, "internal_IP": student_entry_ip }], 'guacamole_startup_script': guac_startup_script } self._update_build_spec() ServerSpecToCloud(server_spec, self.build_id, student_entry=True).commit_to_cloud()
def parse_workout_yaml(yaml_filename): """ Pull and validate yaml contents from a designated project cloud bucket :param yaml_filename: Both the work type and filename of the yaml file in the cloud bucket :return: The yaml string from the cloud bucket """ bucket = storage_client.get_bucket(workout_globals.yaml_bucket) blob = bucket.get_blob(workout_globals.yaml_folder + yaml_filename + ".yaml") if not blob: cloud_log(LogIDs.MAIN_APP, f"YAML file not found: {yaml_filename}", LOG_LEVELS.ERROR) raise FileNotFoundError yaml_string = blob.download_as_string() try: y = yaml.safe_load(yaml_string) except yaml.YAMLError as err: cloud_log(LogIDs.MAIN_APP, f"Error parsing specified yaml file: {yaml_filename}", LOG_LEVELS.ERROR) if hasattr(err, 'problem_mark'): mark = err.problem_mark cloud_log(LogIDs.MAIN_APP, f"Error position: ({mark.line + 1}:{mark.column + 1})", LOG_LEVELS.ERROR) raise return y
def _validate_build_spec(self): """ Use the marshmallow schemas for validation only before using the data loaded from the yaml @param build_spec: Build specification pulled from the yaml @type build_spec: @return: None @rtype: """ try: if self.build_type == BuildTypes.COMPUTE: validation_result = WorkoutComputeSchema().validate( self.build_spec) elif self.build_type == BuildTypes.CONTAINER: validation_result = WorkoutContainerSchema().validate( self.build_spec) elif self.build_type == BuildTypes.ARENA: validation_result = ArenaSchema().validate(self.build_spec) except ValidationError as err: error_message = f"Error when trying to load build specification of type {self.build_type}. " \ f"Validation errors: {err.messages}" cloud_log(LogIDs.MAIN_APP, error_message, LOG_LEVELS.ERROR) raise InvalidBuildSpecification(error_message) return
def commit_to_cloud(self): config = {'name': self.server_name} if self.machine_type: config[ 'machineType'] = f"zones/{zone}/machineTypes/{self.machine_type}" if self.tags: config['tags'] = self.tags if self.build_type != ComputeBuildTypes.MACHINE_IMAGE: image_response = compute.images().get( project=project, image=self.custom_image).execute() source_disk_image = image_response['selfLink'] config['disks'] = [{ 'boot': True, 'autoDelete': True, 'initializeParams': { 'sourceImage': source_disk_image, } }] if self.networks: network_interfaces = [] for network in self.networks: if network.get("external_NAT", None): accessConfigs = { 'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT' } else: accessConfigs = None add_network_interface = { 'network': f'projects/{project}/global/networks/{network["network"]}', 'subnetwork': f'regions/us-central1/subnetworks/{network["subnet"]}', 'accessConfigs': [accessConfigs] } if 'internal_IP' in network: add_network_interface['networkIP'] = network["internal_IP"] if 'aliasIpRanges' in network: add_network_interface['aliasIpRanges'] = network[ 'aliasIpRanges'] network_interfaces.append(add_network_interface) config['networkInterfaces'] = network_interfaces # Allow the instance to access cloud storage and logging. config['serviceAccounts'] = [{ 'email': 'default', 'scopes': [ 'https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write' ] }] if self.meta_data: config['metadata'] = {'items': [self.meta_data]} if self.sshkey: if 'metadata' in config and config[ 'metadata'] and 'items' in config['metadata']: config['metadata']['items'].append({ "key": "ssh-keys", "value": self.sshkey }) else: config['metadata'] = { 'items': [{ "key": "ssh-keys", "value": self.sshkey }] } if self.network_routing: config["canIpForward"] = True if self.add_disk: new_disk = { "mode": "READ_WRITE", "boot": False, "autoDelete": True, "source": "projects/" + project + "/zones/" + zone + "/disks/" + self.server_name + "-disk" } config['disks'].append(new_disk) if self.min_cpu_platform: config['minCpuPlatform'] = self.min_cpu_platform new_server = datastore.Entity( key=ds_client.key('cybergym-server', self.server_name), exclude_from_indexes=['guacamole_startup_script']) new_server.update({ 'name': self.server_name, 'workout': self.build_id, 'build_type': self.build_type, 'machine_image': self.machine_image, 'config': config, 'add_disk': self.add_disk, 'state': BUILD_STATES.READY, 'state-timestamp': str(calendar.timegm(time.gmtime())), 'student_entry': self.student_entry, 'guacamole_startup_script': self.guacamole_startup_script, 'snapshot': self.snapshot }) if self.options: self._process_server_options(new_server, self.options) try: ds_client.put(new_server) except: cloud_log(LogIDs.MAIN_APP, f"Error storing server config for {self.server_name}", LOG_LEVELS.ERROR) raise
def __init__(self, server_spec, build_id, startup_scripts=None, student_entry=False, options=None): """ Initialize the server specification from a build specification. @param server_spec: The raw build specification for the server. @type server_spec: dict @param build_id: The ID associated with the build @type build_id: str @param startup_scripts: The startup scripts for all servers associated with this build @type startup_scripts: dict @param student_entry: Whether this is a student entry server @type: bool @param options: Additional options to include with the server @type options: dict """ base_name = server_spec['name'] self.server_name = f"{build_id}-{base_name}" # First check to see if the server configuration already exists. If so, then return without error exists_check = ds_client.query(kind='cybergym-server') exists_check.add_filter("name", "=", self.server_name) if exists_check.fetch().num_results > 0: cloud_log( LogIDs.MAIN_APP, f"The server {self.server_name} already exists. Skipping configuration.", LOG_LEVELS.ERROR) raise ReferenceError self.build_id = build_id self.custom_image = server_spec.get('image', None) self.build_type = server_spec.get("build_type", None) self.machine_image = server_spec.get("machine_image", None) self.sshkey = server_spec.get("sshkey", None) self.tags = server_spec.get('tags', None) self.machine_type = server_spec.get("machine_type", "n1-standard-1") self.network_routing = server_spec.get("network_routing", None) self.min_cpu_platform = server_spec.get("minCpuPlatform", None) self.add_disk = server_spec.get("add_disk", None) self.options = server_spec.get("options", None) self.snapshot = server_spec.get('snapshot', None) # Add the network configuration self.networks = [] for n in server_spec['nics']: n['external_NAT'] = n[ 'external_NAT'] if 'external_NAT' in n else False nic = { "network": f"{build_id}-{n['network']}", "internal_IP": n['internal_IP'], "subnet": f"{build_id}-{n['network']}-{n['subnet']}", "external_NAT": n['external_NAT'] } # Nested VMs are sometimes used for vulnerable servers. This adds those specified IP addresses as # aliases to the NIC if 'IP_aliases' in n and n['IP_aliases']: alias_ip_ranges = [] for ipaddr in n['IP_aliases']: alias_ip_ranges.append({"ipCidrRange": ipaddr}) nic['aliasIpRanges'] = alias_ip_ranges self.networks.append(nic) # Competitions may have meta_data defined, but compute workouts use assessments. First, check for meta_data # if this is a competition, and then look for startup scripts which have been identified from the assessment self.meta_data = server_spec.get("meta_data", None) if not self.meta_data and startup_scripts and base_name in startup_scripts: self.meta_data = startup_scripts[base_name] self.student_entry = student_entry self.guacamole_startup_script = server_spec.get( 'guacamole_startup_script', None) self.options = options