def read_config(): # 读取已储存的学号密码密钥 rf = open('config', 'rb') nonce, ciphertext, tag = [rf.read(x) for x in (16, 64, -1)] rf.close() data = encrypt_utils.decrypt(nonce, ciphertext, tag) if not data: return read_config() name, password = data.split(split_text) return {'name': name, 'pass': password}
def get(self, request): try: # admin_id = request.employee.id admin_id = 1 ### not sure which one will work admin_auth = Employee.objects.get(id=admin_id).auth.id # admin_auth = request.employee.auth # if the user is not an admin then... if admin_auth != 1: return JsonResponse({"message": "NO_AUTHORIZATION"}, status=403) # pagination limit = 6 queries = dict(request.GET) if queries.get('offset'): offset = int(queries['offset'][0]) else: offset = 0 if offset > len(employee_list): return JsonRespose({'message': 'OFFSET_OUT_OF_RANGE'}, status=400) # search filter by name_kor if queries.get('search'): conditions = [] search_list = queries.get('search')[0].split(' ') for name in search_list: conditions.append(Q(name_kor__icontains=name)) employee_list = Employee.objects.filter(reduce(OR, conditions)) else: employee_list = Employee.objects.all() # pagination hr_mgmt_page_list = [hr for hr in employee_list ][offset:offset + limit] returning_list = [{ 'id': hr.id, 'no': hr_mgmt_page_list.index(hr) + 1, 'name': hr.name_kor, 'nickname': hr.nickname, 'mobile': hr.mobile, 'dob': encrypt_utils.decrypt( hr.rrn, my_settings.SECRET.get('random')).decode('utf-8')[:6], 'email': hr.company_email, 'joined_at': EmployeeDetail.objects.get(employee_id=hr.id).joined_at } for hr in hr_mgmt_page_list] return JsonResponse( { "employees": returning_list, "total_employees": len(employee_list) }, status=200) except ValueError as e: return JsonResponse({"message": f"VALUE_ERROR:{e}"}, status=400)
def decryption(info): return encrypt_utils.decrypt( target_employee[info], my_settings.SECRET.get('random')).decode('utf-8')
def execute_actor(actor_id, worker_id, execution_id, image, msg, user=None, d={}, privileged=False, mounts=[], leave_container=False, fifo_host_path=None, socket_host_path=None, mem_limit=None, max_cpus=None, tenant=None): """ Creates and runs an actor container and supervises the execution, collecting statistics about resource consumption from the Docker daemon. :param actor_id: the dbid of the actor; for updating worker status :param worker_id: the worker id; also for updating worker status :param execution_id: the id of the execution. :param image: the actor's image; worker must have already downloaded this image to the local docker registry. :param msg: the message being passed to the actor. :param user: string in the form {uid}:{gid} representing the uid and gid to run the command as. :param d: dictionary representing the environment to instantiate within the actor container. :param privileged: whether this actor is "privileged"; i.e., its container should run in privileged mode with the docker daemon mounted. :param mounts: list of dictionaries representing the mounts to add; each dictionary mount should have 3 keys: host_path, container_path and format (which should have value 'ro' or 'rw'). :param fifo_host_path: If not None, a string representing a path on the host to a FIFO used for passing binary data to the actor. :param socket_host_path: If not None, a string representing a path on the host to a socket used for collecting results from the actor. :param mem_limit: The maximum amount of memory the Actor container can use; should be the same format as the --memory Docker flag. :param max_cpus: The maximum number of CPUs each actor will have available to them. Does not guarantee these CPU resources; serves as upper bound. :return: result (dict), logs (str) - `result`: statistics about resource consumption; `logs`: output from docker logs. """ logger.debug( f"top of execute_actor(); actor_id: {actor_id}; tenant: {tenant} (worker {worker_id};{execution_id})" ) # get any configs for this actor actor_configs = {} config_list = [] # list of all aliases for the actor alias_list = [] # the actor_id passed in is the dbid actor_human_id = Actor.get_display_id(tenant, actor_id) for alias in alias_store.items(): logger.debug(f"checking alias: {alias}") if actor_human_id == alias['actor_id'] and tenant == alias['tenant']: alias_list.append(alias['alias']) logger.debug(f"alias_list: {alias_list}") # loop through configs to look for any that apply to this actor for config in configs_store.items(): # first look for the actor_id itself if actor_human_id in config['actors']: logger.debug(f"actor_id matched; adding config {config}") config_list.append(config) else: logger.debug("actor id did not match; checking aliases...") # if we didn't find the actor_id, look for ay of its aliases for alias in alias_list: if alias in config['actors']: # as soon as we find it, append and get out (only want to add once) logger.debug( f"alias {alias} matched; adding config: {config}") config_list.append(config) break logger.debug(f"got config_list: {config_list}") # for each config, need to check for secrets and decrypt --- for config in config_list: logger.debug('checking for secrets') try: if config['is_secret']: value = encrypt_utils.decrypt(config['value']) actor_configs[config['name']] = value else: actor_configs[config['name']] = config['value'] except Exception as e: logger.error( f'something went wrong checking is_secret for config: {config}; e: {e}' ) logger.debug(f"final actor configs: {actor_configs}") d['_actor_configs'] = actor_configs # initially set the global force_quit variable to False globals.force_quit = False # initial stats object, environment, binds and volumes result = {'cpu': 0, 'io': 0, 'runtime': 0} # instantiate docker client cli = docker.APIClient(base_url=dd, version="auto") # don't try to pass binary messages through the environment as these can cause # broken pipe errors. the binary data will be passed through the FIFO momentarily. if not fifo_host_path: d['MSG'] = msg binds = {} volumes = [] # if container is privileged, mount the docker daemon so that additional # containers can be started. logger.debug("privileged: {};(worker {};{})".format( privileged, worker_id, execution_id)) if privileged: binds = { '/var/run/docker.sock': { 'bind': '/var/run/docker.sock', 'ro': False } } volumes = ['/var/run/docker.sock'] # add a bind key and dictionary as well as a volume for each mount for m in mounts: binds[m.get('host_path')] = { 'bind': m.get('container_path'), 'ro': m.get('format') == 'ro' } volumes.append(m.get('host_path')) # mem_limit # -1 => unlimited memory if mem_limit == '-1': mem_limit = None # max_cpus try: max_cpus = int(max_cpus) except: max_cpus = None # -1 => unlimited cpus if max_cpus == -1: max_cpus = None host_config = cli.create_host_config(binds=binds, privileged=privileged, mem_limit=mem_limit, nano_cpus=max_cpus) logger.debug("host_config object created by (worker {};{}).".format( worker_id, execution_id)) # write binary data to FIFO if it exists: fifo = None if fifo_host_path: try: fifo = os.open(fifo_host_path, os.O_RDWR) os.write(fifo, msg) except Exception as e: logger.error( "Error writing the FIFO. Exception: {};(worker {};{})".format( e, worker_id, execution_id)) os.remove(fifo_host_path) raise DockerStartContainerError("Error writing to fifo: {}; " "(worker {};{})".format( e, worker_id, execution_id)) # set up results socket ----------------------- # make sure socket doesn't already exist: try: os.unlink(socket_host_path) except OSError as e: if os.path.exists(socket_host_path): logger.error( "socket at {} already exists; Exception: {}; (worker {};{})". format(socket_host_path, e, worker_id, execution_id)) raise DockerStartContainerError( "Got an OSError trying to create the results docket; " "exception: {}".format(e)) # use retry logic since, when the compute node is under load, we see errors initially trying to create the socket # server object. keep_trying = True count = 0 server = None while keep_trying and count < 10: keep_trying = False count = count + 1 try: server = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) except Exception as e: keep_trying = True logger.info("Could not instantiate socket at {}. " "Count: {}; Will keep trying. " "Exception: {}; type: {}; (worker {};{})".format( socket_host_path, count, e, type(e), worker_id, execution_id)) try: server.bind(socket_host_path) except Exception as e: keep_trying = True logger.info("Could not bind socket at {}. " "Count: {}; Will keep trying. " "Exception: {}; type: {}; (worker {};{})".format( socket_host_path, count, e, type(e), worker_id, execution_id)) try: os.chmod(socket_host_path, 0o777) logger.debug( "results socket permissions set to 777. socket_host_path: {}". format(socket_host_path)) except Exception as e: msg = f"Got exception trying to set permissions on the results socket. Not sure what to do. e: {e}" logger.error(msg) # for now, we'll just swallow it but this is really a TODO. try: server.settimeout(RESULTS_SOCKET_TIMEOUT) except Exception as e: keep_trying = True logger.info("Could not set timeout for socket at {}. " "Count: {}; Will keep trying. " "Exception: {}; type: {}; (worker {};{})".format( socket_host_path, count, e, type(e), worker_id, execution_id)) if not server: msg = "Failed to instantiate results socket. " \ "Abaco compute host could be overloaded. (worker {};{})".format(worker_id, execution_id) logger.error(msg) raise DockerStartContainerError(msg) logger.debug( "results socket server instantiated. path: {} (worker {};{})".format( socket_host_path, worker_id, execution_id)) # instantiate the results channel: results_ch = ExecutionResultsChannel(actor_id, execution_id) # create and start the container logger.debug("Final container environment: {};(worker {};{})".format( d, worker_id, execution_id)) logger.debug( "Final binds: {} and host_config: {} for the container.(worker {};{})". format(binds, host_config, worker_id, execution_id)) container = cli.create_container(image=image, environment=d, user=user, volumes=volumes, host_config=host_config) # get the UTC time stamp start_time = get_current_utc_time() # start the timer to track total execution time. start = timeit.default_timer() logger.debug("right before cli.start: {}; container id: {}; " "(worker {};{})".format(start, container.get('Id'), worker_id, execution_id)) try: cli.start(container=container.get('Id')) except Exception as e: # if there was an error starting the container, user will need to debug logger.info( "Got exception starting actor container: {}; (worker {};{})". format(e, worker_id, execution_id)) raise DockerStartContainerError( "Could not start container {}. Exception {}".format( container.get('Id'), str(e))) # local bool tracking whether the actor container is still running running = True Execution.update_status(actor_id, execution_id, RUNNING) logger.debug("right before creating stats_cli: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) # create a separate cli for checking stats objects since these should be fast and we don't want to wait stats_cli = docker.APIClient(base_url=dd, timeout=1, version="auto") logger.debug("right after creating stats_cli: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) # under load, we can see UnixHTTPConnectionPool ReadTimeout's trying to create the stats_obj # so here we are trying up to 3 times to create the stats object for a possible total of 3s # timeouts ct = 0 stats_obj = None logs = None while ct < 3: try: stats_obj = stats_cli.stats(container=container.get('Id'), decode=True) break except ReadTimeout: ct += 1 except Exception as e: logger.error( "Unexpected exception creating stats_obj. Exception: {}; (worker {};{})" .format(e, worker_id, execution_id)) # in this case, we need to kill the container since we cannot collect stats; # UPDATE - 07-2018: under load, a errors can occur attempting to create the stats object. # the container could still be running; we need to explicitly check the container status # to be sure. logger.debug( "right after attempting to create stats_obj: {}; (worker {};{})". format(timeit.default_timer(), worker_id, execution_id)) # a counter of the number of iterations through the main "running" loop; # this counter is used to determine when less frequent actions, such as log aggregation, need to run. loop_idx = 0 log_ex = Actor.get_actor_log_ttl(actor_id) while running and not globals.force_quit: loop_idx += 1 logger.debug( "top of while running loop; loop_idx: {}".format(loop_idx)) datagram = None stats = None try: datagram = server.recv(MAX_RESULT_FRAME_SIZE) except socket.timeout: pass except Exception as e: logger.error( "got exception from server.recv: {}; (worker {};{})".format( e, worker_id, execution_id)) logger.debug( "right after try/except datagram block: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) if datagram: try: results_ch.put(datagram) except Exception as e: logger.error( "Error trying to put datagram on results channel. " "Exception: {}; (worker {};{})".format( e, worker_id, execution_id)) logger.debug("right after results ch.put: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) # only try to collect stats if we have a stats_obj: if stats_obj: logger.debug( "we have a stats_obj; trying to collect stats. (worker {};{})". format(worker_id, execution_id)) try: logger.debug( "waiting on a stats obj: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) stats = next(stats_obj) logger.debug("got the stats obj: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) except StopIteration: # we have read the last stats object - no need for processing logger.debug( "Got StopIteration; no stats object. (worker {};{})". format(worker_id, execution_id)) except ReadTimeoutError: # this is a ReadTimeoutError from docker, not requests. container is finished. logger.info( "next(stats) just timed out: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) # UPDATE - 07-2018: under load, a ReadTimeoutError from the attempt to get a stats object # does NOT imply the container has stopped; we need to explicitly check the container status # to be sure. # if we got a stats object, add it to the results; it is possible stats collection timed out and the object # is None if stats: logger.debug("adding stats to results; (worker {};{})".format( worker_id, execution_id)) try: result['cpu'] += stats['cpu_stats']['cpu_usage']['total_usage'] except KeyError as e: logger.info( "Got a KeyError trying to fetch the cpu object: {}; " "(worker {};{})".format(e, worker_id, execution_id)) try: result['io'] += stats['networks']['eth0']['rx_bytes'] except KeyError as e: logger.info( "Got KeyError exception trying to grab the io object. " "running: {}; Exception: {}; (worker {};{})".format( running, e, worker_id, execution_id)) # grab the logs every 5th iteration -- if loop_idx % 5 == 0: logs = cli.logs(container.get('Id')) Execution.set_logs(execution_id, logs, actor_id, tenant, worker_id, log_ex) logs = None # checking the container status to see if it is still running ---- if running: logger.debug( "about to check container status: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) # we need to wait for the container id to be available i = 0 c = None while i < 10: try: c = cli.containers(all=True, filters={'id': container.get('Id')})[0] break except IndexError: logger.error( "Got an IndexError trying to get the container object. " "(worker {};{})".format(worker_id, execution_id)) time.sleep(0.1) i += 1 logger.debug( "done checking status: {}; i: {}; (worker {};{})".format( timeit.default_timer(), i, worker_id, execution_id)) # if we were never able to get the container object, we need to stop processing and kill this # worker; the docker daemon could be under heavy load, but we need to not launch another # actor container with this worker, because the existing container may still be running, if i == 10 or not c: # we'll try to stop the container logger.error( "Never could retrieve the container object! Attempting to stop container; " "container id: {}; (worker {};{})".format( container.get('Id'), worker_id, execution_id)) # stop_container could raise an exception - if so, we let it pass up and have the worker # shut itself down. stop_container(cli, container.get('Id')) logger.info("container {} stopped. (worker {};{})".format( container.get('Id'), worker_id, execution_id)) # if we were able to stop the container, we can set running to False and keep the # worker running running = False continue state = c.get('State') if not state == 'running': logger.debug( "container finished, final state: {}; (worker {};{})". format(state, worker_id, execution_id)) running = False continue else: # container still running; check if a force_quit has been sent OR # we are beyond the max_run_time runtime = timeit.default_timer() - start if globals.force_quit or (max_run_time > 0 and max_run_time < runtime): logs = cli.logs(container.get('Id')) if globals.force_quit: logger.info( "issuing force quit: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) else: logger.info( "hit runtime limit: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) cli.stop(container.get('Id')) running = False logger.debug( "right after checking container state: {}; (worker {};{})". format(timeit.default_timer(), worker_id, execution_id)) logger.info("container stopped:{}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) stop = timeit.default_timer() globals.force_quit = False # get info from container execution, including exit code; Exceptions from any of these commands # should not cause the worker to shutdown or prevent starting subsequent actor containers. exit_code = 'undetermined' try: container_info = cli.inspect_container(container.get('Id')) try: container_state = container_info['State'] try: exit_code = container_state['ExitCode'] except KeyError as e: logger.error("Could not determine ExitCode for container {}. " "Exception: {}; (worker {};{})".format( container.get('Id'), e, worker_id, execution_id)) exit_code = 'undetermined' # Converting ISO8601 times to unix timestamps try: # Slicing to 23 to account for accuracy up to milliseconds and replace to get rid of ISO 8601 'Z' startedat_ISO = container_state['StartedAt'].replace('Z', '')[:23] finishedat_ISO = container_state['FinishedAt'].replace( 'Z', '')[:23] container_state['StartedAt'] = datetime.datetime.strptime( startedat_ISO, "%Y-%m-%dT%H:%M:%S.%f") container_state['FinishedAt'] = datetime.datetime.strptime( finishedat_ISO, "%Y-%m-%dT%H:%M:%S.%f") except Exception as e: logger.error( f"Datetime conversion failed for container {container.get('Id')}. ", f"Exception: {e}; (worker {worker_id};{execution_id})") container_state = {'unavailable': True} except KeyError as e: logger.error( f"Could not determine final state for container {container.get('Id')}. ", f"Exception: {e}; (worker {worker_id};{execution_id})") container_state = {'unavailable': True} except docker.errors.APIError as e: logger.error(f"Could not inspect container {container.get('Id')}. ", f"Exception: {e}; (worker {worker_id};{execution_id})") logger.debug( "right after getting container_info: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) # get logs from container if not logs: logs = cli.logs(container.get('Id')) if not logs: # there are issues where container do not have logs associated with them when they should. logger.info("Container id {} had NO logs associated with it. " "(worker {};{})".format(container.get('Id'), worker_id, execution_id)) logger.debug( "right after getting container logs: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) # get any additional results from the execution: while True: datagram = None try: datagram = server.recv(MAX_RESULT_FRAME_SIZE) except socket.timeout: break except Exception as e: logger.error( "Got exception from server.recv: {}; (worker {};{})".format( e, worker_id, execution_id)) if datagram: try: results_ch.put(datagram) except Exception as e: logger.error( "Error trying to put datagram on results channel. " "Exception: {}; (worker {};{})".format( e, worker_id, execution_id)) logger.debug( "right after getting last execution results from datagram socket: {}; " "(worker {};{})".format(timeit.default_timer(), worker_id, execution_id)) if socket_host_path: server.close() os.remove(socket_host_path) logger.debug("right after removing socket: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) # remove actor container with retrying logic -- check for specific filesystem errors from the docker daemon: if not leave_container: keep_trying = True count = 0 while keep_trying and count < 10: keep_trying = False count = count + 1 try: cli.remove_container(container=container) logger.info("Actor container removed. (worker {};{})".format( worker_id, execution_id)) except Exception as e: # if the container is already gone we definitely want to quit: if 'No such container' in str(e): logger.info("Got 'no such container' exception - quiting. " "Exception: {}; (worker {};{})".format( e, worker_id, execution_id)) break # if we get a resource busy/internal server error from docker, we need to keep trying to remove the # container. elif 'device or resource busy' in str( e) or 'failed to remove root filesystem' in str(e): logger.error( "Got resource busy/failed to remove filesystem exception trying to remove " "actor container; will keep trying." "Count: {}; Exception: {}; (worker {};{})".format( count, e, worker_id, execution_id)) time.sleep(1) keep_trying = True else: logger.error( "Unexpected exception trying to remove actor container. Giving up." "Exception: {}; type: {}; (worker {};{})".format( e, type(e), worker_id, execution_id)) else: logger.debug("leaving actor container since leave_container was True. " "(worker {};{})".format(worker_id, execution_id)) logger.debug( "right after removing actor container: {}; (worker {};{})".format( timeit.default_timer(), worker_id, execution_id)) if fifo_host_path: try: os.close(fifo) os.remove(fifo_host_path) except Exception as e: logger.debug( f"got Exception trying to clean up fifo_host_path; e: {e}") if results_ch: # check if the length of the results channel is empty and if so, delete it -- if len(results_ch._queue._queue) == 0: try: results_ch.delete() except Exception as e: logger.warn( f"Got exception trying to delete the results_ch, swallowing it; Exception: {e}" ) else: try: results_ch.close() except Exception as e: logger.warn( f"Got exception trying to close the results_ch, swallowing it; Exception: {e}" ) result['runtime'] = int(stop - start) logger.debug( "right after removing fifo; about to return: {}; (worker {};{})". format(timeit.default_timer(), worker_id, execution_id)) return result, logs, container_state, exit_code, start_time