def function_delete(function_id): """Delete the container image which contains the function :param function_id: :return: """ try: f = models.Function.get(id=function_id) except NotFoundError: return not_found() utils.delete_image(f.image_id, driver_endpoint) utils.delete_function(function_id) return Response(status=204)
def image_name_for_tests(docker_client, tmpdir, request): if tarantool_enterprise_is_used(): docker_client.images.pull('centos', '8') return 'centos:8' build_path = os.path.join(tmpdir, 'build_image') os.makedirs(build_path) test_image_dockerfile_path = os.path.join(build_path, 'Dockerfile') with open(test_image_dockerfile_path, 'w') as f: f.write(''' FROM centos:8 RUN curl -s \ https://packagecloud.io/install/repositories/tarantool/{}/script.rpm.sh | bash \ && yum -y install tarantool tarantool-devel '''.format(tarantool_repo_version())) IMAGE_NAME = 'test-image' docker_client.images.build( path=build_path, forcerm=True, tag=IMAGE_NAME, ) request.addfinalizer(lambda: delete_image(docker_client, IMAGE_NAME)) return IMAGE_NAME
def instance_container_with_unpacked_tgz(docker_client, tmpdir, tgz_archive_with_cartridge, request): project = tgz_archive_with_cartridge.project # build image with installed Tarantool build_path = os.path.join(tmpdir, 'build_image') os.makedirs(build_path) dockerfile_layers = ["FROM centos:7"] if not tarantool_enterprise_is_used(): tarantool_installer = get_tarantool_installer_cmd("yum") dockerfile_layers.append(f"RUN {tarantool_installer}") with open(os.path.join(build_path, 'Dockerfile'), 'w') as f: f.write('\n'.join(dockerfile_layers)) image_name = '%s-test-tgz' % project.name build_image(build_path, image_name) request.addfinalizer(lambda: delete_image(docker_client, image_name)) # create container instance_name = 'instance-1' http_port = '8183' advertise_port = '3302' environment = [ 'TARANTOOL_APP_NAME=%s' % project.name, 'TARANTOOL_INSTANCE_NAME=%s' % instance_name, 'TARANTOOL_ADVERTISE_URI=%s' % advertise_port, 'TARANTOOL_HTTP_PORT=%s' % http_port, ] container_proj_path = os.path.join('/opt', project.name) init_script_path = os.path.join(container_proj_path, 'init.lua') tarantool_executable = \ os.path.join(container_proj_path, 'tarantool') \ if tarantool_enterprise_is_used() \ else 'tarantool' cmd = [tarantool_executable, init_script_path] container = docker_client.containers.create( image_name, cmd, environment=environment, ports={http_port: http_port}, name='{}-{}'.format(project.name, instance_name), detach=True, ) with gzip.open(tgz_archive_with_cartridge.filepath, 'rb') as f: container.put_archive('/opt', f.read()) request.addfinalizer(lambda: container.remove(force=True)) return InstanceContainer(container=container, instance_name=instance_name, http_port=http_port, advertise_port=advertise_port)
def container_with_installed_deb(docker_client, deb_archive_with_cartridge, request, tmpdir): project = deb_archive_with_cartridge.project # build image with installed DEB build_path = os.path.join(tmpdir, 'build_image') os.makedirs(build_path) shutil.copy(deb_archive_with_cartridge.filepath, build_path) dockerfile_layers = ["FROM jrei/systemd-ubuntu"] if not tarantool_enterprise_is_used(): dockerfile_layers.append( '''RUN apt-get update && apt-get install -y curl \ && DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata \ && curl -s \ https://packagecloud.io/install/repositories/tarantool/{}/script.deb.sh | bash '''.format(tarantool_repo_version())) dockerfile_layers.append(''' COPY {deb_filename} /opt RUN apt-get install -y /opt/{deb_filename} '''.format( deb_filename=os.path.basename(deb_archive_with_cartridge.filepath))) with open(os.path.join(build_path, 'Dockerfile'), 'w') as f: f.write('\n'.join(dockerfile_layers)) image_name = '%s-test-deb' % project.name build_image(build_path, image_name) request.addfinalizer(lambda: delete_image(docker_client, image_name)) # create container http_port = '8183' container = docker_client.containers.create( image_name, command='/lib/systemd/systemd', ports={http_port: http_port}, name='%s-test-deb' % project.name, detach=True, privileged=True, volumes=['/sys/fs/cgroup:/sys/fs/cgroup:ro'], ) request.addfinalizer(lambda: container.remove(force=True)) return ProjectContainer(project=project, container=container, http_port=http_port)
def main(): previous_time = '' previous_progress_of_the_day = '' with TelegramClient(config.session_name, config.api_id, config.api_hash) as client: while True: if not previous_time == get_current_time(): current_time = get_current_time() previous_time = current_time generate_image(current_time) image = client.upload_file(config.image_filename) client(UploadProfilePhotoRequest(image)) client( DeletePhotosRequest([client.get_profile_photos('me')[-1]])) delete_image() time.sleep(1) if not previous_progress_of_the_day == get_progress_of_the_day(): current_progress_of_the_day = get_progress_of_the_day() previous_progress_of_the_day = current_progress_of_the_day profile_bio = config.profile_bio.format( current_progress_of_the_day) client(UpdateProfileRequest(about=profile_bio))
def docker_image(tmpdir, light_project, request, docker_client): project = light_project cmd = [os.path.join(basepath, "cartridge"), "pack", "docker", project.path] process = subprocess.run(cmd, cwd=tmpdir) assert process.returncode == 0, \ "Error during creating of docker image" image_name = find_image(docker_client, project.name) assert image_name is not None, "Docker image isn't found" request.addfinalizer(lambda: delete_image(docker_client, image_name)) image = Image(image_name, project) return image
def docker_image_with_cartridge(cartridge_cmd, tmpdir, project_with_cartridge, request, docker_client): project = project_with_cartridge cmd = [cartridge_cmd, "pack", "docker", project.path] process = subprocess.run(cmd, cwd=tmpdir) assert process.returncode == 0, \ "Error during creating of docker image" image_name = find_image(docker_client, project.name) assert image_name is not None, "Docker image isn't found" request.addfinalizer(lambda: delete_image(docker_client, image_name)) image = Image(image_name, project) return image
def container_with_installed_rpm(docker_client, rpm_archive_with_cartridge, request, tmpdir): project = rpm_archive_with_cartridge.project # build image with installed RPM build_path = os.path.join(tmpdir, 'build_image') os.makedirs(build_path) shutil.copy(rpm_archive_with_cartridge.filepath, build_path) dockerfile_layers = ["FROM centos:7"] if not tarantool_enterprise_is_used(): installer_cmd = get_tarantool_installer_cmd("yum") dockerfile_layers.append(f"RUN {installer_cmd}") else: dockerfile_layers.append("RUN yum update -y") dockerfile_layers.append(''' COPY {rpm_filename} /opt RUN yum install -y /opt/{rpm_filename} '''.format( rpm_filename=os.path.basename(rpm_archive_with_cartridge.filepath))) with open(os.path.join(build_path, 'Dockerfile'), 'w') as f: f.write('\n'.join(dockerfile_layers)) image_name = '%s-test-rpm' % project.name build_image(build_path, image_name) request.addfinalizer(lambda: delete_image(docker_client, image_name)) # create container http_port = '8183' container = docker_client.containers.create( image_name, command='/sbin/init', ports={http_port: http_port}, name='%s-test-rpm' % project.name, detach=True, privileged=True, volumes=['/sys/fs/cgroup:/sys/fs/cgroup:ro'], ) request.addfinalizer(lambda: container.remove(force=True)) return ProjectContainer(project=project, container=container, http_port=http_port)
def docker_image(cartridge_cmd, session_tmpdir, session_light_project, request, docker_client): project = session_light_project add_runtime_requirements_file(project) cmd = [cartridge_cmd, "pack", "docker", project.path] process = subprocess.run(cmd, cwd=session_tmpdir) assert process.returncode == 0, \ "Error during creating of docker image" image_name = find_image(docker_client, project.name) assert image_name is not None, "Docker image isn't found" request.addfinalizer(lambda: delete_image(docker_client, image_name)) image = Image(image_name, project) return image
def docker_image_print_environment(cartridge_cmd, tmpdir, project_without_dependencies, request, docker_client): project = project_without_dependencies replace_project_file(project, 'init.lua', INIT_PRINT_ENV_FILEPATH) cmd = [ cartridge_cmd, "pack", "docker", "--tag", project.name, project.path, ] process = subprocess.run(cmd, cwd=tmpdir) assert process.returncode == 0, \ "Error during creating of docker image" image_name = find_image(docker_client, project.name) assert image_name is not None, "Docker image isn't found" request.addfinalizer(lambda: delete_image(docker_client, image_name)) image = Image(image_name, project) return image
def run(self): self.slotsStartWatch.start() self.slotsEndWatch.start() while not self.stopped_thread(): self.check_timeouts() new_timeout = self.get_next_timeout() readable, writable, exceptional = select.select( self.inputs, [], self.inputs, new_timeout) for s in readable: if s is self.sock_tcp: (connection, address) = self.sock_tcp.accept() connection.setblocking(False) self.logger.info('TCP CON (%s, %d)' % (address[0], address[1])) self.inputs.append(connection) else: buffer = utils.read_data_from_socket(s) if buffer: self.logger.debug( 'Req_data = %s\t client = (%s, %d)' % (buffer, s.getpeername()[0], s.getpeername()[1])) action, data = utils.segment_packet(buffer) if action == utils.GATEWAY_NODES_FLASH: image_name = self.get_image_name(s) result = [] for node in data[db_utils.NODES]: if node['status'] == utils.FLASHED: gateway_id = self.dbConnector.find_gateway_by_addr( s.getpeername()) node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id( gateway_id, node['node_id']) self.dbConnector.node_update_flash_info( node_uid, 'FINISHED', image_name) _node = { '_id': node_uid, 'status': node['status'] } else: # node['status'] = utils.ERROR gateway_id = self.dbConnector.find_gateway_by_addr( s.getpeername()) node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id( gateway_id, node['node_id']) _node = { '_id': node_uid, 'status': node['status'] } result.append(_node) self.handle_gateway_request(s, result) self.inputs.remove(s) s.close() elif action == utils.GATEWAY_NODES_ERASE: result = [] for node in data[db_utils.NODES]: if node['status'] == utils.ERASED: gateway_id = self.dbConnector.find_gateway_by_addr( s.getpeername()) node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id( gateway_id, node['node_id']) self.dbConnector.node_update_flash_info( node_uid, 'NOT_STARTED', None) _node = { '_id': node_uid, 'status': node['status'] } else: # node['status'] = utils.ERROR gateway_id = self.dbConnector.find_gateway_by_addr( s.getpeername()) node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id( gateway_id, node['node_id']) _node = { '_id': node_uid, 'status': node['status'] } result.append(_node) self.handle_gateway_request(s, result) self.inputs.remove(s) s.close() elif action == utils.GATEWAY_NODES_RESET: result = [] for node in data[db_utils.NODES]: gateway_id = self.dbConnector.find_gateway_by_addr( s.getpeername()) node_uid = self.dbConnector.get_node_uid_by_gateway_id_and_node_id( gateway_id, node['node_id']) _node = { '_id': node_uid, 'status': node['status'] } result.append(_node) self.handle_gateway_request(s, result) self.inputs.remove(s) s.close() elif action == utils.IMAGES_GET: # { token } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: nodetypes_images = utils.get_nodetypes_images( decoded_token[db_utils.USER]) pck = utils.create_response_packet( json.dumps({ 'data': nodetypes_images, 'status': 200 }).encode()) # OnSuccess: { data, status: 200 } s.sendall(pck) elif action == utils.IMAGE_SAVE: # { token, image_name, image_data, nodetype_id } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: utils.save_image(decoded_token[db_utils.USER], data[db_utils.NODETYPE_ID], data[db_utils.IMAGE_NAME], data[db_utils.IMAGE_DATA]) pck = utils.create_response_packet( json.dumps({ 'status': 200 }).encode()) # OnSuccess: { status: 200 } s.sendall(pck) elif action == utils.IMAGE_DELETE: # { token, image_name } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: nodetype_id = utils.get_nodetype_by_user_and_image_name( decoded_token[db_utils.USER], data[db_utils.IMAGE_NAME]) res = utils.delete_image( decoded_token[db_utils.USER], data[db_utils.IMAGE_NAME], nodetype_id) pck = utils.create_response_packet( json.dumps(res).encode()) # OnSuccess: { status: 204 } # OnError : { status: 404 } s.sendall(pck) elif action == utils.NODES_GET: # { token } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: nodes = self.dbConnector.get_nodes() pck = utils.create_response_packet( json.dumps({ 'nodes': nodes, 'status': 200 }).encode()) # OnSuccess: { nodes, status: 200 } s.sendall(pck) elif action == utils.NODES_FLASH: # { token, slot_id, image_name, node_uids} token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: gateway_socks = self.send_flash_request( decoded_token[db_utils.USER], data[db_utils.IMAGE_NAME], data[db_utils.NODE_UIDS]) self.inputs.extend(gateway_socks) self.gateway_sockets.append(gateway_socks) self.gateway_sockets_info.append({ 'web_socket': s, 'data': [], db_utils.IMAGE_NAME: data[db_utils.IMAGE_NAME] }) elif action == utils.NODES_ERASE: # { token, slot_id, node_uids } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: gateway_socks = self.send_erase_request( data[db_utils.NODE_UIDS]) self.inputs.extend(gateway_socks) self.gateway_sockets.append(gateway_socks) self.gateway_sockets_info.append({ 'web_socket': s, 'data': [], db_utils.IMAGE_NAME: '' }) elif action == utils.NODES_RESET: # { token, slot_id, node_uids } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: gateway_socks = self.send_reset_request( data[db_utils.NODE_UIDS]) self.inputs.extend(gateway_socks) self.gateway_sockets.append(gateway_socks) self.gateway_sockets_info.append({ 'web_socket': s, 'data': [], db_utils.IMAGE_NAME: '' }) elif action == utils.TIMESLOTS_SAVE: # { token, slots: [{start, end}] } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: slots = db_utils.convert_isoformat_to_datetime( data[db_utils.SLOTS]) slots_saved = self.dbConnector.save_timeslots( decoded_token[db_utils.USER], slots) slots = db_utils.convert_datetime_to_isoformat( slots_saved) pck = utils.create_response_packet( json.dumps({ 'slots': slots, 'status': 200 }).encode()) # OnSuccess: { slots: [{start, end}], status: 200 } s.sendall(pck) elif action == utils.TIMESLOTS_GET_DAYSLOTS: # { token, date } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: slots_day = self.dbConnector.get_day_slots( data[db_utils.DATE]) slots = db_utils.convert_datetime_to_isoformat( slots_day) pck = utils.create_response_packet( json.dumps({ 'slots': slots, 'status': 200 }).encode()) # OnSuccess: { slots: [{start, end, user_id}], status: 200 } s.sendall(pck) elif action == utils.TIMESLOTS_GET_USERSLOTS: # { token } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: user_slots = self.dbConnector.get_user_slots( decoded_token[db_utils.USER]) slots = db_utils.convert_datetime_to_isoformat( user_slots) pck = utils.create_response_packet( json.dumps({ 'slots': slots, 'status': 200 }).encode()) # OnSuccess: { slots: [{slot_id, start, end}], status: 200 } s.sendall(pck) elif action == utils.NODETYPES_GET: # { token } token = data[db_utils.TOKEN] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) else: nodetypes = self.dbConnector.get_nodetypes() pck = utils.create_response_packet( json.dumps({ 'nodetypes': nodetypes, 'status': 200 }).encode()) # OnSuccess: { nodetypes, status: 201 } s.sendall(pck) elif action == utils.USERS_SIGNUP: # { email, username, password } res = self.dbConnector.create_user(data) pck = utils.create_response_packet( json.dumps(res).encode()) # OnSuccess: { status: 201 } # OnError : { message, status: 403 } s.sendall(pck) elif action == utils.USERS_LOGIN: # { email, username } res = self.dbConnector.login_user(data) pck = utils.create_response_packet( json.dumps(res).encode()) # OnSuccess: { token, status: 200} # OnError : { message, status: 401 } s.sendall(pck) elif action == utils.DEBUG_START: # { token, slot_id } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: if self.sock_debug: utils.remove_from_list( self.inputs, self.sock_debug) self.sock_debug.close() log_data = [ '=== DEBUG CHANNEL START ===\n===========================\n' ] pck = utils.create_response_packet( json.dumps({ 'data': log_data }).encode()) self.sock_debug = s self.sock_debug.sendall(pck) elif action == utils.DEBUG_END: # { token, slot_id } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: log_data = [ '=== DEBUG CHANNEL END ===\n=========================\n' ] if self.sock_debug: self.experiment_info = {} pck = utils.create_response_packet( json.dumps({ 'data': log_data, 'message': 'STOP DEBUG' }).encode()) self.sock_debug.sendall(pck) utils.remove_from_list( self.inputs, self.sock_debug) self.sock_debug.close() self.sock_debug = None # { status: 204 } pck = utils.create_response_packet( json.dumps({ 'status': 204 }).encode()) s.sendall(pck) elif action == utils.DEBUG_CLEAR_LOG: # { token, slot_id } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: self.clear_debug_log( decoded_token[db_utils.USER], slot_id) pck = utils.create_response_packet( json.dumps({ 'status': 204 }).encode()) # OnSuccess: { status: 204 } s.sendall(pck) elif action == utils.DEBUG_GET_LOG: # { token, slot_id } token = data[db_utils.TOKEN] slot_id = data[db_utils.TIMESLOT_ID] decoded_token = utils.decode_auth_token(token) if not decoded_token: utils.send_invalid_token_error(s) elif not self.dbConnector.get_slot_by_id(slot_id): utils.send_invalid_slot_error(s) else: self.send_debug_log( s, decoded_token[db_utils.USER], slot_id) elif action == utils.DEBUG_GATEWAY: # [ TIMESTAMP, NODE_ID, DATA ] print(data[0], '|', data[1], '|', data[2]) if self.experiment_info: self.write_debug_log(data) if self.sock_debug: pck = utils.create_response_packet( json.dumps({ 'data': data }).encode()) self.sock_debug.sendall(pck) else: self.logger.info( 'TCP DISCON (%s, %d)' % (s.getpeername()[0], s.getpeername()[1])) self.inputs.remove(s) s.close() for s in exceptional: self.inputs.remove(s) s.close() for sock in self.inputs: self.logger.debug('Exiting. . . Closing [%s]' % sock) sock.close() self.log_timer.cancel() self.sock_tcp.close() sys.exit('Exiting. . .')
def main(): global image, boxes, points # getting arguments from cli args = utils.cli_arguments() out = utils.open_file(args) img_dir = args["dir"] auto_landm = args["land"] is not None auto_faces = args["auto"] mirror_points = args["mirror"] if args["train"] is True: # skip everything and train the model return utils.train_model(out.path, args["train"]) # cnn face detector utils.init_face_detector(args) # load shape predictor if requested if auto_faces and auto_landm: global predictor predictor = dlib.shape_predictor(args["land"]) # recover last state (if append is true) resume, lastpath = utils.load_state(args["append"]) for file in os.listdir(img_dir): # consider only images if not is_image(file): continue # avoid mirrored images if is_mirrored(file): continue # load image: path = os.path.join(img_dir, file) # trying to resume from the image located at lastpath if resume: if path == lastpath: resume = False else: continue image = cv2.imread(path) # clear: stack, boxes and points stack.clear() boxes.clear() points.clear() # automatically detect faces if auto_faces: draw_face(image) # and landmarks if auto_landm: detect_landmarks(image, boxes[-1]) stack.append(image) # create a window with disabled menu when right-clicking with the mouse window = file cv2.namedWindow(window, cv2.WINDOW_AUTOSIZE | cv2.WINDOW_GUI_NORMAL) # mouse callback to window cv2.setMouseCallback(window, mouse_callback) # removing or skipping the current image without affecting output file skipped = False removed = False # showing image until esc is pressed while (1): cv2.imshow(window, image) key = cv2.waitKey(20) & 0Xff # listen to key events if key == KEY_ESC: break elif key == KEY_S: skipped = True break elif key == KEY_R: removed = True utils.delete_image(path) break elif key == KEY_Q: return quit(path, out) if not (skipped or removed): # clear point log print() # write annotations add_entry(out, path, boxes, points, mirror_points) # close window cv2.destroyAllWindows() # delete checkpoint file and close output file utils.delete_state() out.close()