def __init__(self, sconf, logger, app, version, account, container, obj): self.logger = logger self.app = app self.version = version self.account = account self.container = container self.obj = obj self.sconf = sconf self.storlet_metadata = None self.storlet_timeout = int(self.sconf['storlet_timeout']) self.paths = RunTimePaths(account, sconf)
def __init__(self, sconf, logger, app, version, account, container, obj): self.logger = logger self.app = app self.version = version self.account = account self.container = container self.obj = obj self.sconf = sconf self.storlet_metadata = None self.storlet_timeout = int(self.sconf["storlet_timeout"]) self.paths = RunTimePaths(account, sconf)
def __init__(self, sconf, logger, app, version, account, container, obj): self.logger = logger # TODO(eranr): Add sconf defaults, and get rid of validate_conf below self.app = app self.version = version self.account = account self.container = container self.obj = obj self.sconf = sconf self.storlet_timeout = int(self.sconf['storlet_timeout']) self.paths = RunTimePaths(account, sconf)
class StorletGatewayDocker(StorletGatewayBase): def __init__(self, sconf, logger, app, version, account, container, obj): self.logger = logger self.app = app self.version = version self.account = account self.container = container self.obj = obj self.sconf = sconf self.storlet_metadata = None self.storlet_timeout = int(self.sconf['storlet_timeout']) self.paths = RunTimePaths(account, sconf) class IterLike(object): def __init__(self, obj_data, timeout, cancel_func): self.closed = False self.obj_data = obj_data self.timeout = timeout self.cancel_func = cancel_func def __iter__(self): return self def read_with_timeout(self, size): timeout = Timeout(self.timeout) try: chunk = os.read(self.obj_data, size) except Timeout as t: if t is timeout: if self.cancel_func: self.cancel_func() self.close() raise t except Exception as e: self.close() raise e finally: timeout.cancel() return chunk def next(self, size=1024): chunk = None r, w, e = select.select([self.obj_data], [], [], self.timeout) if len(r) == 0: self.close() if self.obj_data in r: chunk = self.read_with_timeout(size) if chunk == '': raise StopIteration('Stopped iterator ex') else: return chunk raise StopIteration('Stopped iterator ex') def read(self, size=1024): return self.next(size) def readline(self, size=-1): return '' def readlines(self, sizehint=-1): pass def close(self): if self.closed == True: return self.closed = True os.close(self.obj_data) def __del__(self): self.close() def validateStorletUpload(self, req): if (self.container == self.sconf['storlet_container']): if (self.obj.find('-') < 0 or self.obj.find('.') < 0): return 'Storlet name is incorrect' ret = self._validate_mandatory_headers(req) if ret: return ret return False def authorizeStorletExecution(self, req): res, headers = self.verify_access(req.environ, self.version, self.account, self.sconf['storlet_container'], req.headers['X-Run-Storlet']) if not res: return False # keep the storlets headers for later use. self.storlet_metadata = headers return True def augmentStorletRequest(self, req): if self.storlet_metadata: self._fix_request_headers(req) def gatewayProxyPutFlow(self, orig_req, container, obj): sreq = StorletPUTRequest(self.account, orig_req) req = sreq._getInitialRequest() self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) # Clean all Storlet stuff from the request headers # we do not need them anymore, and they # may interfere with the rest of the execution. self._clean_storlet_stuff_from_request(req.headers) req.headers.pop('X-Run-Storlet') slog_path = self.\ paths.slog_path(self.idata['storlet_main_class']) storlet_pipe_path = self.\ paths.host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationPUTProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() self._set_metadata_in_headers(req.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, self.storlet_timeout, sprotocol._cancel) def gatewayProxyGETFlow(self, req, container, obj, orig_resp): # Flow for running the GET computation on the proxy sreq = StorletSLORequest(self.account, orig_resp, req.params) self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) slog_path = self.\ paths.slog_path(self.idata['storlet_main_class']) storlet_pipe_path = self.\ paths.host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationSLOProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() self._set_metadata_in_headers(orig_resp.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, self.storlet_timeout, sprotocol._cancel) def gatewayObjectGetFlow(self, req, container, obj, orig_resp): sreq = StorletGETRequest(self.account, orig_resp, req.params) self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) slog_path = self.\ paths.slog_path(self.idata['storlet_main_class']) storlet_pipe_path = self.paths.\ host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationGETProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() orig_resp = sreq._getInitialRequest() self._set_metadata_in_headers(orig_resp.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, self.storlet_timeout, sprotocol._cancel) def verify_access(self, env, version, account, container, object): self.logger.info('Verify access to {0}/{1}/{2}'.format( account, container, object)) new_env = dict(env) if 'HTTP_TRANSFER_ENCODING' in new_env.keys(): del new_env['HTTP_TRANSFER_ENCODING'] new_env['REQUEST_METHOD'] = 'HEAD' new_env['swift.source'] = 'SE' new_env['PATH_INFO'] = os.path.join('/' + version, account, container, object) new_env['RAW_PATH_INFO'] = os.path.join('/' + version, account, container, object) storlet_req = Request.blank(new_env['PATH_INFO'], new_env) resp = storlet_req.get_response(self.app) if resp.status_int < 300 and resp.status_int >= 200: return True, resp.headers return False, [] def _validate_mandatory_headers(self, req): mandatory_md = None if self.container in [self.sconf['storlet_container']]: self.logger.info('PUT method for storlet dependency. Sanity check') mandatory_md = [ 'X-Object-Meta-Storlet-Language', 'X-Object-Meta-Storlet-Interface-Version', 'X-Object-Meta-Storlet-Dependency', 'X-Object-Meta-Storlet-Object-Metadata', 'X-Object-Meta-Storlet-Main' ] elif self.container in [self.sconf['storlet_dependency']]: self.logger.info('PUT method for storlet container. Sanity check') mandatory_md = ['X-Object-Meta-Storlet-Dependency-Version'] if mandatory_md is not None: for md in mandatory_md: if md not in req.headers: self.logger.info('Mandatory header ' + 'is missing: {0}'.format(md)) return 'Mandatory header is missing: {0}'.format(md) return None def _fix_request_headers(self, req): # add to request the storlet metadata to be used in case the request # is forwarded to the data node (GET case) for key, val in self.storlet_metadata.iteritems(): if key.startswith('X-Object-Meta-Storlet'): req.headers[key] = val elif key in ['X-Timestamp', 'Content-Length']: req.headers['X-Storlet-' + key] = val def _add_system_params(self, params): ''' Adds Storlet engine specific parameters to the invocation currently, this consists only of the execution path of the Storlet within the Docker container. ''' params['storlet_execution_path'] = self.\ paths.sbox_storlet_exec(self.idata['storlet_main_class']) def _clean_storlet_stuff_from_request(self, headers): for key in headers: if (key.startswith('X-Storlet') or key.startswith('X-Object-Meta-Storlet')): del headers[key] return headers def _get_storlet_invocation_data(self, req): data = dict() data['storlet_name'] = req.headers.get('X-Run-Storlet') data['generate_log'] = req.headers.get('X-Storlet-Generate-Log', False) data['storlet_original_timestamp'] = req.headers.\ get('X-Storlet-X-Timestamp') data['storlet_original_size'] = req.headers.\ get('X-Storlet-Content-Length') data['storlet_md'] = { 'storlet_original_timestamp': data['storlet_original_timestamp'], 'storlet_original_size': data['storlet_original_size'] } data['storlet_main_class'] = req.headers.\ get('X-Object-Meta-Storlet-Main') scope = self.account data['scope'] = scope if data['scope'].rfind(':') > 0: data['scope'] = data['scope'][:data['scope'].rfind(':')] data['storlet_dependency'] = req.headers.\ get('X-Object-Meta-Storlet-Dependency') data['request_params'] = req.params return data def _set_metadata_in_headers(self, headers, md): if md: for key, val in md.iteritems(): headers['X-Object-Meta-%s' % key] = val def _upload_storlet_logs(self, slog_path): if (config_true_value(self.idata['generate_log'])): logfile = open(slog_path, 'r') client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1) try: headers = dict() headers['CONTENT_TYPE'] = 'text/html' log_obj_name = '%s.log' %\ self.idata['storlet_name'][:self.idata['storlet_name']. find('-')] client.upload_object(logfile, self.account, self.sconf['storlet_logcontainer'], log_obj_name, headers) except Exception as e: raise e def bring_from_cache(self, obj_name, is_storlet): ''' Auxiliary function that: (1) Brings from Swift obj_name, whether this is a storlet or a storlet dependency. (2) Copies from local cache into the Docker conrainer If this is a Storlet then also validates that the cache is updated with most recent copy of the Storlet compared to the copy residing in Swift. Retunrs wheather the Docker container was updated with obj_name ''' # Determine the cache we are to work with # e.g. dependency or storlet if not is_storlet: cache_dir = self.paths.get_host_dependency_cache_dir() swift_source_container = self.paths.storlet_dependency else: cache_dir = self.paths.get_host_storlet_cache_dir() swift_source_container = self.paths.storlet_container if not os.path.exists(cache_dir): os.makedirs(cache_dir, 0755) # cache_target_path is the actual object we need to deal with # e.g. a concrete storlet or dependency we need to bring/update cache_target_path = os.path.join(cache_dir, obj_name) # Determine if we need to update the cache for cache_target_path # We default for no update_cache = False # If it does not exist in cache, we obviously need to bring if not os.path.isfile(cache_target_path): update_cache = True elif is_storlet: # The cache_target_path exists, we test if it is up-to-date # with the metadata we got. # We mention that this is currenlty applicable for storlets # only, and not for dependencies. # This will change when we will head dependencies as well storlet_md = self.idata['storlet_md'] fstat = os.stat(cache_target_path) storlet_or_size = long(storlet_md['storlet_original_size']) storlet_or_time = float(storlet_md['storlet_original_timestamp']) b_storlet_size_changed = fstat.st_size != storlet_or_size b_storlet_file_updated = float(fstat.st_mtime) < storlet_or_time if b_storlet_size_changed or b_storlet_file_updated: update_cache = True expected_perm = '' if update_cache: # If the cache needs to be updated, then get on with it # bring the object from Swift using ic client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1) path = client.make_path(self.account, swift_source_container, obj_name) self.logger.debug('Invoking ic on path %s' % path) resp = client.make_request('GET', path, {'PATH_INFO': path}, [200]) fn = open(cache_target_path, 'w') fn.write(resp.body) fn.close() if not is_storlet: expected_perm = resp.headers.\ get('X-Object-Meta-Storlet-Dependency-Permissions', '') if expected_perm != '': os.chmod(cache_target_path, int(expected_perm, 8)) # The node's local cache is now updated. # We now verify if we need to update the # Docker container itself. # The Docker container needs to be updated if: # 1. The Docker container does not hold a copy of the object # 2. The Docker container holds an older version of the object update_docker = False docker_storlet_path = self.paths.\ host_storlet(self.idata['storlet_main_class']) docker_target_path = os.path.join(docker_storlet_path, obj_name) if not os.path.exists(docker_storlet_path): os.makedirs(docker_storlet_path, 0755) update_docker = True elif not os.path.isfile(docker_target_path): update_docker = True else: fstat_cached_object = os.stat(cache_target_path) fstat_docker_object = os.stat(docker_target_path) b_size_changed = fstat_cached_object.st_size \ != fstat_docker_object.st_size b_time_changed = float(fstat_cached_object.st_mtime) <\ float(fstat_docker_object.st_mtime) if (b_size_changed or b_time_changed): update_docker = True if update_docker: # need to copy from cache to docker # copy2 also copies the permissions shutil.copy2(cache_target_path, docker_target_path) return update_docker def update_docker_container_from_cache(self): ''' Iterates over the storlet name and its dependencies appearing in the invocation data and make sure they are brought to the local cache, and from there to the Docker container. Uses the bring_from_cache auxiliary function. Returns True if the Docker container was updated ''' # where at the host side, reside the storlet containers storlet_path = self.paths.host_storlet_prefix() if not os.path.exists(storlet_path): os.makedirs(storlet_path, 0755) # Iterate over storlet and dependencies, and make sure # they are updated within the Docker container. # return True if any of them wea actually # updated within the Docker container docker_updated = False updated = self.bring_from_cache(self.idata['storlet_name'], True) docker_updated = docker_updated or updated if self.idata['storlet_dependency']: for dep in self.idata['storlet_dependency'].split(','): updated = self.bring_from_cache(dep, False) docker_updated = docker_updated or updated return docker_updated
class StorletGatewayDocker(StorletGatewayBase): def __init__(self, sconf, logger, app, version, account, container, obj): self.logger = logger self.app = app self.version = version self.account = account self.container = container self.obj = obj self.sconf = sconf self.storlet_metadata = None self.storlet_timeout = int(self.sconf['storlet_timeout']) self.paths = RunTimePaths(account, sconf) def validateStorletUpload(self, req): if (self.container == self.sconf['storlet_container']): if (self.obj.find('-') < 0 or self.obj.find('.') < 0): return 'Storlet name is incorrect' ret = self._validate_mandatory_headers(req) if ret: return ret return False def authorizeStorletExecution(self, req): res, headers = self.verify_access(req.environ, self.version, self.account, self.sconf['storlet_container'], req.headers['X-Run-Storlet']) if not res: return False # keep the storlets headers for later use. self.storlet_metadata = headers return True def augmentStorletRequest(self, req): if self.storlet_metadata: self._fix_request_headers(req) def gatewayProxyPutFlow(self, orig_req, container, obj): sreq = StorletPUTRequest(self.account, orig_req) req = sreq._getInitialRequest() self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) # Clean all Storlet stuff from the request headers # we do not need them anymore, and they # may interfere with the rest of the execution. self._clean_storlet_stuff_from_request(req.headers) req.headers.pop('X-Run-Storlet') slog_path = self.\ paths.slog_path(self.idata['storlet_main_class']) storlet_pipe_path = self.\ paths.host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationPUTProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() self._set_metadata_in_headers(req.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, self.data_read_fd def gatewayProxySloFlow(self, req, container, obj, orig_resp): # Adapted from PUT flow to SLO sreq = StorletSLORequest(self.account, orig_resp, req.params) self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) slog_path = self.\ paths.slog_path(self.idata['storlet_main_class']) storlet_pipe_path = self.\ paths.host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationSLOProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() self._set_metadata_in_headers(req.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, self.data_read_fd def gatewayObjectGetFlow(self, req, container, obj, orig_resp): sreq = StorletGETRequest(self.account, orig_resp, req.params) self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) slog_path = self.\ paths.slog_path(self.idata['storlet_main_class']) storlet_pipe_path = self.paths.\ host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationGETProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() orig_resp = sreq._getInitialRequest() self._set_metadata_in_headers(orig_resp.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, self.data_read_fd def verify_access(self, env, version, account, container, object): self.logger.info('Verify access to {0}/{1}/{2}'.format(account, container, object)) new_env = dict(env) if 'HTTP_TRANSFER_ENCODING' in new_env.keys(): del new_env['HTTP_TRANSFER_ENCODING'] new_env['REQUEST_METHOD'] = 'HEAD' new_env['swift.source'] = 'SE' new_env['PATH_INFO'] = os.path.join('/' + version, account, container, object) new_env['RAW_PATH_INFO'] = os.path.join('/' + version, account, container, object) storlet_req = Request.blank(new_env['PATH_INFO'], new_env) resp = storlet_req.get_response(self.app) if resp.status_int < 300 and resp.status_int >= 200: return True, resp.headers return False, [] def _validate_mandatory_headers(self, req): mandatory_md = None if self.container in [self.sconf['storlet_container']]: self.logger.info('PUT method for storlet dependency. Sanity check') mandatory_md = ['X-Object-Meta-Storlet-Language', 'X-Object-Meta-Storlet-Interface-Version', 'X-Object-Meta-Storlet-Dependency', 'X-Object-Meta-Storlet-Object-Metadata', 'X-Object-Meta-Storlet-Main'] elif self.container in [self.sconf['storlet_dependency']]: self.logger.info('PUT method for storlet container. Sanity check') mandatory_md = ['X-Object-Meta-Storlet-Dependency-Version'] if mandatory_md is not None: for md in mandatory_md: if md not in req.headers: self.logger.info('Mandatory header ' + 'is missing: {0}'.format(md)) return 'Mandatory header is missing: {0}'.format(md) return None def _fix_request_headers(self, req): # add to request the storlet metadata to be used in case the request # is forwarded to the data node (GET case) for key, val in self.storlet_metadata.iteritems(): if key.startswith('X-Object-Meta-Storlet'): req.headers[key] = val elif key in ['X-Timestamp', 'Content-Length']: req.headers['X-Storlet-' + key] = val def _add_system_params(self, params): ''' Adds Storlet engine specific parameters to the invocation currently, this consists only of the execution path of the Storlet within the Docker container. ''' params['storlet_execution_path'] = self.\ paths.sbox_storlet_exec(self.idata['storlet_main_class']) def _clean_storlet_stuff_from_request(self, headers): for key in headers: if (key.startswith('X-Storlet') or key.startswith('X-Object-Meta-Storlet')): del headers[key] return headers def _get_storlet_invocation_data(self, req): data = dict() data['storlet_name'] = req.headers.get('X-Run-Storlet') data['generate_log'] = req.headers.get('X-Storlet-Generate-Log', False) data['storlet_original_timestamp'] = req.headers.\ get('X-Storlet-X-Timestamp') data['storlet_original_size'] = req.headers.\ get('X-Storlet-Content-Length') data['storlet_md'] = {'storlet_original_timestamp': data['storlet_original_timestamp'], 'storlet_original_size': data['storlet_original_size']} data['storlet_main_class'] = req.headers.\ get('X-Object-Meta-Storlet-Main') scope = self.account data['scope'] = scope if data['scope'].rfind(':') > 0: data['scope'] = data['scope'][:data['scope'].rfind(':')] data['storlet_dependency'] = req.headers.\ get('X-Object-Meta-Storlet-Dependency') data['request_params'] = req.params return data def _set_metadata_in_headers(self, headers, md): if md: for key, val in md.iteritems(): headers['X-Object-Meta-%s' % key] = val def _upload_storlet_logs(self, slog_path): if (config_true_value(self.idata['generate_log'])): logfile = open(slog_path, 'r') client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1) try: headers = dict() headers['CONTENT_TYPE'] = 'text/html' log_obj_name = '%s.log' %\ self.idata['storlet_name'][:self.idata['storlet_name']. find('-')] client.upload_object(logfile, self.account, self.sconf['storlet_logcontainer'], log_obj_name, headers) except Exception as e: raise e def bring_from_cache(self, obj_name, is_storlet): ''' Auxiliary function that: (1) Brings from Swift obj_name, whether this is a storlet or a storlet dependency. (2) Copies from local cache into the Docker conrainer If this is a Storlet then also validates that the cache is updated with most recent copy of the Storlet compared to the copy residing in Swift. Retunrs wheather the Docker container was updated with obj_name ''' # Determine the cache we are to work with # e.g. dependency or storlet if not is_storlet: cache_dir = self.paths.get_host_dependency_cache_dir() swift_source_container = self.paths.storlet_dependency else: cache_dir = self.paths.get_host_storlet_cache_dir() swift_source_container = self.paths.storlet_container if not os.path.exists(cache_dir): os.makedirs(cache_dir, 0755) # cache_target_path is the actual object we need to deal with # e.g. a concrete storlet or dependency we need to bring/update cache_target_path = os.path.join(cache_dir, obj_name) # Determine if we need to update the cache for cache_target_path # We default for no update_cache = False # If it does not exist in cache, we obviously need to bring if not os.path.isfile(cache_target_path): update_cache = True elif is_storlet: # The cache_target_path exists, we test if it is up-to-date # with the metadata we got. # We mention that this is currenlty applicable for storlets # only, and not for dependencies. # This will change when we will head dependencies as well storlet_md = self.idata['storlet_md'] fstat = os.stat(cache_target_path) storlet_or_size = long(storlet_md['storlet_original_size']) storlet_or_time = float(storlet_md['storlet_original_timestamp']) b_storlet_size_changed = fstat.st_size != storlet_or_size b_storlet_file_updated = float(fstat.st_mtime) < storlet_or_time if b_storlet_size_changed or b_storlet_file_updated: update_cache = True expected_perm = '' if update_cache: # If the cache needs to be updated, then get on with it # bring the object from Swift using ic client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1) path = client.make_path(self.account, swift_source_container, obj_name) self.logger.debug('Invoking ic on path %s' % path) resp = client.make_request('GET', path, {'PATH_INFO' : path}, [200]) fn = open(cache_target_path, 'w') fn.write(resp.body) fn.close() if not is_storlet: expected_perm = resp.headers.\ get('X-Object-Meta-Storlet-Dependency-Permissions', '') if expected_perm != '': os.chmod(cache_target_path, int(expected_perm, 8)) # The node's local cache is now updated. # We now verify if we need to update the # Docker container itself. # The Docker container needs to be updated if: # 1. The Docker container does not hold a copy of the object # 2. The Docker container holds an older version of the object update_docker = False docker_storlet_path = self.paths.\ host_storlet(self.idata['storlet_main_class']) docker_target_path = os.path.join(docker_storlet_path, obj_name) if not os.path.exists(docker_storlet_path): os.makedirs(docker_storlet_path, 0755) update_docker = True elif not os.path.isfile(docker_target_path): update_docker = True else: fstat_cached_object = os.stat(cache_target_path) fstat_docker_object = os.stat(docker_target_path) b_size_changed = fstat_cached_object.st_size \ != fstat_docker_object.st_size b_time_changed = float(fstat_cached_object.st_mtime) <\ float(fstat_docker_object.st_mtime) if (b_size_changed or b_time_changed): update_docker = True if update_docker: # need to copy from cache to docker # copy2 also copies the permissions shutil.copy2(cache_target_path, docker_target_path) return update_docker def update_docker_container_from_cache(self): ''' Iterates over the storlet name and its dependencies appearing in the invocation data and make sure they are brought to the local cache, and from there to the Docker container. Uses the bring_from_cache auxiliary function. Returns True if the Docker container was updated ''' # where at the host side, reside the storlet containers storlet_path = self.paths.host_storlet_prefix() if not os.path.exists(storlet_path): os.makedirs(storlet_path, 0755) # Iterate over storlet and dependencies, and make sure # they are updated within the Docker container. # return True if any of them wea actually # updated within the Docker container docker_updated = False updated = self.bring_from_cache(self.idata['storlet_name'], True) docker_updated = docker_updated or updated if self.idata['storlet_dependency']: for dep in self.idata['storlet_dependency'].split(','): updated = self.bring_from_cache(dep, False) docker_updated = docker_updated or updated return docker_updated
class StorletGatewayDocker(StorletGatewayBase): def __init__(self, sconf, logger, app, version, account, container, obj): self.logger = logger # TODO(eranr): Add sconf defaults, and get rid of validate_conf below self.app = app self.version = version self.account = account self.container = container self.obj = obj self.sconf = sconf self.storlet_timeout = int(self.sconf['storlet_timeout']) self.paths = RunTimePaths(account, sconf) class IterLike(object): def __init__(self, obj_data, timeout, cancel_func): self.closed = False self.obj_data = obj_data self.timeout = timeout self.cancel_func = cancel_func self.buf = b'' def __iter__(self): return self def read_with_timeout(self, size): try: with StorletTimeout(self.timeout): chunk = os.read(self.obj_data, size) except StorletTimeout: if self.cancel_func: self.cancel_func() self.close() raise except Exception: self.close() raise return chunk def next(self, size=64 * 1024): if len(self.buf) < size: r, w, e = select.select([self.obj_data], [], [], self.timeout) if len(r) == 0: self.close() if self.obj_data in r: self.buf += self.read_with_timeout(size - len(self.buf)) if self.buf == b'': raise StopIteration('Stopped iterator ex') else: raise StopIteration('Stopped iterator ex') if len(self.buf) > size: data = self.buf[:size] self.buf = self.buf[size:] else: data = self.buf self.buf = b'' return data def _close_check(self): if self.closed: raise ValueError('I/O operation on closed file') def read(self, size=64 * 1024): self._close_check() return self.next(size) def readline(self, size=-1): self._close_check() # read data into self.buf if there is not enough data while b'\n' not in self.buf and \ (size < 0 or len(self.buf) < size): if size < 0: chunk = self.read() else: chunk = self.read(size - len(self.buf)) if not chunk: break self.buf += chunk # Retrieve one line from buf data, sep, rest = self.buf.partition(b'\n') data += sep self.buf = rest # cut out size from retrieved line if size >= 0 and len(data) > size: self.buf = data[size:] + self.buf data = data[:size] return data def readlines(self, sizehint=-1): self._close_check() lines = [] try: while True: line = self.readline(sizehint) if not line: break lines.append(line) if sizehint >= 0: sizehint -= len(line) if sizehint <= 0: break except StopIteration: pass return lines def close(self): if self.closed: return self.closed = True os.close(self.obj_data) def __del__(self): self.close() @classmethod def validate_storlet_registration(cls, params, name): mandatory = ['Language', 'Interface-Version', 'Dependency', 'Object-Metadata', 'Main'] StorletGatewayDocker._check_mandatory_params(params, mandatory) if '-' not in name or '.' not in name: raise ValueError('Storlet name is incorrect') @classmethod def validate_dependency_registration(cls, params, name): mandatory = ['Dependency-Version'] StorletGatewayDocker._check_mandatory_params(params, mandatory) perm = params.get('Dependency-Permissions') if perm is not None: try: perm_int = int(perm, 8) except ValueError: raise ValueError('Dependency permission is incorrect') if (perm_int & int('600', 8)) != int('600', 8): raise ValueError('The owner hould have rw permission') @classmethod def _check_mandatory_params(cls, params, mandatory): for md in mandatory: if md not in params: raise ValueError('Mandatory parameter is missing' ': {0}'.format(md)) def augmentStorletRequest(self, req, params): """ Add to request the storlet parameters to be used in case the request is forwarded to the data node (GET case) """ for key, val in params.iteritems(): req.headers['X-Storlet-' + key] = val def gateway_proxy_put_copy_flow(self, sreq): req = sreq._getInitialRequest() self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) # Clean all Storlet stuff from the request headers # we do not need them anymore, and they # may interfere with the rest of the execution. self._clean_storlet_stuff_from_request(req.headers) req.headers.pop('X-Run-Storlet') slog_path = self. \ paths.slog_path(self.idata['storlet_main_class']) storlet_pipe_path = self. \ paths.host_storlet_pipe(self.idata['storlet_main_class']) if isinstance(sreq, StorletCopyRequest): sprotocol = StorletInvocationCOPYProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) else: sprotocol = StorletInvocationPUTProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() self._set_metadata_in_headers(req.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, self.storlet_timeout, sprotocol._cancel) def gatewayProxyPutFlow(self, orig_req): sreq = StorletPUTRequest(self.account, orig_req) return self.gateway_proxy_put_copy_flow(sreq) def gatewayProxyCopyFlow(self, orig_req, src_response): sreq = StorletCopyRequest(self.account, orig_req, src_response) return self.gateway_proxy_put_copy_flow(sreq) def gatewayProxyGetFlow(self, req, orig_resp): # Flow for running the GET computation on the proxy sreq = StorletSLORequest(self.account, orig_resp, req.params) self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) slog_path = self. \ paths.slog_path(self.idata['storlet_main_class']) storlet_pipe_path = self. \ paths.host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationSLOProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() self._set_metadata_in_headers(orig_resp.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, self.storlet_timeout, sprotocol._cancel) def gatewayObjectGetFlow(self, req, orig_resp): sreq = StorletGETRequest(self.account, orig_resp, req.params) self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) slog_path = self. \ paths.slog_path(self.idata['storlet_main_class']) storlet_pipe_path = self.paths. \ host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationGETProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() orig_resp = sreq._getInitialRequest() self._set_metadata_in_headers(orig_resp.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, self.storlet_timeout, sprotocol._cancel) def _add_system_params(self, params): """ Adds Storlet engine specific parameters to the invocation currently, this consists only of the execution path of the Storlet within the Docker container. :params params: Request parameters """ params['storlet_execution_path'] = self. \ paths.sbox_storlet_exec(self.idata['storlet_main_class']) def _clean_storlet_stuff_from_request(self, headers): for key in headers: if (key.startswith('X-Storlet') or key.startswith('X-Object-Meta-Storlet')): del headers[key] return headers def _get_storlet_invocation_data(self, req): data = dict() data['storlet_name'] = req.headers.get('X-Run-Storlet') data['generate_log'] = req.headers.get('X-Storlet-Generate-Log', False) data['storlet_original_timestamp'] = req.headers. \ get('X-Storlet-X-Timestamp') data['storlet_original_size'] = req.headers. \ get('X-Storlet-Content-Length') data['storlet_md'] = {'storlet_original_timestamp': data['storlet_original_timestamp'], 'storlet_original_size': data['storlet_original_size']} data['storlet_main_class'] = req.headers. \ get('X-Storlet-Main') scope = self.account data['scope'] = scope if data['scope'].rfind(':') > 0: data['scope'] = data['scope'][:data['scope'].rfind(':')] data['storlet_dependency'] = req.headers. \ get('X-Storlet-Dependency') data['request_params'] = req.params return data def _set_metadata_in_headers(self, headers, md): if md: for key, val in md.iteritems(): headers['X-Object-Meta-%s' % key] = val def _upload_storlet_logs(self, slog_path): if (config_true_value(self.idata['generate_log'])): with open(slog_path, 'r') as logfile: client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1) # TODO(takashi): Is it really html? # (I suppose it should be text/plain) headers = {'CONTENT_TYPE': 'text/html'} storlet_name = self.idata['storlet_name'].split('-')[0] log_obj_name = '%s.log' % storlet_name # TODO(takashi): we had better retrieve required values from # sconf in __init__ client.upload_object(logfile, self.account, self.sconf['storlet_logcontainer'], log_obj_name, headers) def bring_from_cache(self, obj_name, is_storlet): """ Auxiliary function that: (1) Brings from Swift obj_name, whether this is a storlet or a storlet dependency. (2) Copies from local cache into the Docker conrainer If this is a Storlet then also validates that the cache is updated with most recent copy of the Storlet compared to the copy residing in Swift. :params obj_name: name of the object :params is_storlet: True if the object is a storlet object False if the object is a dependency object :returns: Wheather the Docker container was updated with obj_name """ # Determine the cache we are to work with # e.g. dependency or storlet if not is_storlet: cache_dir = self.paths.get_host_dependency_cache_dir() swift_source_container = self.paths.storlet_dependency else: cache_dir = self.paths.get_host_storlet_cache_dir() swift_source_container = self.paths.storlet_container if not os.path.exists(cache_dir): os.makedirs(cache_dir, 0o755) # cache_target_path is the actual object we need to deal with # e.g. a concrete storlet or dependency we need to bring/update cache_target_path = os.path.join(cache_dir, obj_name) # Determine if we need to update the cache for cache_target_path # We default for no update_cache = False # If it does not exist in cache, we obviously need to bring if not os.path.isfile(cache_target_path): update_cache = True elif is_storlet: # The cache_target_path exists, we test if it is up-to-date # with the metadata we got. # We mention that this is currenlty applicable for storlets # only, and not for dependencies. # This will change when we will head dependencies as well storlet_md = self.idata['storlet_md'] fstat = os.stat(cache_target_path) storlet_or_size = long(storlet_md['storlet_original_size']) storlet_or_time = float(storlet_md['storlet_original_timestamp']) b_storlet_size_changed = fstat.st_size != storlet_or_size b_storlet_file_updated = float(fstat.st_mtime) < storlet_or_time if b_storlet_size_changed or b_storlet_file_updated: update_cache = True expected_perm = '' if update_cache: # If the cache needs to be updated, then get on with it # bring the object from Swift using ic client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1) path = client.make_path(self.account, swift_source_container, obj_name) self.logger.debug('Invoking ic on path %s' % path) resp = client.make_request('GET', path, {'PATH_INFO': path}, [200]) with open(cache_target_path, 'w') as fn: fn.write(resp.body) if not is_storlet: expected_perm = resp.headers. \ get('X-Object-Meta-Storlet-Dependency-Permissions', '') if expected_perm != '': os.chmod(cache_target_path, int(expected_perm, 8)) else: os.chmod(cache_target_path, int('0600', 8)) # The node's local cache is now updated. # We now verify if we need to update the # Docker container itself. # The Docker container needs to be updated if: # 1. The Docker container does not hold a copy of the object # 2. The Docker container holds an older version of the object update_docker = False docker_storlet_path = self.paths. \ host_storlet(self.idata['storlet_main_class']) docker_target_path = os.path.join(docker_storlet_path, obj_name) if not os.path.exists(docker_storlet_path): os.makedirs(docker_storlet_path, 0o755) update_docker = True elif not os.path.isfile(docker_target_path): update_docker = True else: fstat_cached_object = os.stat(cache_target_path) fstat_docker_object = os.stat(docker_target_path) b_size_changed = fstat_cached_object.st_size \ != fstat_docker_object.st_size b_time_changed = float(fstat_cached_object.st_mtime) < \ float(fstat_docker_object.st_mtime) if (b_size_changed or b_time_changed): update_docker = True if update_docker: # need to copy from cache to docker # copy2 also copies the permissions shutil.copy2(cache_target_path, docker_target_path) return update_docker def update_docker_container_from_cache(self): """ Iterates over the storlet name and its dependencies appearing in the invocation data and make sure they are brought to the local cache, and from there to the Docker container. Uses the bring_from_cache auxiliary function. :returns: True if the Docker container was updated """ # where at the host side, reside the storlet containers storlet_path = self.paths.host_storlet_prefix() if not os.path.exists(storlet_path): os.makedirs(storlet_path, 0o755) # Iterate over storlet and dependencies, and make sure # they are updated within the Docker container. # return True if any of them wea actually # updated within the Docker container docker_updated = False updated = self.bring_from_cache(self.idata['storlet_name'], True) docker_updated = docker_updated or updated if self.idata['storlet_dependency']: for dep in self.idata['storlet_dependency'].split(','): updated = self.bring_from_cache(dep, False) docker_updated = docker_updated or updated return docker_updated
class StorletGatewayDocker(StorletGatewayBase): def __init__(self, sconf, logger, app, version, account, container, obj): self.logger = logger self.app = app self.version = version self.account = account self.container = container self.obj = obj self.sconf = sconf self.storlet_metadata = None self.storlet_timeout = int(self.sconf["storlet_timeout"]) self.paths = RunTimePaths(account, sconf) def validateStorletUpload(self, req): if self.container == self.sconf["storlet_container"]: if self.obj.find("-") < 0 or self.obj.find(".") < 0: return "Storlet name is incorrect" ret = self._validate_mandatory_headers(req) if ret: return ret return False def authorizeStorletExecution(self, req): res, headers = self.verify_access( req.environ, self.version, self.account, self.sconf["storlet_container"], req.headers["X-Run-Storlet"] ) if not res: return False # keep the storlets headers for later use. self.storlet_metadata = headers return True def augmentStorletRequest(self, req): if self.storlet_metadata: self._fix_request_headers(req) def gatewayProxyPutFlow(self, orig_req, container, obj): sreq = StorletPUTRequest(self.account, orig_req) req = sreq._getInitialRequest() self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) # Clean all Storlet stuff from the request headers # we do not need them anymore, and they # may interfere with the rest of the execution. self._clean_storlet_stuff_from_request(req.headers) req.headers.pop("X-Run-Storlet") slog_path = self.paths.slog_path(self.idata["storlet_main_class"]) storlet_pipe_path = self.paths.host_storlet_pipe(self.idata["storlet_main_class"]) sprotocol = StorletInvocationPUTProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() self._set_metadata_in_headers(req.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, self.data_read_fd def gatewayProxyGETFlow(self, req, container, obj, orig_resp): # Flow for running the GET computation on the proxy sreq = StorletSLORequest(self.account, orig_resp, req.params) self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) slog_path = self.paths.slog_path(self.idata["storlet_main_class"]) storlet_pipe_path = self.paths.host_storlet_pipe(self.idata["storlet_main_class"]) sprotocol = StorletInvocationSLOProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() self._set_metadata_in_headers(orig_resp.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, self.data_read_fd def gatewayObjectGetFlow(self, req, container, obj, orig_resp): sreq = StorletGETRequest(self.account, orig_resp, req.params) self.idata = self._get_storlet_invocation_data(req) run_time_sbox = RunTimeSandbox(self.account, self.sconf, self.logger) docker_updated = self.update_docker_container_from_cache() run_time_sbox.activate_storlet_daemon(self.idata, docker_updated) self._add_system_params(req.params) slog_path = self.paths.slog_path(self.idata["storlet_main_class"]) storlet_pipe_path = self.paths.host_storlet_pipe(self.idata["storlet_main_class"]) sprotocol = StorletInvocationGETProtocol(sreq, storlet_pipe_path, slog_path, self.storlet_timeout) out_md, self.data_read_fd = sprotocol.communicate() orig_resp = sreq._getInitialRequest() self._set_metadata_in_headers(orig_resp.headers, out_md) self._upload_storlet_logs(slog_path) return out_md, self.data_read_fd def verify_access(self, env, version, account, container, object): self.logger.info("Verify access to {0}/{1}/{2}".format(account, container, object)) new_env = dict(env) if "HTTP_TRANSFER_ENCODING" in new_env.keys(): del new_env["HTTP_TRANSFER_ENCODING"] new_env["REQUEST_METHOD"] = "HEAD" new_env["swift.source"] = "SE" new_env["PATH_INFO"] = os.path.join("/" + version, account, container, object) new_env["RAW_PATH_INFO"] = os.path.join("/" + version, account, container, object) storlet_req = Request.blank(new_env["PATH_INFO"], new_env) resp = storlet_req.get_response(self.app) if resp.status_int < 300 and resp.status_int >= 200: return True, resp.headers return False, [] def _validate_mandatory_headers(self, req): mandatory_md = None if self.container in [self.sconf["storlet_container"]]: self.logger.info("PUT method for storlet dependency. Sanity check") mandatory_md = [ "X-Object-Meta-Storlet-Language", "X-Object-Meta-Storlet-Interface-Version", "X-Object-Meta-Storlet-Dependency", "X-Object-Meta-Storlet-Object-Metadata", "X-Object-Meta-Storlet-Main", ] elif self.container in [self.sconf["storlet_dependency"]]: self.logger.info("PUT method for storlet container. Sanity check") mandatory_md = ["X-Object-Meta-Storlet-Dependency-Version"] if mandatory_md is not None: for md in mandatory_md: if md not in req.headers: self.logger.info("Mandatory header " + "is missing: {0}".format(md)) return "Mandatory header is missing: {0}".format(md) return None def _fix_request_headers(self, req): # add to request the storlet metadata to be used in case the request # is forwarded to the data node (GET case) for key, val in self.storlet_metadata.iteritems(): if key.startswith("X-Object-Meta-Storlet"): req.headers[key] = val elif key in ["X-Timestamp", "Content-Length"]: req.headers["X-Storlet-" + key] = val def _add_system_params(self, params): """ Adds Storlet engine specific parameters to the invocation currently, this consists only of the execution path of the Storlet within the Docker container. """ params["storlet_execution_path"] = self.paths.sbox_storlet_exec(self.idata["storlet_main_class"]) def _clean_storlet_stuff_from_request(self, headers): for key in headers: if key.startswith("X-Storlet") or key.startswith("X-Object-Meta-Storlet"): del headers[key] return headers def _get_storlet_invocation_data(self, req): data = dict() data["storlet_name"] = req.headers.get("X-Run-Storlet") data["generate_log"] = req.headers.get("X-Storlet-Generate-Log", False) data["storlet_original_timestamp"] = req.headers.get("X-Storlet-X-Timestamp") data["storlet_original_size"] = req.headers.get("X-Storlet-Content-Length") data["storlet_md"] = { "storlet_original_timestamp": data["storlet_original_timestamp"], "storlet_original_size": data["storlet_original_size"], } data["storlet_main_class"] = req.headers.get("X-Object-Meta-Storlet-Main") scope = self.account data["scope"] = scope if data["scope"].rfind(":") > 0: data["scope"] = data["scope"][: data["scope"].rfind(":")] data["storlet_dependency"] = req.headers.get("X-Object-Meta-Storlet-Dependency") data["request_params"] = req.params return data def _set_metadata_in_headers(self, headers, md): if md: for key, val in md.iteritems(): headers["X-Object-Meta-%s" % key] = val def _upload_storlet_logs(self, slog_path): if config_true_value(self.idata["generate_log"]): logfile = open(slog_path, "r") client = ic("/etc/swift/storlet-proxy-server.conf", "SA", 1) try: headers = dict() headers["CONTENT_TYPE"] = "text/html" log_obj_name = "%s.log" % self.idata["storlet_name"][: self.idata["storlet_name"].find("-")] client.upload_object(logfile, self.account, self.sconf["storlet_logcontainer"], log_obj_name, headers) except Exception as e: raise e def bring_from_cache(self, obj_name, is_storlet): """ Auxiliary function that: (1) Brings from Swift obj_name, whether this is a storlet or a storlet dependency. (2) Copies from local cache into the Docker conrainer If this is a Storlet then also validates that the cache is updated with most recent copy of the Storlet compared to the copy residing in Swift. Retunrs wheather the Docker container was updated with obj_name """ # Determine the cache we are to work with # e.g. dependency or storlet if not is_storlet: cache_dir = self.paths.get_host_dependency_cache_dir() swift_source_container = self.paths.storlet_dependency else: cache_dir = self.paths.get_host_storlet_cache_dir() swift_source_container = self.paths.storlet_container if not os.path.exists(cache_dir): os.makedirs(cache_dir, 0755) # cache_target_path is the actual object we need to deal with # e.g. a concrete storlet or dependency we need to bring/update cache_target_path = os.path.join(cache_dir, obj_name) # Determine if we need to update the cache for cache_target_path # We default for no update_cache = False # If it does not exist in cache, we obviously need to bring if not os.path.isfile(cache_target_path): update_cache = True elif is_storlet: # The cache_target_path exists, we test if it is up-to-date # with the metadata we got. # We mention that this is currenlty applicable for storlets # only, and not for dependencies. # This will change when we will head dependencies as well storlet_md = self.idata["storlet_md"] fstat = os.stat(cache_target_path) storlet_or_size = long(storlet_md["storlet_original_size"]) storlet_or_time = float(storlet_md["storlet_original_timestamp"]) b_storlet_size_changed = fstat.st_size != storlet_or_size b_storlet_file_updated = float(fstat.st_mtime) < storlet_or_time if b_storlet_size_changed or b_storlet_file_updated: update_cache = True expected_perm = "" if update_cache: # If the cache needs to be updated, then get on with it # bring the object from Swift using ic client = ic("/etc/swift/storlet-proxy-server.conf", "SA", 1) path = client.make_path(self.account, swift_source_container, obj_name) self.logger.debug("Invoking ic on path %s" % path) resp = client.make_request("GET", path, {"PATH_INFO": path}, [200]) fn = open(cache_target_path, "w") fn.write(resp.body) fn.close() if not is_storlet: expected_perm = resp.headers.get("X-Object-Meta-Storlet-Dependency-Permissions", "") if expected_perm != "": os.chmod(cache_target_path, int(expected_perm, 8)) # The node's local cache is now updated. # We now verify if we need to update the # Docker container itself. # The Docker container needs to be updated if: # 1. The Docker container does not hold a copy of the object # 2. The Docker container holds an older version of the object update_docker = False docker_storlet_path = self.paths.host_storlet(self.idata["storlet_main_class"]) docker_target_path = os.path.join(docker_storlet_path, obj_name) if not os.path.exists(docker_storlet_path): os.makedirs(docker_storlet_path, 0755) update_docker = True elif not os.path.isfile(docker_target_path): update_docker = True else: fstat_cached_object = os.stat(cache_target_path) fstat_docker_object = os.stat(docker_target_path) b_size_changed = fstat_cached_object.st_size != fstat_docker_object.st_size b_time_changed = float(fstat_cached_object.st_mtime) < float(fstat_docker_object.st_mtime) if b_size_changed or b_time_changed: update_docker = True if update_docker: # need to copy from cache to docker # copy2 also copies the permissions shutil.copy2(cache_target_path, docker_target_path) return update_docker def update_docker_container_from_cache(self): """ Iterates over the storlet name and its dependencies appearing in the invocation data and make sure they are brought to the local cache, and from there to the Docker container. Uses the bring_from_cache auxiliary function. Returns True if the Docker container was updated """ # where at the host side, reside the storlet containers storlet_path = self.paths.host_storlet_prefix() if not os.path.exists(storlet_path): os.makedirs(storlet_path, 0755) # Iterate over storlet and dependencies, and make sure # they are updated within the Docker container. # return True if any of them wea actually # updated within the Docker container docker_updated = False updated = self.bring_from_cache(self.idata["storlet_name"], True) docker_updated = docker_updated or updated if self.idata["storlet_dependency"]: for dep in self.idata["storlet_dependency"].split(","): updated = self.bring_from_cache(dep, False) docker_updated = docker_updated or updated return docker_updated