def _upload_storlet_logs(self, slog_path):
if config_true_value(self.idata["generate_log"]):
logfile = open(slog_path, "r")
client = ic("/etc/swift/storlet-proxy-server.conf", "SA", 1)
try:
headers = dict()
headers["CONTENT_TYPE"] = "text/html"
log_obj_name = "%s.log" % self.idata["storlet_name"][: self.idata["storlet_name"].find("-")]
client.upload_object(logfile, self.account, self.sconf["storlet_logcontainer"], log_obj_name, headers)
except Exception as e:
raise e
def _upload_storlet_logs(self, slog_path):
if (config_true_value(self.idata['generate_log'])):
with open(slog_path, 'r') as logfile:
client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1)
# TODO(takashi): Is it really html?
# (I suppose it should be text/plain)
headers = {'CONTENT_TYPE': 'text/html'}
storlet_name = self.idata['storlet_name'].split('-')[0]
log_obj_name = '%s.log' % storlet_name
# TODO(takashi): we had better retrieve required values from
# sconf in __init__
client.upload_object(logfile, self.account,
self.sconf['storlet_logcontainer'],
log_obj_name, headers)
def _upload_storlet_logs(self, slog_path):
if (config_true_value(self.idata['generate_log'])):
logfile = open(slog_path, 'r')
client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1)
try:
headers = dict()
headers['CONTENT_TYPE'] = 'text/html'
log_obj_name = '%s.log' %\
self.idata['storlet_name'][:self.idata['storlet_name'].
find('-')]
client.upload_object(logfile, self.account,
self.sconf['storlet_logcontainer'],
log_obj_name, headers)
except Exception as e:
raise e
def _upload_storlet_logs(self, slog_path):
if (config_true_value(self.idata['generate_log'])):
logfile = open(slog_path, 'r')
client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1)
try:
headers = dict()
headers['CONTENT_TYPE'] = 'text/html'
log_obj_name = '%s.log' %\
self.idata['storlet_name'][:self.idata['storlet_name'].
find('-')]
client.upload_object(logfile, self.account,
self.sconf['storlet_logcontainer'],
log_obj_name, headers)
except Exception as e:
raise e
def bring_from_cache(self, obj_name, is_storlet):
'''
Auxiliary function that:
(1) Brings from Swift obj_name, whether this is a
storlet or a storlet dependency.
(2) Copies from local cache into the Docker conrainer
If this is a Storlet then also validates that the cache is updated
with most recent copy of the Storlet compared to the copy residing in
Swift.
Retunrs wheather the Docker container was updated with obj_name
'''
# Determine the cache we are to work with
# e.g. dependency or storlet
if not is_storlet:
cache_dir = self.paths.get_host_dependency_cache_dir()
swift_source_container = self.paths.storlet_dependency
else:
cache_dir = self.paths.get_host_storlet_cache_dir()
swift_source_container = self.paths.storlet_container
if not os.path.exists(cache_dir):
os.makedirs(cache_dir, 0755)
# cache_target_path is the actual object we need to deal with
# e.g. a concrete storlet or dependency we need to bring/update
cache_target_path = os.path.join(cache_dir, obj_name)
# Determine if we need to update the cache for cache_target_path
# We default for no
update_cache = False
# If it does not exist in cache, we obviously need to bring
if not os.path.isfile(cache_target_path):
update_cache = True
elif is_storlet:
# The cache_target_path exists, we test if it is up-to-date
# with the metadata we got.
# We mention that this is currenlty applicable for storlets
# only, and not for dependencies.
# This will change when we will head dependencies as well
storlet_md = self.idata['storlet_md']
fstat = os.stat(cache_target_path)
storlet_or_size = long(storlet_md['storlet_original_size'])
storlet_or_time = float(storlet_md['storlet_original_timestamp'])
b_storlet_size_changed = fstat.st_size != storlet_or_size
b_storlet_file_updated = float(fstat.st_mtime) < storlet_or_time
if b_storlet_size_changed or b_storlet_file_updated:
update_cache = True
expected_perm = ''
if update_cache:
# If the cache needs to be updated, then get on with it
# bring the object from Swift using ic
client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1)
path = client.make_path(self.account, swift_source_container,
obj_name)
self.logger.debug('Invoking ic on path %s' % path)
resp = client.make_request('GET', path, {'PATH_INFO': path}, [200])
fn = open(cache_target_path, 'w')
fn.write(resp.body)
fn.close()
if not is_storlet:
expected_perm = resp.headers.\
get('X-Object-Meta-Storlet-Dependency-Permissions', '')
if expected_perm != '':
os.chmod(cache_target_path, int(expected_perm, 8))
# The node's local cache is now updated.
# We now verify if we need to update the
# Docker container itself.
# The Docker container needs to be updated if:
# 1. The Docker container does not hold a copy of the object
# 2. The Docker container holds an older version of the object
update_docker = False
docker_storlet_path = self.paths.\
host_storlet(self.idata['storlet_main_class'])
docker_target_path = os.path.join(docker_storlet_path, obj_name)
if not os.path.exists(docker_storlet_path):
os.makedirs(docker_storlet_path, 0755)
update_docker = True
elif not os.path.isfile(docker_target_path):
update_docker = True
else:
fstat_cached_object = os.stat(cache_target_path)
fstat_docker_object = os.stat(docker_target_path)
b_size_changed = fstat_cached_object.st_size \
!= fstat_docker_object.st_size
b_time_changed = float(fstat_cached_object.st_mtime) <\
float(fstat_docker_object.st_mtime)
if (b_size_changed or b_time_changed):
update_docker = True
if update_docker:
# need to copy from cache to docker
# copy2 also copies the permissions
shutil.copy2(cache_target_path, docker_target_path)
return update_docker
def bring_from_cache(self, obj_name, is_storlet):
'''
Auxiliary function that:
(1) Brings from Swift obj_name, whether this is a
storlet or a storlet dependency.
(2) Copies from local cache into the Docker conrainer
If this is a Storlet then also validates that the cache is updated
with most recent copy of the Storlet compared to the copy residing in
Swift.
Retunrs wheather the Docker container was updated with obj_name
'''
# Determine the cache we are to work with
# e.g. dependency or storlet
if not is_storlet:
cache_dir = self.paths.get_host_dependency_cache_dir()
swift_source_container = self.paths.storlet_dependency
else:
cache_dir = self.paths.get_host_storlet_cache_dir()
swift_source_container = self.paths.storlet_container
if not os.path.exists(cache_dir):
os.makedirs(cache_dir, 0755)
# cache_target_path is the actual object we need to deal with
# e.g. a concrete storlet or dependency we need to bring/update
cache_target_path = os.path.join(cache_dir, obj_name)
# Determine if we need to update the cache for cache_target_path
# We default for no
update_cache = False
# If it does not exist in cache, we obviously need to bring
if not os.path.isfile(cache_target_path):
update_cache = True
elif is_storlet:
# The cache_target_path exists, we test if it is up-to-date
# with the metadata we got.
# We mention that this is currenlty applicable for storlets
# only, and not for dependencies.
# This will change when we will head dependencies as well
storlet_md = self.idata['storlet_md']
fstat = os.stat(cache_target_path)
storlet_or_size = long(storlet_md['storlet_original_size'])
storlet_or_time = float(storlet_md['storlet_original_timestamp'])
b_storlet_size_changed = fstat.st_size != storlet_or_size
b_storlet_file_updated = float(fstat.st_mtime) < storlet_or_time
if b_storlet_size_changed or b_storlet_file_updated:
update_cache = True
expected_perm = ''
if update_cache:
# If the cache needs to be updated, then get on with it
# bring the object from Swift using ic
client = ic('/etc/swift/storlet-proxy-server.conf', 'SA', 1)
path = client.make_path(self.account, swift_source_container,
obj_name)
self.logger.debug('Invoking ic on path %s' % path)
resp = client.make_request('GET', path, {'PATH_INFO' : path}, [200])
fn = open(cache_target_path, 'w')
fn.write(resp.body)
fn.close()
if not is_storlet:
expected_perm = resp.headers.\
get('X-Object-Meta-Storlet-Dependency-Permissions', '')
if expected_perm != '':
os.chmod(cache_target_path, int(expected_perm, 8))
# The node's local cache is now updated.
# We now verify if we need to update the
# Docker container itself.
# The Docker container needs to be updated if:
# 1. The Docker container does not hold a copy of the object
# 2. The Docker container holds an older version of the object
update_docker = False
docker_storlet_path = self.paths.\
host_storlet(self.idata['storlet_main_class'])
docker_target_path = os.path.join(docker_storlet_path, obj_name)
if not os.path.exists(docker_storlet_path):
os.makedirs(docker_storlet_path, 0755)
update_docker = True
elif not os.path.isfile(docker_target_path):
update_docker = True
else:
fstat_cached_object = os.stat(cache_target_path)
fstat_docker_object = os.stat(docker_target_path)
b_size_changed = fstat_cached_object.st_size \
!= fstat_docker_object.st_size
b_time_changed = float(fstat_cached_object.st_mtime) <\
float(fstat_docker_object.st_mtime)
if (b_size_changed or b_time_changed):
update_docker = True
if update_docker:
# need to copy from cache to docker
# copy2 also copies the permissions
shutil.copy2(cache_target_path, docker_target_path)
return update_docker