def open(self, filepath, mode):
"""Opens an hpss file"""
#want to close any open files first
try:
if self._file:
self.close()
except ArchiveInterfaceError as ex:
err_str = "Can't close previous hpss file before "\
"opening new one with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
#try to open file
try:
fpath = un_abs_path(filepath)
filename = os.path.join(self._prefix, path_info_munge(fpath))
self._filepath = filename
hpss = HpssExtended(self._filepath, self._latency)
hpss.ping_core()
hpss_fopen = self._hpsslib.hpss_Fopen
hpss_fopen.restype = c_void_p
self._file = hpss_fopen(filename, mode)
if self._file < 0:
err_str = "Failed opening Hpss File, code: " + str(self._file)
raise ArchiveInterfaceError(err_str)
return self
except Exception as ex:
err_str = "Can't open hpss file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def write(self, buf):
"""Write a file to the hpss archive"""
try:
if self._file:
hpss = HpssExtended(self._filepath, self._latency)
hpss.ping_core()
buf_char_p = cast(buf, c_char_p)
rcode = self._hpsslib.hpss_Fwrite(buf_char_p, 1, len(buf),
self._file)
if rcode != len(buf):
raise ArchiveInterfaceError("Short write for hpss file")
except Exception as ex:
err_str = "Can't write hpss file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def read_config_value(section, field):
"""reads the value from the config file if exists"""
try:
config = ConfigParser.RawConfigParser()
dataset = config.read(CONFIG_FILE)
if not len(dataset):
raise ValueError, "Failed to open config file with name: " + str(CONFIG_FILE)
value = config.get(section, field)
return value
except ConfigParser.NoSectionError:
raise ArchiveInterfaceError("Error reading config file, no section: " + section)
except ConfigParser.NoOptionError:
raise ArchiveInterfaceError("Error reading config file, no field: " + field +
" in section: " + section)
def close(self):
"""Close an HPSS File"""
try:
if self._file:
hpss = HpssExtended(self._filepath, self._latency)
hpss.ping_core()
rcode = self._hpsslib.hpss_Fclose(self._file)
if rcode < 0:
err_str = "Failed to close hpss file with code: " + str(
rcode)
raise ArchiveInterfaceError(err_str)
self._file = None
except Exception as ex:
err_str = "Can't close hpss file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def put(self, env, start_response):
"""Writes a file passed in the request to the archive"""
archivefile = None
resp = interface_responses.Responses()
path_info = env['PATH_INFO']
mod_time = get_http_modified_time(env)
stderr.flush()
archivefile = self._archive.open(path_info, "w")
try:
content_length = int(env['CONTENT_LENGTH'])
except Exception as ex:
raise ArchiveInterfaceError("Can't get file content length with "\
"error: " + str(ex))
while content_length > 0:
if content_length > BLOCK_SIZE:
buf = env['wsgi.input'].read(BLOCK_SIZE)
else:
buf = env['wsgi.input'].read(content_length)
archivefile.write(buf)
content_length -= len(buf)
self._response = resp.successful_put_response(start_response,
env['CONTENT_LENGTH'])
archivefile.close()
archivefile.set_mod_time(mod_time)
return self.return_response()
def read(self, blocksize):
"""Read a file from the hpss archive"""
try:
if self._file:
hpss = HpssExtended(self._filepath, self._latency)
hpss.ping_core()
buf = create_string_buffer('\000' * blocksize)
rcode = self._hpsslib.hpss_Fread(buf, 1, blocksize, self._file)
if rcode < 0:
err_str = "Failed During HPSS Fread,"\
"return value is: " + str(rcode)
raise ArchiveInterfaceError(err_str)
return buf.value
except Exception as ex:
err_str = "Can't read hpss file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def un_abs_path(path_name):
"""Removes absolute path piece"""
try:
if path.isabs(path_name):
path_name = path_name[1:]
return path_name
except AttributeError as ex:
raise ArchiveInterfaceError("Cant remove absolute path: " + str(ex))
def write(self, buf):
"""Write a HMS Sideband file to the archive"""
try:
if self._file:
return self._file.write(buf)
except Exception as ex:
err_str = "Can't write HMS Sideband file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def read(self, blocksize):
"""Read a HMS Sideband file"""
try:
if self._file:
return self._file.read(blocksize)
except Exception as ex:
err_str = "Can't read HMS SIdeband file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def status(self):
"""Get the status of a posix file"""
try:
if self._file:
return self._file.status()
except Exception as ex:
err_str = "Can't get posix file status with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def stage(self):
"""Stage a posix file (no-opt essentially)"""
try:
if self._file:
return self._file.stage()
except Exception as ex:
err_str = "Can't stage posix file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def set_mod_time(self, mod_time):
"""Set the mod time on a posix file"""
try:
if self._file:
self._file.set_mod_time(mod_time)
except Exception as ex:
err_str = "Can't set posix file mod time with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def stage(self):
"""Stage a HMS Sideband file"""
try:
if self._file:
return self._file.stage()
except Exception as ex:
err_str = "Can't stage HMS Sideband file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def close(self):
"""Close a HMS Sideband file"""
try:
if self._file:
self._file.close()
self._file = None
except Exception as ex:
err_str = "Can't close HMS Sideband file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def open(self, filepath, mode):
"""Open a posix file"""
#want to close any open files first
try:
if self._file:
self.close()
except ArchiveInterfaceError as ex:
err_str = "Can't close previous posix file before opening new "\
"one with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
try:
fpath = un_abs_path(filepath)
filename = os.path.join(self._prefix, fpath)
self._file = ExtendedFile(filename, mode)
return self
except Exception as ex:
err_str = "Can't open posix file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def set_mod_time(self, mod_time):
"""Use extensions to set the mod time on an hpss file"""
try:
_hpssExtensions.hpss_utime(self._filepath, float(mod_time))
except Exception as ex:
#Push the excpetion up the chain to the response
err_str = "Error using c extension for hpss utime"\
" exception: " + str(ex)
raise ArchiveInterfaceError(err_str)
def authenticate(self):
"""Authenticates the user with the hpss system """
rcode = self._hpsslib.hpss_SetLoginCred(self._user,
HPSS_AUTHN_MECH_UNIX,
HPSS_RPC_CRED_CLIENT,
HPSS_RPC_AUTH_TYPE_KEYTAB,
self._auth)
if rcode != 0:
err_str = "Could Not Authenticate, error code is:" + str(rcode)
raise ArchiveInterfaceError(err_str)
def set_mod_time(self, mod_time):
"""Set the mod time for an hpss archive file """
try:
if self._file:
hpss = HpssExtended(self._filepath, self._latency)
hpss.ping_core()
hpss.set_mod_time(mod_time)
except Exception as ex:
err_str = "Can't set hpss file mod time with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def status(self):
"""Get the status of a file in the hpss archive """
try:
if self._file:
hpss = HpssExtended(self._filepath, self._latency)
hpss.ping_core()
return hpss.status()
except Exception as ex:
err_str = "Can't get hpss status with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def stage(self):
"""Stage an hpss file to the top level drive """
try:
if self._file:
hpss = HpssExtended(self._filepath, self._latency)
hpss.ping_core()
hpss.stage()
except Exception as ex:
err_str = "Can't stage hpss file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def get_http_modified_time(env):
"""Gets the modified time from the request in unix timestamp.
Returns current time if no time was passed"""
try:
mod_time = None
if 'HTTP_LAST_MODIFIED' in env:
mod_time = eut.mktime_tz(eut.parsedate_tz(env['HTTP_LAST_MODIFIED']))
else:
mod_time = time.time()
return mod_time
except (TypeError, IndexError, AttributeError) as ex:
raise ArchiveInterfaceError("Cant parse the files modtime: " + str(ex))
def __init__(self, prefix):
super(HpssBackendArchive, self).__init__(prefix)
self._prefix = prefix
self._user = read_config_value('hpss', 'user')
self._auth = read_config_value('hpss', 'auth')
self._file = None
self._filepath = None
self._hpsslib = None
self._latency = 5 #number not significant
#need to load the hpss libraries/ extensions
try:
self._hpsslib = cdll.LoadLibrary(HPSS_LIBRARY_PATH)
except Exception as ex:
err_str = "Can't load hpss libraries with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
#need to authenticate with hpss
try:
self.authenticate()
except Exception as ex:
err_str = "Can't authenticate with hpss, error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def open(self, filepath, mode):
"""Open a hms sideband file"""
#want to close any open files first
try:
if self._file:
self.close()
except ArchiveInterfaceError as ex:
err_str = "Can't close previous HMS Sideband file before opening new "\
"one with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
try:
self._fpath = un_abs_path(filepath)
filename = os.path.join(self._prefix, path_info_munge(self._fpath))
#path database refers to, rather then just the file system mount path
sam_qfs_path = os.path.join(self._sam_qfs_prefix,
path_info_munge(self._fpath))
self._file = ExtendedHmsSideband(filename, mode, sam_qfs_path)
return self
except Exception as ex:
err_str = "Can't open HMS Sideband file with error: " + str(ex)
raise ArchiveInterfaceError(err_str)
def ping_core(self):
"""Ping the Core server to see if its still active"""
#Define acceptable latency in seconds
acceptable_latency = self._accept_latency
latency_tuple = _hpssExtensions.hpss_ping_core()
# Get the latency
latency = self.parse_latency(latency_tuple)
if latency > acceptable_latency:
err_str = "The archive core server is slow to respond"\
" Latency is: " + str(latency) + " second(s)"
raise ArchiveInterfaceError(err_str)
def stage(self):
"""
Stage an hpss file so that it moves to disk
doesnt need to return anything. will throw
exception on error however
"""
try:
_hpssExtensions.hpss_stage(self._filepath)
except Exception as ex:
#Push the excpetion up the chain to the response
err_str = "Error using c extension for hpss stage"\
" exception: " + str(ex)
raise ArchiveInterfaceError(err_str)
def status(self):
"""
Get the status of a file if it is on tape or disk
Found the documentation for this in the hpss programmers reference
section 2.3.6.2.8 "Get Extanded Attributes"
"""
mtime = ""
ctime = ""
bytes_per_level = ""
filesize = ""
try:
mtime = _hpssExtensions.hpss_mtime(self._filepath)
ctime = _hpssExtensions.hpss_ctime(self._filepath)
bytes_per_level = _hpssExtensions.hpss_status(self._filepath)
filesize = _hpssExtensions.hpss_filesize(self._filepath)
status = HpssStatus(mtime, ctime, bytes_per_level, filesize)
status.set_filepath(self._filepath)
except Exception as ex:
#Push the excpetion up the chain to the response
err_str = "Error using c extensions for hpss status"\
" exception: " + str(ex)
raise ArchiveInterfaceError(err_str)
return status
def close_error():
raise ArchiveInterfaceError("this is an error")