def fs_entry_list_creation(self):
# Function to create entry list for results in a file-system
# (archive or backup) directory.
tarlist = glob.iglob(os.path.join(self.dirname, "*", "*.tar.xz"))
self.content_list = []
self.missing_list = []
self.error_list = []
for tar in tarlist:
tar = Path(tar)
result_name = tar.name
controller = tar.parent.name
md5_file = f"{tar}.md5"
try:
with open(md5_file) as k:
md5 = k.readline().split(" ")[0]
except OSError as ex:
if ex.errno == errno.ENOENT:
self.missing_list.append(tar)
else:
self.error_list.append(tar)
self.logger.exception("ERROR reading file {}", md5_file)
continue
except Exception:
self.error_list.append(tar)
self.logger.exception("ERROR reading file {}", md5_file)
continue
else:
self.content_list.append(
Entry(os.path.join(controller, result_name), md5))
return Status.SUCCESS
def s3_entry_list_creation(self):
self.content_list = []
kwargs = {"Bucket": self.s3_config_obj.bucket_name}
try:
# We loop because the objects are returned a "page" at a
# time and all the pages except the last return a
# continuation token.
done = False
while not done:
resp = self.s3_config_obj.list_objects(**kwargs)
for obj in resp["Contents"]:
md5_returned = self.s3_config_obj.header_md5(obj)
self.content_list.append(Entry(obj["Key"], md5_returned))
try:
kwargs["ContinuationToken"] = resp["NextContinuationToken"]
except KeyError:
# that was the last page
done = True
self.logger.debug(
"list_objects: got {} objects{}",
len(resp["Contents"]),
" - continuing..." if not done else "",
)
except Exception:
self.logger.exception("ERROR fetching list of objects from S3")
return Status.FAIL
else:
return Status.SUCCESS