def gen_upload_files(self):
if not os.path.isdir(self.root):
return
for logname in listdir_by_creation(self.root):
path = os.path.join(self.root, logname)
try:
names = os.listdir(path)
except OSError:
continue
if any(name.endswith(".lock") for name in names):
continue
for name in sorted(names, key=self.get_upload_sort):
key = os.path.join(logname, name)
fn = os.path.join(path, name)
if name.endswith(".bz2"):
os.remove(fn)
# skip files already uploaded
try:
is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME)
except OSError:
cloudlog.event("uploader_getxattr_failed",
exc=self.last_exc,
key=key,
fn=fn)
is_uploaded = True # deleter could have deleted
if is_uploaded:
continue
yield (name, key, fn)
def get_logs_to_send_sorted():
# TODO: scan once then use inotify to detect file creation/deletion
curr_time = int(time.time())
logs = []
for log_entry in os.listdir(SWAGLOG_DIR):
log_path = os.path.join(SWAGLOG_DIR, log_entry)
try:
time_sent = int.from_bytes(getxattr(log_path, LOG_ATTR_NAME), sys.byteorder)
except (ValueError, TypeError):
time_sent = 0
# assume send failed and we lost the response if sent more than one hour ago
if not time_sent or curr_time - time_sent > 3600:
logs.append(log_entry)
# excluding most recent (active) log file
return sorted(logs)[:-1]
def list_upload_files(self):
if not os.path.isdir(self.root):
return
self.raw_size = 0
self.raw_count = 0
self.immediate_size = 0
self.immediate_count = 0
for logname in listdir_by_creation(self.root):
path = os.path.join(self.root, logname)
try:
names = os.listdir(path)
except OSError:
continue
if any(name.endswith(".lock") for name in names):
continue
for name in sorted(names, key=self.get_upload_sort):
key = os.path.join(logname, name)
fn = os.path.join(path, name)
# skip files already uploaded
try:
is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME)
except OSError:
cloudlog.event("uploader_getxattr_failed",
exc=self.last_exc,
key=key,
fn=fn)
is_uploaded = True # deleter could have deleted
if is_uploaded:
continue
try:
if name in self.immediate_priority:
self.immediate_count += 1
self.immediate_size += os.path.getsize(fn)
else:
self.raw_count += 1
self.raw_size += os.path.getsize(fn)
except OSError:
pass
yield (name, key, fn)
def get_is_uploaded(filename):
return getxattr(filename, UPLOAD_ATTR_NAME) is not None
def _get_is_uploaded(self, filename):
_debug("%s is uploaded: %s" %
(filename, getxattr(filename, UPLOAD_ATTR_NAME) is not None))
return getxattr(filename, UPLOAD_ATTR_NAME) is not None
