def invoke(self):
"""
Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file
"""
if 'debexpo.upload.incoming' not in self.config or not os.path.isdir(self.config['debexpo.upload.incoming']):
self.log.critical("debexpo.upload.incoming was not configured")
return
# 1) Process uploads
for file in glob.glob( os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), '*.changes') ):
try:
changes = Changes(filename=file)
except:
self.log.error('Invalid changes file: %s' % (file))
continue
try:
for filename in changes.get_files() + [ changes.get_filename(), ]:
source_file = os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), filename)
destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)
if not os.path.exists(source_file):
self.log.debug("Source file %s does not exist - putting upload on hold" % (source_file))
raise NotCompleteUpload;
if os.path.exists(destination_file):
self.log.debug("Do not import %s: already exists on destination path - removing file instead" % (source_file))
os.remove(source_file)
raise NotCompleteUpload;
shutil.move(source_file, self.config['debexpo.upload.incoming'])
except NotCompleteUpload:
continue
self.log.info("Import upload: %s" % (file))
importer = Importer(changes.get_filename(),
self.config['global_conf']['__file__'],
False,
False)
returncode = importer.main()
if returncode != 0:
self.log.critical("Importer failed to import package %s [err=%d]." % (file, returncode))
for filename in changes.get_files() + [ changes.get_filename(), ]:
destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)
if os.path.exists(destination_file):
self.log.debug("Remove stale file %s - the importer probably crashed" % (destination_file))
os.remove(destination_file)
# 2) Mark unprocessed files and get rid of them after some time
pub = os.path.join(self.config['debexpo.upload.incoming'], "pub")
for file in glob.glob( os.path.join(pub, '*') ):
if self.files.allowed_upload(file):
self.log.debug("Incomplete upload: %s" % (file))
last_change = time.time() - os.stat(file).st_mtime
# the file was uploaded more than 6 hours ago
if last_change > 6 * 60 * 60:
self.log.warning("Remove old file: %s (last modified %.2f hours ago)" % (file, last_change / 3600.))
os.remove(file)
else:
if os.path.isfile(file):
self.log.warning("Remove unknown file: %s" % (file))
os.remove(file)
def invoke(self):
"""
Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file
"""
if 'debexpo.upload.incoming' not in self.config or not os.path.isdir(
self.config['debexpo.upload.incoming']):
self.log.critical("debexpo.upload.incoming was not configured")
return
# 1) Process uploads
for file in glob.glob(
os.path.join(
os.path.join(self.config['debexpo.upload.incoming'],
"pub"), '*.changes')):
try:
changes = Changes(filename=file)
except:
self.log.error('Invalid changes file: %s' % (file))
continue
try:
for filename in changes.get_files() + [
changes.get_filename(),
]:
source_file = os.path.join(
os.path.join(self.config['debexpo.upload.incoming'],
"pub"), filename)
destination_file = os.path.join(
self.config['debexpo.upload.incoming'], filename)
if not os.path.exists(source_file):
self.log.debug(
"Source file %s does not exist - putting upload on hold"
% (source_file))
raise NotCompleteUpload
if os.path.exists(destination_file):
self.log.debug(
"Do not import %s: already exists on destination path - removing file instead"
% (source_file))
os.remove(source_file)
raise NotCompleteUpload
shutil.move(source_file,
self.config['debexpo.upload.incoming'])
except NotCompleteUpload:
continue
self.log.info("Import upload: %s" % (file))
importer = Importer(changes.get_filename(),
self.config['global_conf']['__file__'], False,
False)
returncode = importer.main()
if returncode != 0:
self.log.critical(
"Importer failed to import package %s [err=%d]." %
(file, returncode))
for filename in changes.get_files() + [
changes.get_filename(),
]:
destination_file = os.path.join(
self.config['debexpo.upload.incoming'], filename)
if os.path.exists(destination_file):
self.log.debug(
"Remove stale file %s - the importer probably crashed"
% (destination_file))
os.remove(destination_file)
# 2) Mark unprocessed files and get rid of them after some time
pub = os.path.join(self.config['debexpo.upload.incoming'], "pub")
for file in glob.glob(os.path.join(pub, '*')):
if self.files.allowed_upload(file):
self.log.debug("Incomplete upload: %s" % (file))
last_change = time.time() - os.stat(file).st_mtime
# the file was uploaded more than 6 hours ago
if last_change > 6 * 60 * 60:
self.log.warning(
"Remove old file: %s (last modified %.2f hours ago)" %
(file, last_change / 3600.))
os.remove(file)
else:
if os.path.isfile(file):
self.log.warning("Remove unknown file: %s" % (file))
os.remove(file)
def invoke(self):
"""
Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file
"""
if 'debexpo.upload.incoming' not in self.config or not os.path.isdir(self.config['debexpo.upload.incoming']):
self.log.critical("debexpo.upload.incoming was not configured")
return
# 1) Process uploads
for file in glob.glob( os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), '*.changes') ):
try:
changes = Changes(filename=file)
except:
self.log.error('Invalid changes file: %s' % (file))
continue
try:
for filename in changes.get_files() + [ changes.get_filename(), ]:
source_file = os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), filename)
destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)
if not os.path.exists(source_file):
self.log.debug("Source file %s does not exist - putting upload on hold" % (source_file))
raise NotCompleteUpload;
if os.path.exists(destination_file):
self.log.debug("Do not import %s: already exists on destination path - removing file instead" % (source_file))
os.remove(source_file)
raise NotCompleteUpload;
shutil.move(source_file, self.config['debexpo.upload.incoming'])
except NotCompleteUpload:
continue
self.log.info("Import upload: %s" % (file))
command = [ self.config['debexpo.importer'], '-i', self.config['global_conf']['__file__'], '-c', changes.get_filename() ]
self.log.debug("Executing: %s" % (" ".join(command)))
proc = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(istdout, istderr) = proc.communicate()
if proc.returncode != 0:
self.log.critical("Importer failed to import package %s [err=%d]." % (file, proc.returncode))
self.log.debug("Output was\n%s\n%s" % (istdout,istderr))
for filename in changes.get_files() + [ changes.get_filename(), ]:
destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)
if os.path.exists(destination_file):
self.log.debug("Remove stale file %s - the importer probably crashed" % (destination_file))
os.remove(destination_file)
# 2) Mark unprocessed files and get rid of them after some time
pub = os.path.join(self.config['debexpo.upload.incoming'], "pub")
filenames = [name for (name, _) in self.stale_files]
file_to_check = []
for file in glob.glob( os.path.join(pub, '*') ):
if self.files.allowed_upload(file):
self.log.debug("Incomplete upload: %s" % (file))
if not file in filenames:
self.stale_files.append((file,datetime.datetime.now()))
else:
file_to_check.append(file)
else:
if os.path.isfile(file):
self.log.warning("Remove unknown file: %s" % (file))
os.remove(file)
new_file_list = []
for file in file_to_check:
for (file_known, last_check) in self.stale_files:
if file == file_known and (datetime.datetime.now() - last_check) > datetime.timedelta(hours = 6):
if os.path.isfile(file):
self.log.warning("Remove incomplete upload: %s" % (file))
os.remove(file)
continue
new_file_list.append((file_known, last_check))
self.stale_files = new_file_list
