def invoke(self): """ Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file """ if 'debexpo.upload.incoming' not in self.config or not os.path.isdir(self.config['debexpo.upload.incoming']): self.log.critical("debexpo.upload.incoming was not configured") return # 1) Process uploads for file in glob.glob( os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), '*.changes') ): try: changes = Changes(filename=file) except: self.log.error('Invalid changes file: %s' % (file)) continue try: for filename in changes.get_files() + [ changes.get_filename(), ]: source_file = os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), filename) destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename) if not os.path.exists(source_file): self.log.debug("Source file %s does not exist - putting upload on hold" % (source_file)) raise NotCompleteUpload; if os.path.exists(destination_file): self.log.debug("Do not import %s: already exists on destination path - removing file instead" % (source_file)) os.remove(source_file) raise NotCompleteUpload; shutil.move(source_file, self.config['debexpo.upload.incoming']) except NotCompleteUpload: continue self.log.info("Import upload: %s" % (file)) importer = Importer(changes.get_filename(), self.config['global_conf']['__file__'], False, False) returncode = importer.main() if returncode != 0: self.log.critical("Importer failed to import package %s [err=%d]." % (file, returncode)) for filename in changes.get_files() + [ changes.get_filename(), ]: destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename) if os.path.exists(destination_file): self.log.debug("Remove stale file %s - the importer probably crashed" % (destination_file)) os.remove(destination_file) # 2) Mark unprocessed files and get rid of them after some time pub = os.path.join(self.config['debexpo.upload.incoming'], "pub") for file in glob.glob( os.path.join(pub, '*') ): if self.files.allowed_upload(file): self.log.debug("Incomplete upload: %s" % (file)) last_change = time.time() - os.stat(file).st_mtime # the file was uploaded more than 6 hours ago if last_change > 6 * 60 * 60: self.log.warning("Remove old file: %s (last modified %.2f hours ago)" % (file, last_change / 3600.)) os.remove(file) else: if os.path.isfile(file): self.log.warning("Remove unknown file: %s" % (file)) os.remove(file)
def invoke(self): """ Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file """ if 'debexpo.upload.incoming' not in self.config or not os.path.isdir( self.config['debexpo.upload.incoming']): self.log.critical("debexpo.upload.incoming was not configured") return # 1) Process uploads for file in glob.glob( os.path.join( os.path.join(self.config['debexpo.upload.incoming'], "pub"), '*.changes')): try: changes = Changes(filename=file) except: self.log.error('Invalid changes file: %s' % (file)) continue try: for filename in changes.get_files() + [ changes.get_filename(), ]: source_file = os.path.join( os.path.join(self.config['debexpo.upload.incoming'], "pub"), filename) destination_file = os.path.join( self.config['debexpo.upload.incoming'], filename) if not os.path.exists(source_file): self.log.debug( "Source file %s does not exist - putting upload on hold" % (source_file)) raise NotCompleteUpload if os.path.exists(destination_file): self.log.debug( "Do not import %s: already exists on destination path - removing file instead" % (source_file)) os.remove(source_file) raise NotCompleteUpload shutil.move(source_file, self.config['debexpo.upload.incoming']) except NotCompleteUpload: continue self.log.info("Import upload: %s" % (file)) importer = Importer(changes.get_filename(), self.config['global_conf']['__file__'], False, False) returncode = importer.main() if returncode != 0: self.log.critical( "Importer failed to import package %s [err=%d]." % (file, returncode)) for filename in changes.get_files() + [ changes.get_filename(), ]: destination_file = os.path.join( self.config['debexpo.upload.incoming'], filename) if os.path.exists(destination_file): self.log.debug( "Remove stale file %s - the importer probably crashed" % (destination_file)) os.remove(destination_file) # 2) Mark unprocessed files and get rid of them after some time pub = os.path.join(self.config['debexpo.upload.incoming'], "pub") for file in glob.glob(os.path.join(pub, '*')): if self.files.allowed_upload(file): self.log.debug("Incomplete upload: %s" % (file)) last_change = time.time() - os.stat(file).st_mtime # the file was uploaded more than 6 hours ago if last_change > 6 * 60 * 60: self.log.warning( "Remove old file: %s (last modified %.2f hours ago)" % (file, last_change / 3600.)) os.remove(file) else: if os.path.isfile(file): self.log.warning("Remove unknown file: %s" % (file)) os.remove(file)
def invoke(self): """ Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file """ if 'debexpo.upload.incoming' not in self.config or not os.path.isdir(self.config['debexpo.upload.incoming']): self.log.critical("debexpo.upload.incoming was not configured") return # 1) Process uploads for file in glob.glob( os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), '*.changes') ): try: changes = Changes(filename=file) except: self.log.error('Invalid changes file: %s' % (file)) continue try: for filename in changes.get_files() + [ changes.get_filename(), ]: source_file = os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), filename) destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename) if not os.path.exists(source_file): self.log.debug("Source file %s does not exist - putting upload on hold" % (source_file)) raise NotCompleteUpload; if os.path.exists(destination_file): self.log.debug("Do not import %s: already exists on destination path - removing file instead" % (source_file)) os.remove(source_file) raise NotCompleteUpload; shutil.move(source_file, self.config['debexpo.upload.incoming']) except NotCompleteUpload: continue self.log.info("Import upload: %s" % (file)) command = [ self.config['debexpo.importer'], '-i', self.config['global_conf']['__file__'], '-c', changes.get_filename() ] self.log.debug("Executing: %s" % (" ".join(command))) proc = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (istdout, istderr) = proc.communicate() if proc.returncode != 0: self.log.critical("Importer failed to import package %s [err=%d]." % (file, proc.returncode)) self.log.debug("Output was\n%s\n%s" % (istdout,istderr)) for filename in changes.get_files() + [ changes.get_filename(), ]: destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename) if os.path.exists(destination_file): self.log.debug("Remove stale file %s - the importer probably crashed" % (destination_file)) os.remove(destination_file) # 2) Mark unprocessed files and get rid of them after some time pub = os.path.join(self.config['debexpo.upload.incoming'], "pub") filenames = [name for (name, _) in self.stale_files] file_to_check = [] for file in glob.glob( os.path.join(pub, '*') ): if self.files.allowed_upload(file): self.log.debug("Incomplete upload: %s" % (file)) if not file in filenames: self.stale_files.append((file,datetime.datetime.now())) else: file_to_check.append(file) else: if os.path.isfile(file): self.log.warning("Remove unknown file: %s" % (file)) os.remove(file) new_file_list = [] for file in file_to_check: for (file_known, last_check) in self.stale_files: if file == file_known and (datetime.datetime.now() - last_check) > datetime.timedelta(hours = 6): if os.path.isfile(file): self.log.warning("Remove incomplete upload: %s" % (file)) os.remove(file) continue new_file_list.append((file_known, last_check)) self.stale_files = new_file_list