def test_missing_files(self):
files = set(
File(f)
for f in ('dns.2014.Y.mtbl', 'dns.201501.M.mtbl',
'dns.20150201.W.mtbl', 'dns.20150208.D.mtbl',
'dns.20150209.0000.H.mtbl', 'dns.20150209.0100.X.mtbl',
'dns.20150209.0110.m.mtbl'))
missing = set(
File(f)
for f in ('dns.2012.Y.mtbl', 'dns.20130108.W.mtbl',
'dns.20130202.D.mtbl', 'dns.20130208.0100.H.mtbl',
'dns.20130209.0020.X.mtbl', 'dns.20130209.0109.m.mtbl'))
fs = Fileset(None, self.td)
fs.all_local_files = set(files)
fs.minimal_local_files = set(files)
fs.remote_files = files.union(missing)
self.assertItemsEqual(fs.missing_files(), missing)
class DNSTableManager:
def __init__(self,
fileset_uri,
destination,
base=None,
extension='mtbl',
frequency=1800,
download_timeout=None,
retry_timeout=60,
apikey=None,
validator=None,
digest_required=True,
minimal=True,
download_manager=None):
self.fileset_uri = fileset_uri
if not os.path.isdir(destination):
raise OSError(errno.ENOENT,
'Not a directory: \'{}\''.format(destination))
self.destination = destination
if base:
self.base = base
else:
self.base = os.path.splitext(
os.path.basename(urlparse.urlsplit(fileset_uri)[2]))[0]
self.extension = extension
self.frequency = frequency
self.download_timeout = download_timeout
self.retry_timeout = retry_timeout
self.minimal = minimal
self.fileset = Fileset(uri=self.fileset_uri,
dname=self.destination,
base=self.base,
extension=self.extension,
apikey=apikey,
validator=validator,
timeout=download_timeout,
digest_required=digest_required)
if download_manager:
self.download_manager = download_manager
else:
self.download_manager = DownloadManager(
download_timeout=download_timeout, retry_timeout=retry_timeout)
self.download_manager.start()
self.thread = None
def start(self):
if self.thread:
raise Exception
self.thread = threading.Thread(target=self.run)
self.thread.setDaemon(True)
self.thread.start()
def join(self):
if not self.thread:
raise Exception
self.thread.join()
self.thread = None
def run(self):
next_remote_load = 0
while True:
now = time.time()
self.fileset.load_local_fileset()
try:
if now >= next_remote_load:
self.fileset.load_remote_fileset()
next_remote_load = now + self.frequency
except (FilesetError, urllib2.URLError, urllib2.HTTPError,
httplib.HTTPException, socket.error) as e:
logger.error('Failed to load remote fileset {}: {}'.format(
self.fileset_uri, str(e)))
logger.debug(traceback.format_exc())
next_remote_load = now + self.retry_timeout
for f in sorted(self.fileset.missing_files(), reverse=True):
if f not in self.download_manager:
self.download_manager.enqueue(f)
self.fileset.prune_obsolete_files(minimal=self.minimal)
self.fileset.prune_redundant_files(minimal=self.minimal)
try:
self.fileset.write_local_fileset()
if not self.minimal:
self.fileset.write_local_fileset(minimal=False)
except (IOError, OSError) as e:
logger.error('Failed to write fileset {}: {}'.format(
self.fileset.get_fileset_name(), str(e)))
logger.debug(traceback.format_exc())
try:
self.fileset.purge_deleted_files()
except OSError as e:
logger.error('Failed to purge deleted files in {}: {}'.format(
self.destination, str(e)))
logger.debug(traceback.format_exc())
time.sleep(1)
def clean_tempfiles(self):
open_files = set()
for p in psutil.process_iter():
try:
try:
func = p.get_open_files
except AttributeError:
func = p.open_files
for f in func():
open_files.add(f.path)
except psutil.AccessDenied:
pass
for filename in self.fileset.list_temporary_files():
if filename in open_files:
logger.debug(
'Not unlinking tempfile {!r}: In use.'.format(filename))
continue
logger.debug('Unlinking tempfile: {!r}'.format(filename))
os.unlink(filename)