def begin(self, background=False):
if self.session == None:
raise SessionNotSetException()
if self._started:
raise SiteCheckStartedException()
self._started = True
# Start output thread
self.report_thread = ReportThread(self)
self.report_thread.setDaemon(True)
self.report_thread.start()
# Initialise modules
self.session.modules = [
m for m in self.session.modules if self._initialise_module(m)
]
# Create worker thread pool
for i in range(self.session.thread_pool):
thread = Checker(self)
thread.setDaemon(True)
thread.start()
self._threads.append(thread)
# Unless a module has added a start page
if self.request_queue.empty():
self._begin()
if not background:
self._wait()
self.end()
def begin(self, background=False): if self.session == None: raise SessionNotSetException() if self._started: raise SiteCheckStartedException() self._started = True # Start output thread self.report_thread = ReportThread(self) self.report_thread.setDaemon(True) self.report_thread.start() # Initialise modules self.session.modules = [m for m in self.session.modules if self._initialise_module(m)] # Create worker thread pool for i in range(self.session.thread_pool): thread = Checker(self) thread.setDaemon(True) thread.start() self._threads.append(thread) # Unless a module has added a start page if self.request_queue.empty(): self._begin() if not background: self._wait() self.end()
class SiteCheck(object):
def __init__(self, settings):
self.output_queue = OutputQueue()
self.request_queue = None
self._started = False
self._threads = []
self.sleep_time = 5
if type(settings) == tuple:
self._resume_data = settings
self._set_session(self._resume_data[0])
if hasattr(self.session, '_cookie'):
del self.session._cookie
else:
self._resume_data = None
self._set_session(settings)
def _set_session(self, session):
if self._started:
raise SiteCheckStartedException()
self.session = session
self.request_queue = RequestQueue(session)
if not hasattr(self.session, '_debug'):
self.session._debug = False
if len(os.path.splitext(self.session.domain)[1]) == 0:
self.session.domain = append(self.session.domain, '/')
if not re.match('^http', self.session.domain, re.IGNORECASE):
self.session.domain = 'http://{0}'.format(self.session.domain)
if len(urllib.parse.urlparse(self.session.domain).netloc) == 0:
raise Exception('Invalid domain')
# Organise file type sets
self.session.include_ext = self.session.include_ext.difference(self.session.ignore_ext)
self.session.test_ext = self.session.test_ext.difference(self.session.ignore_ext.union(self.session.include_ext))
def _initialise_module(self, module):
if not hasattr(module, 'source'):
self.output_queue.put_error('Module {0} has no source'.format(module.__class__.__name__))
return False
if not hasattr(module, 'name'):
self.output_queue.put_error('Module {0} has no name'.format(module.__class__.__name__), module.source)
return False
if not hasattr(module, 'initialise'):
self.output_queue.put_error('Initialise method not defined in module {0}'.format(module.name), module.source)
return False
if not hasattr(module, 'process'):
self.output_queue.put_error('Process method not defined in module {0}'.format(module.name), module.source)
return False
try:
module.initialise(self)
except:
if self.session._debug:
raise
self.output_queue.put_error('{0} in module {1}'.format(str(sys.exc_info()[1]), module.name), module.source)
return False
else:
return True
@property
def complete(self):
if self.session == None:
raise SessionNotSetException()
if not self._started:
return False
cmpl = False
if self.request_queue.empty():
cmpl = True
for t in self._threads:
if t.active:
cmpl = False
return cmpl
@property
def started(self):
return self._started
def begin(self, background=False):
if self.session == None:
raise SessionNotSetException()
if self._started:
raise SiteCheckStartedException()
self._started = True
# Start output thread
self.report_thread = ReportThread(self)
self.report_thread.setDaemon(True)
self.report_thread.start()
# Initialise modules
self.session.modules = [m for m in self.session.modules if self._initialise_module(m)]
# Create worker thread pool
for i in range(self.session.thread_pool):
thread = Checker(self)
thread.setDaemon(True)
thread.start()
self._threads.append(thread)
# Unless a module has added a start page
if self.request_queue.empty():
self._begin()
if not background:
self._wait()
self.end()
def _begin(self):
if self._resume_data:
for module in self.session.modules:
if hasattr(module, 'resume'):
try:
module.resume()
except:
if self.session._debug:
raise
self.output_queue.put_error('{0} in module {1}'.format(str(sys.exc_info()[1]), module.name), module.source)
self.session.modules.remove(module)
self.request_queue.load(self._resume_data[1], self._resume_data[2], self._resume_data[3])
del self._resume_data
else:
self.request_queue.requests = set() # Clear authentication requests
for module in self.session.modules:
if hasattr(module, 'begin'):
try:
module.begin()
except:
if self.session._debug:
raise
self.output_queue.put_error('{0} in module {1}'.format(str(sys.exc_info()[1]), module.name), module.source)
self.session.modules.remove(module)
self.request_queue.put_url('', self.session.page, self.session.domain)
def _wait(self):
while True:
if self.complete:
break
else:
time.sleep(self.sleep_time)
def end(self):
if self._started:
if self.session == None:
raise SessionNotSetException()
if self.complete:
for mod in self.session.modules:
if hasattr(mod, 'end'):
try:
mod.end()
except:
if self.session._debug:
raise
self.output_queue.put_error('{0} in module {1}'.format(str(sys.exc_info()[1]), mod.name), mod.source)
self._wait()
for mod in self.session.modules:
if hasattr(mod, 'complete'):
try:
mod.complete()
except:
if self.session._debug:
raise
self.output_queue.put_error('{0} in module {1}'.format(str(sys.exc_info()[1]), mod.name), mod.source)
# Wait for worker threads to complete
Checker.terminate.set()
for thread in self._threads:
thread.join()
if self.complete:
self.output_queue.put_message('Total URLs: {0}'.format(str(len(self.request_queue.urls))))
# Wait for log entries to be written
self.report_thread.end()
self.report_thread.join()
def suspend(self):
if self.session == None:
raise SessionNotSetException()
dat = self.request_queue.save()
return (self.session, dat[0], dat[1])
class SiteCheck(object):
def __init__(self, settings):
self.output_queue = OutputQueue()
self.request_queue = None
self._started = False
self._threads = []
self.sleep_time = 5
if type(settings) == tuple:
self._resume_data = settings
self._set_session(self._resume_data[0])
if hasattr(self.session, '_cookie'):
del self.session._cookie
else:
self._resume_data = None
self._set_session(settings)
def _set_session(self, session):
if self._started:
raise SiteCheckStartedException()
self.session = session
self.request_queue = RequestQueue(session)
if not hasattr(self.session, '_debug'):
self.session._debug = False
if len(os.path.splitext(self.session.domain)[1]) == 0:
self.session.domain = append(self.session.domain, '/')
if not re.match('^http', self.session.domain, re.IGNORECASE):
self.session.domain = 'http://{0}'.format(self.session.domain)
if len(urllib.parse.urlparse(self.session.domain).netloc) == 0:
raise Exception('Invalid domain')
# Organise file type sets
self.session.include_ext = self.session.include_ext.difference(
self.session.ignore_ext)
self.session.test_ext = self.session.test_ext.difference(
self.session.ignore_ext.union(self.session.include_ext))
def _initialise_module(self, module):
if not hasattr(module, 'source'):
self.output_queue.put_error('Module {0} has no source'.format(
module.__class__.__name__))
return False
if not hasattr(module, 'name'):
self.output_queue.put_error(
'Module {0} has no name'.format(module.__class__.__name__),
module.source)
return False
if not hasattr(module, 'initialise'):
self.output_queue.put_error(
'Initialise method not defined in module {0}'.format(
module.name), module.source)
return False
if not hasattr(module, 'process'):
self.output_queue.put_error(
'Process method not defined in module {0}'.format(module.name),
module.source)
return False
try:
module.initialise(self)
except:
if self.session._debug:
raise
self.output_queue.put_error(
'{0} in module {1}'.format(str(sys.exc_info()[1]),
module.name), module.source)
return False
else:
return True
@property
def complete(self):
if self.session == None:
raise SessionNotSetException()
if not self._started:
return False
cmpl = False
if self.request_queue.empty():
cmpl = True
for t in self._threads:
if t.active:
cmpl = False
return cmpl
@property
def started(self):
return self._started
def begin(self, background=False):
if self.session == None:
raise SessionNotSetException()
if self._started:
raise SiteCheckStartedException()
self._started = True
# Start output thread
self.report_thread = ReportThread(self)
self.report_thread.setDaemon(True)
self.report_thread.start()
# Initialise modules
self.session.modules = [
m for m in self.session.modules if self._initialise_module(m)
]
# Create worker thread pool
for i in range(self.session.thread_pool):
thread = Checker(self)
thread.setDaemon(True)
thread.start()
self._threads.append(thread)
# Unless a module has added a start page
if self.request_queue.empty():
self._begin()
if not background:
self._wait()
self.end()
def _begin(self):
if self._resume_data:
for module in self.session.modules:
if hasattr(module, 'resume'):
try:
module.resume()
except:
if self.session._debug:
raise
self.output_queue.put_error(
'{0} in module {1}'.format(str(sys.exc_info()[1]),
module.name),
module.source)
self.session.modules.remove(module)
self.request_queue.load(self._resume_data[1], self._resume_data[2],
self._resume_data[3])
del self._resume_data
else:
self.request_queue.requests = set(
) # Clear authentication requests
for module in self.session.modules:
if hasattr(module, 'begin'):
try:
module.begin()
except:
if self.session._debug:
raise
self.output_queue.put_error(
'{0} in module {1}'.format(str(sys.exc_info()[1]),
module.name),
module.source)
self.session.modules.remove(module)
self.request_queue.put_url('', self.session.page,
self.session.domain)
def _wait(self):
while True:
if self.complete:
break
else:
time.sleep(self.sleep_time)
def end(self):
if self._started:
if self.session == None:
raise SessionNotSetException()
if self.complete:
for mod in self.session.modules:
if hasattr(mod, 'end'):
try:
mod.end()
except:
if self.session._debug:
raise
self.output_queue.put_error(
'{0} in module {1}'.format(
str(sys.exc_info()[1]), mod.name),
mod.source)
self._wait()
for mod in self.session.modules:
if hasattr(mod, 'complete'):
try:
mod.complete()
except:
if self.session._debug:
raise
self.output_queue.put_error(
'{0} in module {1}'.format(
str(sys.exc_info()[1]), mod.name),
mod.source)
# Wait for worker threads to complete
Checker.terminate.set()
for thread in self._threads:
thread.join()
if self.complete:
self.output_queue.put_message('Total URLs: {0}'.format(
str(len(self.request_queue.urls))))
# Wait for log entries to be written
self.report_thread.end()
self.report_thread.join()
def suspend(self):
if self.session == None:
raise SessionNotSetException()
dat = self.request_queue.save()
return (self.session, dat[0], dat[1])