コード例 #1
0
ファイル: core.py プロジェクト: cash2one/sitecheck
    def begin(self, background=False):
        if self.session == None:
            raise SessionNotSetException()
        if self._started:
            raise SiteCheckStartedException()

        self._started = True

        # Start output thread
        self.report_thread = ReportThread(self)
        self.report_thread.setDaemon(True)
        self.report_thread.start()

        # Initialise modules
        self.session.modules = [
            m for m in self.session.modules if self._initialise_module(m)
        ]

        # Create worker thread pool
        for i in range(self.session.thread_pool):
            thread = Checker(self)
            thread.setDaemon(True)
            thread.start()
            self._threads.append(thread)

        # Unless a module has added a start page
        if self.request_queue.empty():
            self._begin()

        if not background:
            self._wait()
            self.end()
コード例 #2
0
ファイル: core.py プロジェクト: ITCorporation/sitecheck
	def begin(self, background=False):
		if self.session == None:
			raise SessionNotSetException()
		if self._started:
			raise SiteCheckStartedException()

		self._started = True

		# Start output thread
		self.report_thread = ReportThread(self)
		self.report_thread.setDaemon(True)
		self.report_thread.start()

		# Initialise modules
		self.session.modules = [m for m in self.session.modules if self._initialise_module(m)]

		# Create worker thread pool
		for i in range(self.session.thread_pool):
			thread = Checker(self)
			thread.setDaemon(True)
			thread.start()
			self._threads.append(thread)

		# Unless a module has added a start page
		if self.request_queue.empty():
			self._begin()

		if not background:
			self._wait()
			self.end()
コード例 #3
0
ファイル: core.py プロジェクト: ITCorporation/sitecheck
class SiteCheck(object):
	def __init__(self, settings):
		self.output_queue = OutputQueue()
		self.request_queue = None
		self._started = False
		self._threads = []
		self.sleep_time = 5

		if type(settings) == tuple:
			self._resume_data = settings
			self._set_session(self._resume_data[0])

			if hasattr(self.session, '_cookie'):
				del self.session._cookie
		else:
			self._resume_data = None
			self._set_session(settings)

	def _set_session(self, session):
		if self._started:
			raise SiteCheckStartedException()

		self.session = session
		self.request_queue = RequestQueue(session)

		if not hasattr(self.session, '_debug'):
			self.session._debug = False

		if len(os.path.splitext(self.session.domain)[1]) == 0:
			self.session.domain = append(self.session.domain, '/')

		if not re.match('^http', self.session.domain, re.IGNORECASE):
			self.session.domain = 'http://{0}'.format(self.session.domain)

		if len(urllib.parse.urlparse(self.session.domain).netloc) == 0:
			raise Exception('Invalid domain')

		# Organise file type sets
		self.session.include_ext = self.session.include_ext.difference(self.session.ignore_ext)
		self.session.test_ext = self.session.test_ext.difference(self.session.ignore_ext.union(self.session.include_ext))

	def _initialise_module(self, module):
		if not hasattr(module, 'source'):
			self.output_queue.put_error('Module {0} has no source'.format(module.__class__.__name__))
			return False

		if not hasattr(module, 'name'):
			self.output_queue.put_error('Module {0} has no name'.format(module.__class__.__name__), module.source)
			return False

		if not hasattr(module, 'initialise'):
			self.output_queue.put_error('Initialise method not defined in module {0}'.format(module.name), module.source)
			return False

		if not hasattr(module, 'process'):
			self.output_queue.put_error('Process method not defined in module {0}'.format(module.name), module.source)
			return False

		try:
			module.initialise(self)
		except:
			if self.session._debug:
				raise
			self.output_queue.put_error('{0} in module {1}'.format(str(sys.exc_info()[1]), module.name), module.source)
			return False
		else:
			return True

	@property
	def complete(self):
		if self.session == None:
			raise SessionNotSetException()

		if not self._started:
			return False

		cmpl = False
		if self.request_queue.empty():
			cmpl = True
			for t in self._threads:
				if t.active:
					cmpl = False

		return cmpl

	@property
	def started(self):
		return self._started

	def begin(self, background=False):
		if self.session == None:
			raise SessionNotSetException()
		if self._started:
			raise SiteCheckStartedException()

		self._started = True

		# Start output thread
		self.report_thread = ReportThread(self)
		self.report_thread.setDaemon(True)
		self.report_thread.start()

		# Initialise modules
		self.session.modules = [m for m in self.session.modules if self._initialise_module(m)]

		# Create worker thread pool
		for i in range(self.session.thread_pool):
			thread = Checker(self)
			thread.setDaemon(True)
			thread.start()
			self._threads.append(thread)

		# Unless a module has added a start page
		if self.request_queue.empty():
			self._begin()

		if not background:
			self._wait()
			self.end()

	def _begin(self):
		if self._resume_data:
			for module in self.session.modules:
				if hasattr(module, 'resume'):
					try:
						module.resume()
					except:
						if self.session._debug:
							raise
						self.output_queue.put_error('{0} in module {1}'.format(str(sys.exc_info()[1]), module.name), module.source)
						self.session.modules.remove(module)
			self.request_queue.load(self._resume_data[1], self._resume_data[2], self._resume_data[3])
			del self._resume_data
		else:
			self.request_queue.requests = set() # Clear authentication requests
			for module in self.session.modules:
				if hasattr(module, 'begin'):
					try:
						module.begin()
					except:
						if self.session._debug:
							raise
						self.output_queue.put_error('{0} in module {1}'.format(str(sys.exc_info()[1]), module.name), module.source)
						self.session.modules.remove(module)
			self.request_queue.put_url('', self.session.page, self.session.domain)

	def _wait(self):
		while True:
			if self.complete:
				break
			else:
				time.sleep(self.sleep_time)

	def end(self):
		if self._started:
			if self.session == None:
				raise SessionNotSetException()

			if self.complete:
				for mod in self.session.modules:
					if hasattr(mod, 'end'):
						try:
							mod.end()
						except:
							if self.session._debug:
								raise
							self.output_queue.put_error('{0} in module {1}'.format(str(sys.exc_info()[1]), mod.name), mod.source)

				self._wait()

				for mod in self.session.modules:
					if hasattr(mod, 'complete'):
						try:
							mod.complete()
						except:
							if self.session._debug:
								raise
							self.output_queue.put_error('{0} in module {1}'.format(str(sys.exc_info()[1]), mod.name), mod.source)

			# Wait for worker threads to complete
			Checker.terminate.set()
			for thread in self._threads:
				thread.join()

			if self.complete:
				self.output_queue.put_message('Total URLs: {0}'.format(str(len(self.request_queue.urls))))

			# Wait for log entries to be written
			self.report_thread.end()
			self.report_thread.join()

	def suspend(self):
		if self.session == None:
			raise SessionNotSetException()

		dat = self.request_queue.save()

		return (self.session, dat[0], dat[1])
コード例 #4
0
ファイル: core.py プロジェクト: cash2one/sitecheck
class SiteCheck(object):
    def __init__(self, settings):
        self.output_queue = OutputQueue()
        self.request_queue = None
        self._started = False
        self._threads = []
        self.sleep_time = 5

        if type(settings) == tuple:
            self._resume_data = settings
            self._set_session(self._resume_data[0])

            if hasattr(self.session, '_cookie'):
                del self.session._cookie
        else:
            self._resume_data = None
            self._set_session(settings)

    def _set_session(self, session):
        if self._started:
            raise SiteCheckStartedException()

        self.session = session
        self.request_queue = RequestQueue(session)

        if not hasattr(self.session, '_debug'):
            self.session._debug = False

        if len(os.path.splitext(self.session.domain)[1]) == 0:
            self.session.domain = append(self.session.domain, '/')

        if not re.match('^http', self.session.domain, re.IGNORECASE):
            self.session.domain = 'http://{0}'.format(self.session.domain)

        if len(urllib.parse.urlparse(self.session.domain).netloc) == 0:
            raise Exception('Invalid domain')

        # Organise file type sets
        self.session.include_ext = self.session.include_ext.difference(
            self.session.ignore_ext)
        self.session.test_ext = self.session.test_ext.difference(
            self.session.ignore_ext.union(self.session.include_ext))

    def _initialise_module(self, module):
        if not hasattr(module, 'source'):
            self.output_queue.put_error('Module {0} has no source'.format(
                module.__class__.__name__))
            return False

        if not hasattr(module, 'name'):
            self.output_queue.put_error(
                'Module {0} has no name'.format(module.__class__.__name__),
                module.source)
            return False

        if not hasattr(module, 'initialise'):
            self.output_queue.put_error(
                'Initialise method not defined in module {0}'.format(
                    module.name), module.source)
            return False

        if not hasattr(module, 'process'):
            self.output_queue.put_error(
                'Process method not defined in module {0}'.format(module.name),
                module.source)
            return False

        try:
            module.initialise(self)
        except:
            if self.session._debug:
                raise
            self.output_queue.put_error(
                '{0} in module {1}'.format(str(sys.exc_info()[1]),
                                           module.name), module.source)
            return False
        else:
            return True

    @property
    def complete(self):
        if self.session == None:
            raise SessionNotSetException()

        if not self._started:
            return False

        cmpl = False
        if self.request_queue.empty():
            cmpl = True
            for t in self._threads:
                if t.active:
                    cmpl = False

        return cmpl

    @property
    def started(self):
        return self._started

    def begin(self, background=False):
        if self.session == None:
            raise SessionNotSetException()
        if self._started:
            raise SiteCheckStartedException()

        self._started = True

        # Start output thread
        self.report_thread = ReportThread(self)
        self.report_thread.setDaemon(True)
        self.report_thread.start()

        # Initialise modules
        self.session.modules = [
            m for m in self.session.modules if self._initialise_module(m)
        ]

        # Create worker thread pool
        for i in range(self.session.thread_pool):
            thread = Checker(self)
            thread.setDaemon(True)
            thread.start()
            self._threads.append(thread)

        # Unless a module has added a start page
        if self.request_queue.empty():
            self._begin()

        if not background:
            self._wait()
            self.end()

    def _begin(self):
        if self._resume_data:
            for module in self.session.modules:
                if hasattr(module, 'resume'):
                    try:
                        module.resume()
                    except:
                        if self.session._debug:
                            raise
                        self.output_queue.put_error(
                            '{0} in module {1}'.format(str(sys.exc_info()[1]),
                                                       module.name),
                            module.source)
                        self.session.modules.remove(module)
            self.request_queue.load(self._resume_data[1], self._resume_data[2],
                                    self._resume_data[3])
            del self._resume_data
        else:
            self.request_queue.requests = set(
            )  # Clear authentication requests
            for module in self.session.modules:
                if hasattr(module, 'begin'):
                    try:
                        module.begin()
                    except:
                        if self.session._debug:
                            raise
                        self.output_queue.put_error(
                            '{0} in module {1}'.format(str(sys.exc_info()[1]),
                                                       module.name),
                            module.source)
                        self.session.modules.remove(module)
            self.request_queue.put_url('', self.session.page,
                                       self.session.domain)

    def _wait(self):
        while True:
            if self.complete:
                break
            else:
                time.sleep(self.sleep_time)

    def end(self):
        if self._started:
            if self.session == None:
                raise SessionNotSetException()

            if self.complete:
                for mod in self.session.modules:
                    if hasattr(mod, 'end'):
                        try:
                            mod.end()
                        except:
                            if self.session._debug:
                                raise
                            self.output_queue.put_error(
                                '{0} in module {1}'.format(
                                    str(sys.exc_info()[1]), mod.name),
                                mod.source)

                self._wait()

                for mod in self.session.modules:
                    if hasattr(mod, 'complete'):
                        try:
                            mod.complete()
                        except:
                            if self.session._debug:
                                raise
                            self.output_queue.put_error(
                                '{0} in module {1}'.format(
                                    str(sys.exc_info()[1]), mod.name),
                                mod.source)

            # Wait for worker threads to complete
            Checker.terminate.set()
            for thread in self._threads:
                thread.join()

            if self.complete:
                self.output_queue.put_message('Total URLs: {0}'.format(
                    str(len(self.request_queue.urls))))

            # Wait for log entries to be written
            self.report_thread.end()
            self.report_thread.join()

    def suspend(self):
        if self.session == None:
            raise SessionNotSetException()

        dat = self.request_queue.save()

        return (self.session, dat[0], dat[1])