Esempio n. 1
0
        def run(self):
            req = None
            while True:

                self.scanner._th_lock.acquire()
                if self.exit == True or len(
                        self.scanner.pending_requests) == 0:
                    self.scanner._th_lock.release()
                    shutil.rmtree(self.tmp_dir)
                    return

                req = self.scanner.pending_requests.pop()

                self.scanner._th_lock.release()

                cmd_options = self.scanner.get_cmd(req, self.tmp_dir)
                if cmd_options == False:
                    self.inc_counter()
                    continue

                cmd = self.scanner.settings['scanner_exe'] + cmd_options

                exe = CommandExecutor(cmd, True)
                out, err = exe.execute(
                    self.scanner.settings['process_timeout'])
                # if err: print "\nError: \n%s\n%s\n%s\n" % (err," ".join(cmd),out)

                self.inc_counter()

                self.scanner.scanner_executed(req, out, err, self.tmp_dir, cmd)
Esempio n. 2
0
		def run(self):
			req = None
			while True:
				
				self.scanner._th_lock.acquire()
				if self.exit == True or len(self.scanner.pending_requests) == 0:
					self.scanner._th_lock.release()
					shutil.rmtree(self.tmp_dir)					
					return

				req = self.scanner.pending_requests.pop()
									
				self.scanner._th_lock.release()						

				
				cmd_options = self.scanner.get_cmd(req, self.tmp_dir)
				if cmd_options == False: 
					self.inc_counter()
					continue

				cmd = self.scanner.settings['scanner_exe'] + cmd_options
				

				exe = CommandExecutor(cmd, True)
				out, err = exe.execute(self.scanner.settings['process_timeout'])
				# if err: print "\nError: \n%s\n%s\n%s\n" % (err," ".join(cmd),out)
				
				self.inc_counter()

				self.scanner.scanner_executed(req, out,err, self.tmp_dir, cmd)
Esempio n. 3
0
    def test_command_timeout_with_errors(self, mock_comm):
        mock_comm.return_value = (None, "error")
        cmd = ['sleep', '10']
        executor = CommandExecutor(cmd, stderr=True)
        result = executor.execute(1)

        self.assertEquals("error", result[1])
Esempio n. 4
0
def execmd(cmd, params=None, timeout=None):
    """
	Execute a shell command

	Parameters
	----------
	cmd: string
		Path to executable or executable name (if in $PATH)
	params: array_like, optional
		List of command arguments
	timeout: int, optional
		Execution timeout in seconds

	Returns
	-------
	dict
		Dictionary with fields:
		 - out str: command output (stdout)
		 - err str: command error or error string if timeout
		 - returncode int: command return code

	Raises
	------

	"""
    if not re.search(r'^\.?/', cmd):
        cmd = get_cmd_path(cmd)
    if not cmd or not os.path.isfile(cmd):
        raise Exception("Command not found")
    cmd = [cmd] + params if params else [cmd]
    exe = CommandExecutor(cmd, stderr=True)
    out, err = exe.execute(timeout)
    return {"out": out, "err": err, "returncode": exe.returncode}
Esempio n. 5
0
    def test_command_responds(self, mock_process):
        mock_process.Popen.side_effect = time.sleep(1)
        mock_process.Popen().communicate.return_value = ('hurray', 'err')

        executor = CommandExecutor(['cmd'])
        result = executor.execute(2)

        self.assertEqual(result, "hurray")
Esempio n. 6
0
	def check_user_script_syntax(self, probe_cmd, user_script):
		try:
			exe = CommandExecutor(probe_cmd + ["-u", user_script, "-v"] , False)
			out = exe.execute(5)
			if out:
				print "\n* USER_SCRIPT error: %s" % out
				sys.exit(1)
			stdoutw(". ")
		except KeyboardInterrupt:
			print "\nAborted"
			sys.exit(0)
Esempio n. 7
0
 def check_user_script_syntax(self, probe_cmd, user_script):
     try:
         exe = CommandExecutor(probe_cmd + ["-u", user_script, "-v"], False)
         out = exe.execute(5)
         if out:
             print "\n* USER_SCRIPT error: %s" % out
             sys.exit(1)
         stdoutw(". ")
     except KeyboardInterrupt:
         print "\nAborted"
         sys.exit(0)
Esempio n. 8
0
    def scanner_executed(self, request, out, err, tmp_dir, cmd):
        out_file = tmp_dir + "/report"

        if not os.path.isfile(out_file):
            return

        json_file = tmp_dir + "/report.json"

        cmd = [
            self.reporter, "--reporter",
            "json:outfile=%s" % json_file, out_file
        ]
        exe = CommandExecutor(cmd, True)
        out, err = exe.execute(30)

        if err:
            print ">>> error exporting arachni to json: %s %s" % (err,
                                                                  request.url)
            return

        if not os.path.isfile(json_file):
            return

        with open(json_file, 'r') as fil:
            jsn = fil.read()

        report = []
        try:
            report = json.loads(jsn)
        except Exception as e:
            print err

        issues = report['issues']

        for i in issues:
            ref = i['references']['OWASP'] if i['references'] and 'OWASP' in i[
                'references'] else "N/A"
            req = "N/A"
            req = None

            if 'request' in i:
                req = i['request']
            elif 'variations' in i and len(i['variations']) > 0:
                req = i['variations'][0]['request']

            fields = (i['name'], ref, i['severity'],
                      req['headers_string'] if req else "N/A")
            descr = "D E T A I L S\n\nName:       %s\nReference:  %s\nSeverity:   %s\n\n\nR E Q U E S T\n\n%s" % fields

            if req and req['method'] == "post":
                descr += "%s" % urllib.urlencode(req['body'])

            self.save_vulnerability(request, i['check']['shortname'], descr)
Esempio n. 9
0
	def scanner_executed(self, request, out, err, tmp_dir, cmd):
		out_file = tmp_dir + "/report"

		if not os.path.isfile(out_file):
			return

		json_file = tmp_dir + "/report.json"

		cmd = [self.reporter, "--reporter", "json:outfile=%s" % json_file, out_file]
		exe = CommandExecutor(cmd, True)
		out, err = exe.execute(30)

		if err:
			print ">>> error exporting arachni to json: %s %s" % (err, request.url)
			return

		if not os.path.isfile(json_file):
			return 

		with open(json_file,'r') as fil:
			jsn = fil.read()		
		
		report = []
		try:
			report = json.loads(jsn)
		except Exception as e:
			print err

		issues = report['issues']
		
		for i in issues:	
			ref = i['references']['OWASP'] if i['references'] and 'OWASP' in i['references'] else "N/A"
			req = "N/A"
			req = None

			if 'request' in i:
				req = i['request']				
			elif 'variations' in i and len(i['variations']) > 0:
				req = i['variations'][0]['request']
				
	
			fields = (i['name'], ref, i['severity'], req['headers_string'] if req else "N/A")
			descr = "D E T A I L S\n\nName:       %s\nReference:  %s\nSeverity:   %s\n\n\nR E Q U E S T\n\n%s" % fields
			
			if req and req['method'] == "post":
				descr += "%s" % urllib.urlencode(req['body'])
			
			self.save_vulnerability(request, i['check']['shortname'], descr)
Esempio n. 10
0
    def execmd(self, cmd, params=None, timeout=None):
        if not re.search(r'^\.?/', cmd):
            cmd = get_cmd_path(cmd)
        if not cmd or not os.path.isfile(cmd):
            raise Exception("Command not found")
        cmd = [cmd] + params if params else [cmd]
        exe = CommandExecutor(cmd, stderr=True)
        self.scanner._th_lock.acquire()
        self.scanner._commands.append(exe)
        self.scanner._th_lock.release()

        out, err = exe.execute(timeout)
        ret = {"out": out, "err": err, "returncode": exe.returncode}

        self.scanner._th_lock.acquire()
        self.scanner._commands.remove(exe)
        self.scanner._th_lock.release()

        return ret
Esempio n. 11
0
    def _send_probe(self, request, errors):

        probe = None
        retries = CrawlerThread._PROCESS_RETRIES
        params = self._set_probe_params(request)

        while retries:
            # DEBUG:
            # print("### INPUT: %s" % repr(Shared.probe_cmd + params))
            cmd = CommandExecutor(Shared.probe_cmd + params)
            jsn = cmd.execute(Shared.options['process_timeout'] + 2)

            # DEBUG:
            # print("### OUTPUT: %s" % repr(jsn))

            if jsn is None:
                errors.append(ERROR_PROBEKILLED)
                sleep(CrawlerThread._PROCESS_RETRIES_INTERVAL)  # ... ???
                retries -= 1
                continue
            else:
                probe_array = self._load_probe_json(jsn)

            if probe_array:
                probe = Probe(probe_array, request)

                if probe.status == "ok":
                    break

                errors.append(probe.errcode)

                if probe.errcode in (ERROR_CONTENTTYPE, ERROR_PROBE_TO,
                                     ERROR_FORCE_STOP):
                    break

            sleep(CrawlerThread._PROCESS_RETRIES_INTERVAL)
            retries -= 1
        return probe
Esempio n. 12
0
	def send_probe(self, request,  errors):

		url = request.url
		jsn = None
		probe = None
		retries = self.process_retries
		params = []
		cookies = []


		if request.method == "POST":
			params.append("-P")
			if request.data:
				params.extend(("-D", request.data))


		if len(request.cookies) > 0:
			for cookie in request.cookies:
				cookies.append(cookie.get_dict())

			with open(self.cookie_file,'w') as fil:
				fil.write(json.dumps(cookies))

			params.extend(("-c", self.cookie_file))



		if request.http_auth:
			params.extend(("-p" ,request.http_auth))

		if Shared.options['set_referer'] and request.referer:
			params.extend(("-r", request.referer))


		params.extend(("-i", str(request.db_id)))

		params.append(url)


		while retries:
		#while False:

			# print cmd_to_str(Shared.probe_cmd + params)
			# print ""

			cmd = CommandExecutor(Shared.probe_cmd + params)
			jsn = cmd.execute(Shared.options['process_timeout'] + 2)

			if jsn == None:
				errors.append(ERROR_PROBEKILLED)
				time.sleep(self.process_retries_interval) # ... ???
				retries -= 1
				continue


			# try to decode json also after an exception .. sometimes phantom crashes BUT returns a valid json ..
			try:
				if jsn and type(jsn) is not str:
					jsn = jsn[0]
				probeArray = self.load_probe_json(jsn)
			except Exception as e:
				raise


			if probeArray:
				probe = Probe(probeArray, request)

				if probe.status == "ok":
					break

				errors.append(probe.errcode)

				if probe.errcode in (ERROR_CONTENTTYPE, ERROR_PROBE_TO):
					break

			time.sleep(self.process_retries_interval)
			retries -= 1

		return probe
Esempio n. 13
0
    def main(self, args, opts):
        passw = None
        format = None
        out_cookies = True
        out_logouts = True
        for o, v in opts:
            if o == "-h":
                print self.usage()
                sys.exit(0)
            elif o == "-p":
                passw = v
            elif o == "-c":
                out_cookies = False
            elif o == "-l":
                out_logouts = False
            elif o in ("-H", "-J", "-A"):
                format = o

        if not passw:
            print "The password is hidden here BUT it will be passed to phantomjs via commandline ..."
            try:
                passw = getpass.getpass()
            except KeyboardInterrupt:
                print "\nAbort..."
                sys.exit(0)

        jspath = "%s%s%s%s" % (getrealdir(__file__), "login", os.sep,
                               "login.js")
        cmd = get_phantomjs_cmd() + [jspath, args[0], args[1], passw]
        if len(args) > 2: cmd.append(args[2])
        #print cmd_to_str(cmd)
        exe = CommandExecutor(cmd, True)
        out, err = exe.execute(20)
        if err:
            print "Unable to login"
            sys.exit(1)

        try:
            ret = json.loads(out)
        except ValueError as e:
            print e
            sys.exit(1)
        allcookies, logouts = ret
        cookies = []
        if out_cookies:
            for c in reversed(allcookies):
                cookie = Cookie(c)
                if not cookie in cookies: cookies.append(cookie)
        if not out_logouts:
            logouts = []

        if not format:
            print "Cookies:"
            for c in cookies:
                print " %s=%s" % (c.name, c.value)
            print "Logout urls:"
            for u in logouts:
                print " %s" % u
        elif format == "-A":
            for c in cookies:
                print cmd_to_str([c.name, c.value])
            for u in logouts:
                print cmd_to_str([u])
        elif format == "-H":
            args = []
            if len(cookies) > 0:
                args = [
                    "-c",
                    ";".join(["%s=%s" % (c.name, c.value) for c in cookies])
                ]
            if len(logouts) > 0:
                args.extend(["-x", ",".join(logouts)])
            if len(args) > 0:
                print cmd_to_str(args)
        elif format == "-J":
            cd = []
            for c in cookies:
                cd.append(c.get_dict())
            if out_cookies:
                print json.dumps(cd)
Esempio n. 14
0
	def main(self, args, opts):
		passw = None
		format = None
		out_cookies = True
		out_logouts = True
		for o,v in opts:
			if o == "-h":
				print self.usage()
				sys.exit(0)
			elif o == "-p":
				passw = v
			elif o == "-c":
				out_cookies = False
			elif o == "-l":
				out_logouts = False
			elif o in  ("-H", "-J", "-A"):
				format = o

		if not passw:
			print "The password is hidden here BUT it will be passed to phantomjs via commandline ..."
			try:
				passw = getpass.getpass()
			except KeyboardInterrupt:
				print "\nAbort..."
				sys.exit(0)

		jspath = "%s%s%s%s" % (getrealdir(__file__), "login", os.sep, "login.js")
		cmd = get_phantomjs_cmd() + [jspath, args[0], args[1], passw]
		if len(args) > 2: cmd.append(args[2])
		#print cmd_to_str(cmd)
		exe = CommandExecutor(cmd, True)
		out, err = exe.execute(20)
		if err:
			print "Unable to login"
			sys.exit(1)

		try:
			ret = json.loads(out)
		except ValueError as e:
			print e
			sys.exit(1)
		allcookies, logouts = ret
		cookies = []
		if out_cookies:
			for c in reversed(allcookies):
				cookie = Cookie(c)
				if not cookie in cookies: cookies.append(cookie)
		if not out_logouts:
			logouts = []

		if not format:
			print "Cookies:"
			for c in cookies:
				print " %s=%s" % (c.name, c.value)
			print "Logout urls:"
			for u in logouts:
				print " %s" % u
		elif format == "-A":
			for c in cookies:
				print cmd_to_str([c.name, c.value])
			for u in logouts:
				print cmd_to_str([u])
		elif format == "-H":
			args = []
			if len(cookies) > 0:
				args = ["-c", ";".join(["%s=%s" % (c.name, c.value) for c in cookies])]
			if len(logouts) > 0:
				args.extend(["-x", ",".join(logouts)])
			if len(args) > 0:
				print cmd_to_str(args)
		elif format == "-J":
			cd = []
			for c in cookies:
				cd.append(c.get_dict())
			if out_cookies:
				print json.dumps(cd)
Esempio n. 15
0
    def test_command_timeout_with_results(self):
        cmd = ['tail', '-f', os.path.realpath(__file__)]
        executor = CommandExecutor(cmd, stderr=True)
        result = executor.execute(1)

        self.assertIn("result", result[0])
Esempio n. 16
0
    def send_probe(self, request, errors):

        url = request.url
        jsn = None
        probe = None
        retries = self.process_retries
        params = []
        cookies = []

        if request.method == "POST":
            params.append("-P")
            if request.data:
                params.extend(("-D", request.data))

        if len(request.cookies) > 0:
            for cookie in request.cookies:
                c = cookie.get_dict()
                if not c['domain']:
                    purl = urlsplit(request.url)
                    c['domain'] = purl.netloc.split(":")[0]
                cookies.append(c)

            with open(self.cookie_file, 'w') as fil:
                fil.write(json.dumps(cookies))

            params.extend(("-c", self.cookie_file))

        if request.http_auth:
            params.extend(("-p", request.http_auth))

        if Shared.options['set_referer'] and request.referer:
            params.extend(("-r", request.referer))

        params.extend(("-i", str(request.db_id)))

        params.extend(("-J", self.out_file))

        params.append(url)

        while retries:
            #while False:

            # print cmd_to_str(Shared.probe_cmd + params)
            # print ""
            jsn = None
            cmd = CommandExecutor(Shared.probe_cmd + params, True)
            out, err = cmd.execute(Shared.options['process_timeout'] + 10)

            if os.path.isfile(self.out_file):
                with open(self.out_file, "r") as f:
                    jsn = f.read()
                os.unlink(self.out_file)

            if err or not jsn:
                errors.append(ERROR_PROBEKILLED)
                if not jsn:
                    break

            # try to decode json also after an exception .. sometimes phantom crashes BUT returns a valid json ..
            try:
                if jsn and type(jsn) is not str:
                    jsn = jsn[0]
                probeArray = self.load_probe_json(jsn)
            except Exception as e:
                raise

            if probeArray:
                probe = Probe(probeArray, request)

                if probe.status == "ok":
                    break

                errors.append(probe.errcode)

                if probe.errcode in (ERROR_CONTENTTYPE, ERROR_PROBE_TO):
                    break

            time.sleep(self.process_retries_interval)
            retries -= 1

        return probe
Esempio n. 17
0
    def send_probe(self, request, errors):

        url = request.url
        jsn = None
        probe = None
        retries = self.process_retries
        params = []
        cookies = []

        if request.method == "POST":
            params.append("-P")
            if request.data:
                params.extend(("-D", request.data))

        if len(request.cookies) > 0:
            for cookie in request.cookies:
                cookies.append(cookie.get_dict())

            with open(self.cookie_file, 'w') as fil:
                fil.write(json.dumps(cookies))

            params.extend(("-c", self.cookie_file))

        if request.http_auth:
            params.extend(("-p", request.http_auth))

        if Shared.options['set_referer'] and request.referer:
            params.extend(("-r", request.referer))

        params.append(url)

        while retries:
            #while False:

            # print cmd_to_str(Shared.probe_cmd + params)
            # print ""

            cmd = CommandExecutor(Shared.probe_cmd + params)
            jsn = cmd.execute(Shared.options['process_timeout'] + 2)

            if jsn == None:
                errors.append(ERROR_PROBEKILLED)
                time.sleep(self.process_retries_interval)  # ... ???
                retries -= 1
                continue

            # try to decode json also after an exception .. sometimes phantom crashes BUT returns a valid json ..
            try:
                if jsn and type(jsn) is not str:
                    jsn = jsn[0]
                probeArray = self.load_probe_json(jsn)
            except Exception as e:
                raise

            if probeArray:
                probe = Probe(probeArray, request)

                if probe.status == "ok":
                    break

                errors.append(probe.errcode)

                if probe.errcode in (ERROR_CONTENTTYPE, ERROR_PROBE_TO):
                    break

            time.sleep(self.process_retries_interval)
            retries -= 1

        return probe
Esempio n. 18
0
class CrawlerThread(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.thread_uuid = uuid.uuid4()
        self.process_retries = 2
        self.process_retries_interval = 0.5

        self.status = THSTAT_RUNNING
        self.exit = False
        self.pause = False

        self.cookie_file = "%s%shtcap_cookiefile-%s.json" % (
            tempfile.gettempdir(), os.sep, self.thread_uuid)
        self.out_file = "%s%shtcap_output-%s.json" % (tempfile.gettempdir(),
                                                      os.sep, self.thread_uuid)
        self.cmd = None

    def run(self):
        self.crawl()

    def wait_request(self):
        request = None
        Shared.th_condition.acquire()
        while True:
            if self.exit == True:
                Shared.th_condition.notifyAll()
                Shared.th_condition.release()
                raise ThreadExitRequestException("exit request received")

            if Shared.requests_index >= len(Shared.requests):
                self.status = THSTAT_WAITING
                Shared.th_condition.wait(
                )  # The wait method releases the lock, blocks the current thread until another thread calls notify
                continue

            request = Shared.requests[Shared.requests_index]
            Shared.requests_index += 1

            break

        Shared.th_condition.release()

        self.status = THSTAT_RUNNING

        return request

    def load_probe_json(self, jsn):
        jsn = jsn.strip()
        if not jsn: jsn = "["
        if jsn[-1] != "]":
            jsn += '{"status":"ok", "partialcontent":true}]'
        try:
            return json.loads(jsn)
        except Exception:
            print "-- JSON DECODE ERROR %s" % jsn
            raise

    def send_probe(self, request, errors):

        url = request.url
        jsn = None
        probe = None
        retries = self.process_retries
        params = []
        cookies = []

        if request.method == "POST":
            params.append("-P")
            if request.data:
                params.extend(("-D", request.data))

        if len(request.cookies) > 0:
            for cookie in request.cookies:
                c = cookie.get_dict()
                if not c['domain']:
                    purl = urlsplit(request.url)
                    c['domain'] = purl.netloc.split(":")[0]
                cookies.append(c)

            with open(self.cookie_file, 'w') as fil:
                fil.write(json.dumps(cookies))

            params.extend(("-c", self.cookie_file))

        if request.http_auth:
            params.extend(("-p", request.http_auth))

        if Shared.options['set_referer'] and request.referer:
            params.extend(("-r", request.referer))

        params.extend(("-i", str(request.db_id)))

        params.extend(("-J", self.out_file))

        params.append(url)

        while retries:
            #while False:

            # print cmd_to_str(Shared.probe_cmd + params)
            # print ""
            jsn = None
            self.cmd = CommandExecutor(Shared.probe_cmd + params, True)
            out, err = self.cmd.execute(Shared.options['process_timeout'] + 10)

            if os.path.isfile(self.out_file):
                with open(self.out_file, "r") as f:
                    jsn = f.read()
                os.unlink(self.out_file)

            if err or not jsn:
                errors.append(ERROR_PROBEKILLED)
                if not jsn:
                    break

            # try to decode json also after an exception .. sometimes phantom crashes BUT returns a valid json ..
            try:
                if jsn and type(jsn) is not str:
                    jsn = jsn[0]
                probeArray = self.load_probe_json(jsn)
            except Exception as e:
                raise

            if probeArray:
                probe = Probe(probeArray, request)

                if probe.status == "ok":
                    break

                errors.append(probe.errcode)

                if probe.errcode in (ERROR_CONTENTTYPE, ERROR_PROBE_TO):
                    break

            time.sleep(self.process_retries_interval)
            retries -= 1

        return probe

    def wait_pause(self):
        while True:
            Shared.th_condition.acquire()
            paused = self.pause
            Shared.th_condition.release()
            if not paused:
                break
            time.sleep(0.5)

    def crawl(self):

        while True:
            url = None
            cookies = []
            requests = []

            requests_to_crawl = []
            redirects = 0
            errors = []

            try:
                request = self.wait_request()
            except ThreadExitRequestException:
                if os.path.exists(self.cookie_file):
                    os.remove(self.cookie_file)
                return
            except Exception as e:
                print "-->" + str(e)
                continue

            url = request.url

            purl = urlsplit(url)

            probe = None

            probe = self.send_probe(request, errors)

            if probe:
                if probe.status == "ok" or probe.errcode == ERROR_PROBE_TO:

                    requests = probe.requests

                    if probe.html:
                        request.html = probe.html

                    if len(probe.user_output) > 0:
                        request.user_output = probe.user_output

            else:
                errors.append(ERROR_PROBEFAILURE)
                # get urls with python to continue crawling
                if Shared.options['use_urllib_onerror'] == False:
                    continue
                try:
                    hr = HttpGet(request, Shared.options['process_timeout'],
                                 self.process_retries,
                                 Shared.options['useragent'],
                                 Shared.options['proxy'],
                                 Shared.options['extra_headers'])
                    requests = hr.get_requests()
                except Exception as e:
                    errors.append(str(e))

            # set out_of_scope, apply user-supplied filters to urls (ie group_qs)
            adjust_requests(requests)

            Shared.main_condition.acquire()
            res = CrawlResult(request, requests, errors,
                              probe.page_hash if probe else "")
            Shared.crawl_results.append(res)
            Shared.main_condition.notify()
            Shared.main_condition.release()

            self.wait_pause()