Beispiel #1
0
def put_scan_control(scan_id):
    # Find the scan
    scan = Scan.get_scan(scan_id)
    if not scan:
        return jsonify(success=False, error='no-such-scan')
    # Check if the state is valid
    state = request.data
    if state not in ('START', 'STOP'):
        return jsonify(success=False, error='unknown-state')


    # Handle start
    if state == 'START':
        if scan.state != 'CREATED':
            return jsonify(success=False, error='invalid-state-transition')


        scan.state = "QUEUED"
        scan.queued = datetime.datetime.utcnow()
        # Queue the scan to start
        db.session.commit()
        tasks.scan.apply_async([scan.scan_uuid], countdown=3, queue='scan')
    # Handle stop
    if state == 'STOP':
        scans.update({"id": scan_id}, {"$set": {"state": "STOPPING", "queued": datetime.datetime.utcnow()}})
        tasks.scan_stop.apply_async([scan.scan_uuid], queue='state')
    return jsonify(success=True)
Beispiel #2
0
def scan_start(scan_id, t):
    scan = Scan.get_scan(scan_id)
    old_state = scan.state
    scan.state = "STARTED"
    scan.started =  datetime.datetime.utcfromtimestamp(t)
    logger.debug("Starting scan [%s] [%s -> %s]" % (scan_id, old_state, scan.state))
    db.session.commit()
Beispiel #3
0
def session_set_task_id(scan_id, session_id, task_id):
    scan = Scan.get_scan(scan_id)
    logger.debug("Setting task id for session [%s]" % (session_id))
    for s in scan.sessions:
        if s.session_uuid == session_id:
            s.task = task_id
    db.session.commit()
Beispiel #4
0
def session_queue(scan_id, session_id, t):
    logger.debug("Queuing session [%s] for scan [%s]" % (session_id, scan_id))
    scan = Scan.get_scan(scan_id)
    for session in scan.sessions:
        session.state = "QUEUED"
        scan.queued = datetime.datetime.utcfromtimestamp(t)
    db.session.commit()
Beispiel #5
0
def scan_stop(scan_id):

    logger.debug("This is scan_stop " + str(scan_id))

    try:

        #
        # Find the scan we are asked to stop
        #

        scan = Scan.get_scan(scan_id)
        if not scan:
            logger.error("Cannot find scan %s" % scan_id)
            return

        #
        # Set the scan to cancelled. Even though some plugins may still run.
        #
        logger.debug("Stopping scan %s [%s -> STOPPED]" % (scan.scan_uuid, scan.state))
        scan.state = "STOPPED"
        scan.started = datetime.datetime.utcnow()


        #
        # Set all QUEUED and STARTED sessions to STOPPED and revoke the sessions that have been queued
        #

        for session in scan.sessions:
            if session.state in ('QUEUED', 'STARTED'):
                session.state = "STOPPED"
                session.finished = datetime.datetime.utcnow()
                db.session.commit()
            if session.task:
                revoke(session.task, terminate=True, signal='SIGUSR1')

    except Exception as e:

        logger.exception("Error while processing task. Marking scan as FAILED.")

        try:
            if scan:
                scan = Scan.get_scan(scan_id)
                scan.state = "FAILED"
                scan.finished = datetime.datetime.utcnow()
                db.session.commit()
        except Exception as e:
            logger.exception("Error when marking scan as FAILED")
Beispiel #6
0
def session_start(scan_id, session_id, t):
    scan = Scan.get_scan(scan_id)
    logger.debug("Starting session [%s] for scan [%s] [%s -> STARTED]" % (session_id, scan_id, scan.state))
    for session in scan.sessions:
        session.state = "STARTED"
        session.started = datetime.datetime.utcfromtimestamp(t)
    
    db.session.commit()
Beispiel #7
0
def post_scan_create():
    # try to decode the configuration
    configuration = request.json
    
    # See if the plan exists
    plan = Plan.get_plan(configuration['plan'])
    if not plan:
        return jsonify(success=False)
    # Merge the configuration
    # Create a scan object

    scan = Scan()

    scan.meta = json.dumps({ "user": configuration['user'], "tags": [] } )

    scan.configuration = json.dumps(configuration['configuration'])
    scan.site = Site.get_site_by_url(configuration['configuration']['target'])

    scan.plan = json.dumps( { "name": plan.name, "revision": 0 })

    db.session.add(scan)
    db.session.commit()
    for step in plan.workflows:

        session_configuration = {}
        if step.configuration:
         session_configuration = json.loads(step.configuration)
        session_configuration.update(configuration['configuration'])


        session = Session()
        session.configuration = json.dumps(session_configuration)
        session.description = step.description

        session.plugin = json.dumps(Plugin.plugins[step.plugin_name]['descriptor'])

        scan.sessions.append(session)
        db.session.add(session)
        db.session.commit()

    db.session.commit()

    return jsonify(success=True, scan=sanitize_scan(scan))
Beispiel #8
0
    def has_permission(*args, **kwargs):
        email = request.args.get('email')

        # If the task is scheduled by crontab, proceed with the task
        if email == 'cron':
            return view(*args, **kwargs)

        if email:
            user = User.get_user(email)
            if not user:
                return jsonify(success=False, reason='user-does-not-exist')
            scan = Scan.get_scan(kwargs['scan_id'])
            if user.role == 'user':
                # XX really, really fix the Scan :: Site association, but for compat we will do the json dance

                if not Scan.site in user.sites:
                    return jsonify(success=False, reason='not-found')

        return view(*args, **kwargs) # if groupz.count is not zero, or user is admin
Beispiel #9
0
def scan_finish(scan_id, state, t, failure=None):
    logger.debug("Attempting to finish scan for [%s] in state [%s]" % (scan_id, state))
    try:

        #
        # Find the scan we are asked to finish
        #
        scan = Scan.get_scan(scan_id)

        if not scan:
            logger.error("Cannot find scan %s" % scan_id)
            return

        #
        # Mark the scan as finished with the provided state
        #

        if failure:
            scan.state = state
            scan.finished =  datetime.datetime.utcfromtimestamp(t)
            scan.failure = failure
        else:
            scan.state = state
            scan.finished =  datetime.datetime.utcfromtimestamp(t)

        db.session.commit()

        #
        # Fire the callback
        #

        try:
            logger.debug("... Attempting to to invoke callback")
            config = json.loads(scan.configuration)

            callback = config.get('callback')
            if callback:
                r = requests.post(callback['url'], headers={"Content-Type": "application/json"},
                                  data=json.dumps({'event': 'scan-state', 'id': scan.scan_uuid, 'state': state}))
                r.raise_for_status()
        except Exception as e:
            logger.exception("(Ignored) failure while calling scan state callback for scan %s" % scan['id'])

        #
        # If there are remaining plugin sessions that are still in the CREATED state
        # then change those to CANCELLED because we wont be executing them anymore.
        #
        for s in scan.sessions:
            if s.state == 'CREATED':
                s.state = 'CANCELLED'
        db.session.commit()

    except Exception as e:

        logger.exception("Error while finishing scan. Trying to mark scan as FAILED.")

        try:
            scan = Scan.get_scan(scan_id)
            scan.state = "FAILED"
            scan.finished = datetime.datetime.utcnow()
            
        except Exception as e:
            logger.exception("Error when marking scan as FAILED")
Beispiel #10
0
def scan(scan_id):
    logger.debug("Starting scan [%s] (scan:572)" % scan_id)
    try:

        #
        # See if the scan exists.
        #
        logger.debug("Retrieving scan for scan() [%s]" % scan_id)
        scan = get_scan(cfg['api']['url'], scan_id)
        if not scan:
            logger.error("Cannot load scan %s" % scan_id)
            return

        #
        # Is the scan in the right state to be started?
        #

        if scan['state'] != 'QUEUED':
            logger.error("Scan %s has invalid state. Expected QUEUED but got %s" % (scan_id, scan['state']))
            return

        #
        # Move the scan to the STARTED state
        #
        db_scan = Scan.get_scan(scan['id'])
        db_scan.state = 'STARTED'
        scan['state'] = 'STARTED'
        db.session.commit()
        
        send_task("minion.backend.tasks.scan_start", [scan_id, time.time()], queue='state').get()

        #
        # Check this site against the access control lists
        #

        if not scannable(scan['configuration']['target'],
                         scan_config().get('whitelist', []),
                         scan_config().get('blacklist', [])):
            failure = {"hostname": socket.gethostname(),
                       "reason": "target-blacklisted",
                       "message": "The target cannot be scanned by Minion because its IP address or hostname has been blacklisted."}
            return set_finished(scan_id, 'ABORTED', failure=failure)

        #
        # Verify ownership prior to running scan
        #

        target = scan['configuration']['target']
        site = get_site_info(cfg['api']['url'], target)
        if not site:
            return set_finished(scan_id, 'ABORTED')

        if site.get('verification') and site['verification']['enabled']:
            verified = ownership.verify(target, site['verification']['value'])
            if not verified:
                failure = {"hostname": socket.gethostname(),
                           "reason": "target-ownership-verification-failed",
                           "message": "The target cannot be scanned because the ownership verification failed."}
                return set_finished(scan_id, 'ABORTED', failure=failure)

        #
        # Run each plugin session
        #

        for session in scan['sessions']:

            #
            # Mark the session as QUEUED
            #

            db_session = Session.get_session(session['id'])
            db_session.state = 'QUEUED'
            session['state'] = 'QUEUED'
            db.session.commit()
            #scans.update({"id": scan['id'], "sessions.id": session['id']}, {"$set": {"sessions.$.state": "QUEUED", "sessions.$.queued": datetime.datetime.utcnow()}})
            send_task("minion.backend.tasks.session_queue",
                      [scan['id'], session['id'], time.time()],
                      queue='state').get()

            #
            # Execute the plugin. The plugin worker will set the session state and issues.
            #
            db.session.commit()
            

            queue = queue_for_session(session, cfg)
            result = send_task("minion.backend.tasks.run_plugin",
                               [scan_id, session['id']],
                               queue=queue)

            #scans.update({"id": scan_id, "sessions.id": session['id']}, {"$set": {"sessions.$._task": result.id}})
            send_task("minion.backend.tasks.session_set_task_id",
                      [scan_id, session['id'], result.id],
                      queue='state').get()

            try:
                plugin_result = result.get()
            except TaskRevokedError as e:
                plugin_result = "STOPPED"

            db_session = Session.get_session(session['id'])
            db_session.state = plugin_result
            session['state'] = plugin_result

            db.session.commit()
            #
            # If the user stopped the workflow or if the plugin aborted then stop the whole scan
            #

            if plugin_result in ('ABORTED', 'STOPPED'):
                # Mark the scan as failed
                #scans.update({"id": scan_id}, {"$set": {"state": plugin_result, "finished": datetime.datetime.utcnow()}})
                send_task("minion.backend.tasks.scan_finish",
                          [scan_id, plugin_result, time.time()],
                          queue='state').get()
                # Mark all remaining sessions as cancelled
                for s in scan['sessions']:
                    if s['state'] == 'CREATED':
                        s['state'] = 'CANCELLED'
                        dbs =  Session.get_session(s['id'])
                        s.state = 'CANCELLED'
                        db.session.commit()
                        #scans.update({"id": scan['id'], "sessions.id": s['id']}, {"$set": {"sessions.$.state": "CANCELLED", "sessions.$.finished": datetime.datetime.utcnow()}})
                        send_task("minion.backend.tasks.session_finish",
                                  [scan['id'], s['id'], "CANCELLED", time.time()],
                                  queue='state').get()
                # We are done with this scan
                return

        #
        # Move the scan to the FINISHED state
        #

        scan['state'] = 'FINISHED'
        db_scan = Scan.get_scan(scan['id'])
        db_scan.state = 'FINISHED'
        db.session.commit()

        #
        # If one of the plugin has failed then marked the scan as failed
        #
        for session in scan['sessions']:
            if session['state'] == 'FAILED':
                db_scan = Scan.get_scan(scan['id'])
                db_scan.state = 'FAILED'
                db.session.commit()
                scan['state'] = 'FAILED'

        #scans.update({"id": scan_id}, {"$set": {"state": "FINISHED", "finished": datetime.datetime.utcnow()}})
        send_task("minion.backend.tasks.scan_finish",
                  [scan_id, scan['state'], time.time()],
                  queue='state').get()

    except Exception as e:

        #
        # Our exception strategy is simple: if anything was thrown above that we did not explicitly catch then
        # we assume there was a non recoverable error that made the scan fail. We mark it as such and
        # record the exception.
        #

        logger.exception("Error while running scan. Marking scan FAILED.")

        try:
            failure = { "hostname": socket.gethostname(),
                        "reason": "backend-exception",
                        "message": str(e),
                        "exception": traceback.format_exc() }
            send_task("minion.backend.tasks.scan_finish",
                      [scan_id, "FAILED", time.time(), failure],
                      queue='state').get()
        except Exception as e:
            logger.exception("Error when marking scan as FAILED")
Beispiel #11
0
def run_plugin(scan_id, session_id):

    logger.debug("This is run_plugin " + str(scan_id) + " " + str(session_id))

    try:

        #
        # Find the scan for this plugin session. Bail out if the scan has been marked as STOPPED or if
        # the state is not STARTED.
        #
        logger.debug("Retrieving scan to run plugin [%s]" % scan_id)
        scan = get_scan(cfg['api']['url'], scan_id)
        if not scan:
            logger.error("Cannot load scan %s" % scan_id)
            return

        if scan['state'] in ('STOPPING', 'STOPPED'):
            return

        if scan['state'] != 'STARTED':
            logger.error("Scan %s has invalid state. Expected STARTED but got %s" % (scan_id, scan['state']))
            return

        #
        # Find the plugin session in the scan. Bail out if the session has been marked as STOPPED or if
        # the state is not QUEUED.
        #

        session = find_session(scan, session_id)
        db_session = Session.get_session(session_id)

        if not session:
            logger.error("Cannot find session %s/%s" % (scan_id, session_id))
            return

        if db_session.state != 'QUEUED':
            logger.error("Session %s/%s has invalid state. Expected QUEUED but got %s" % (scan_id, session_id, session['state']))
            return

        #
        # Move the session in the STARTED state
        #
        send_task("minion.backend.tasks.session_start",
                  [scan_id, session_id, time.time()],
                  queue='state').get()
        scan['state'] = 'STARTED'

        db_scan = Scan.get_scan(scan['id'])
        db_scan.state = 'STARTED'
        db.session.commit()
        finished = None

        #
        # This is an experiment to see if removing Twisted makes the celery workers more stable.
        #

        def enqueue_output(fd, queue):
            try:
                for line in iter(fd.readline, b''):
                    queue.put(line)
            except Exception as e:
                logger.exception("Error while reading a line from the plugin-runner")
            finally:
                fd.close()
                queue.put(None)

        def make_signal_handler(p):
            def signal_handler(signum, frame):
                p.send_signal(signal.SIGUSR1)
            return signal_handler

        arguments = [ "minion-plugin-runner",
                      "-c", json.dumps(session['configuration']),
                      "-p", session['plugin']['class'],
                      "-s", session_id ]

        p = subprocess.Popen(arguments, bufsize=1, stdout=subprocess.PIPE, close_fds=True)

        signal.signal(signal.SIGUSR1, make_signal_handler(p))

        q = Queue.Queue()
        t = threading.Thread(target=enqueue_output, args=(p.stdout, q))
        t.daemon = True
        t.start()

        while True:
            try:
                line = q.get(timeout=0.25)
                if line is None:
                    break

                line = line.strip()

                if finished is not None:
                    logger.error("Plugin emitted (ignored) message after finishing: " + line)
                    return

                msg = json.loads(line)

                # Issue: persist it
                if msg['msg'] == 'issue':
                    send_task("minion.backend.tasks.session_report_issue",
                              args=[scan_id, session_id, msg['data']],
                              queue='state').get()

                # Progress: update the progress
                if msg['msg'] == 'progress':
                    pass # TODO

                # Finish: update the session state, wait for the plugin runner to finish, return the state
                if msg['msg'] == 'finish':
                    logger.debug("MESSAGE : %s" % json.dumps(msg))
                    finished = msg['data']['state']
                    
                    if msg['data']['state'] in ('FINISHED', 'FAILED', 'STOPPED', 'TERMINATED', 'TIMEOUT', 'ABORTED'):
                        try:
                          params = [scan['id'], session['id'], msg['data']['state'], time.time(), msg['data'].get('failure')]
                        except Exception as e:
                            logger.debug("[Error] %s" % e)
                        send_task("minion.backend.tasks.session_finish", args = params, queue='state').get()

            except Queue.Empty:
                pass

        return_code = p.wait()

        signal.signal(signal.SIGUSR1, signal.SIG_DFL)

        if not finished:
            failure = { "hostname": socket.gethostname(),
                        "message": "The plugin did not finish correctly",
                        "exception": None }
            send_task("minion.backend.tasks.session_finish",
                      [scan['id'], session['id'], 'FAILED', time.time(), failure],
                      queue='state').get()

        return finished

    except Exception as e:

        #
        # Our exception strategy is simple: if anything was thrown above that we did not explicitly catch then
        # we assume there was a non recoverable error that made the plugin session fail. We mark it as such and
        # record the exception.
        #

        logger.exception("Error while running plugin session. Marking session FAILED.")

        try:
            failure = { "hostname": socket.gethostname(),
                        "message": str(e),
                        "exception": traceback.format_exc() }
            send_task("minion.backend.tasks.session_finish",
                      [scan_id, session_id, "FAILED", time.time(), failure],
                      queue='state').get()
        except Exception as e:
            logger.exception("Error when marking scan as FAILED")

        return "FAILED"
Beispiel #12
0
def get_scan_summary(scan_id):
    Scan.get_scan(scan_id)
    if not scan:
        return jsonify(success=False, reason='not-found')
    return jsonify(success=True, summary=summarize_scan(sanitize_scan(scan)))
Beispiel #13
0
def get_scan(scan_id):
    scan = Scan.get_scan(scan_id)
    if not scan:
        return jsonify(success=False, reason='not-found')
    return jsonify(success=True, scan=sanitize_scan(scan))