Beispiel #1
0
def find_suspicious(pv_id):
    pageviews = Pageview.query.filter(Pageview.id == pv_id).all()
    suspicious = []

    if not pageviews:
        print('ERROR: no pageview found with id {}'.format(pv_id))
    elif len(pageviews) > 1:
        print('ERROR: >1 pageviews found with id {}'.format(pv_id))
    else:
        pageview = pageviews[0]
        hostname = urlparse(pageview.url).netloc
        root_domain = get_root_domain(hostname)

        domains = [get_root_domain(urlparse(rsc.url).netloc) for rsc in pageview.resources]
 
        # dedup the list of domains and drop all empty values:
        domains = list(set(filter(lambda x: x, domains)))

        # remove any domains that are rooted in the WHITELIST or the current root_domain:
        suspicious = list(filter(lambda x: not any([x.endswith(y) for y in WHITELIST + [root_domain]]), domains))
 
        if suspicious:
            print("FOUND SUSPICIOUS DOMAINS: {}".format(suspicious))       
            sendmail('*****@*****.**',
                     'Suspicious Resource Result! {}'.format(pageview.url),
                     render_template('email/suspicious_resource_result.html', url=pageview.url, suspicious=suspicious, domains=domains))
        else:
             print("Didn't find any suspicious domains")
   
    return suspicious
Beispiel #2
0
def yara_retroscan_for_rule(rule_id):
    rule = YaraRuleset.query.filter_by(id=rule_id).one()
    sources = {rule.namespace: rule.source}

    try:
        yara_rule = yara.compile(sources=sources)
    except:
        # TODO: this should never happen, so this email should go to the site admin
        # TODO: this should not catch any/all exceptions!
        sendmail('*****@*****.**', 'YARA Retroscan Results (error in rule compilation!)', render_template('email/yara_error.html'))
        return

    os.nice(5)
    matches = []
    try:
        for subdir in os.listdir(app.config['RESOURCE_CONTENT_FOLDER']):
            try:
                subdir = os.path.join(app.config['RESOURCE_CONTENT_FOLDER'], subdir)
                for path in os.listdir(subdir):
                    with gzip.open(os.path.join(subdir, path), 'rb') as f:
                        if yara_rule.match(data=f.read()):
                            matches.append(path.split('.')[0]) 
                            if len(matches) > app.config['MAX_HASHES']:
                                break
            except NotADirectoryError:
                pass
    except:
        sendmail(rule.email, 'YARA Retroscan Error ({0})'.format(sys.exc_info()), render_template('email/yara_error.html'))
    else:
        yara_report_matches.apply_async(args=(rule.email, rule.namespace, matches))
    
    os.nice(0)
Beispiel #3
0
def yara_report_matches(email, namespace, hashes):
    print("got match for {0} - {1} !".format(email, namespace))

    now = datetime.datetime.now()
    matches = []

    for hash in hashes:
        record = Resource.query.filter(Resource.hash == hash).limit(
            app.config['MAX_PAGES_PER_HASH']).all()
        urls = [{
            'webpage_url': r.pageview.url,
            'webpage_hash': sha256(r.pageview.url),
            'resource_url': r.url
        } for r in record]
        new_match = {'hash': hash, 'urls': urls}
        matches.append(new_match)

    rule = YaraRuleset.query.filter_by(email=email,
                                       namespace=namespace).first()

    if rule.email_tokens is None:
        rule.email_tokens = 5

    print("email tokens = {}".format(rule.email_tokens))

    if rule.last_received_tokens is None:
        rule.last_received_tokens = now

    print("last rx tokens = {}".format(rule.last_received_tokens))

    rule.email_tokens -= 1
    rule.email_tokens = min(
        10, rule.email_tokens +
        (now - rule.last_received_tokens) // datetime.timedelta(minutes=1))
    rule.last_received_tokens = now

    print("at end, email tokens = {}".format(rule.email_tokens))

    sendmail(
        email, 'YARA Scan Results (success!): {}'.format(namespace),
        render_template('email/yara_match.html',
                        matches=matches,
                        tokens=rule.email_tokens))

    if rule.email_tokens <= 0:
        print("removing rule!")
        sendmail(rule.email, 'YARA Rule Removed ({})'.format(rule.namespace),
                 render_template('email/yara_goodbye.html', rule=rule))

        db.session.delete(rule)

    db.session.commit()
Beispiel #4
0
def yara_index():
    """ yara rule submission UI view """
    errors = []
    if request.method == 'POST':
        email = request.form['email'].strip()
        if len(email) <= 0:
            errors.append("You must provide an email address")
        elif email not in app.config['EMAIL_WHITELIST']:
            errors.append("Email address not in whitelist")

        namespace = request.form['namespace'].strip()
        if len(namespace) <= 0:
            errors.append("You must provide a namespace")
        else:
            dups = YaraRuleset.query.filter_by(email=email,
                                               namespace=namespace).count()
            if dups > 0:
                errors.append("The namespace you have provided has already "
                              "been used for that email address. Please "
                              "choose a new one.")

        source = request.form['source'].strip()
        if len(source) <= 0:
            errors.append("You must provide some Yara rules")

        try:
            if len(errors) <= 0:
                ruleset = YaraRuleset(email, namespace, source, True)
                db.session.add(ruleset)
                db.session.commit()

                # send email
                r = {
                    'email': email,
                    'content': source,
                    'removal_code': ruleset.removal_code
                }
                sendmail(email, 'New YARA Rule Added! {}'.format(namespace),
                         render_template('email/yara_welcome.html', rule=r))

                flash("Rules successfully added!")
                return redirect(url_for('yara_index'))

        except yara.libyara_wrapper.YaraSyntaxError as e:
            errors.append("Syntax error in yara rule: {}".format(e))

    return render_template('yara/index.html', errors=errors)
Beispiel #5
0
def yara_index():
    """ yara rule submission UI view """
    errors = []
    if request.method == 'POST':
        email = request.form['email'].strip()
        if len(email) <= 0:
            errors.append("You must provide an email address")
        elif email not in app.config['EMAIL_WHITELIST']:
            errors.append("Email address not in whitelist")

        namespace = request.form['namespace'].strip()
        if len(namespace) <= 0:
            errors.append("You must provide a namespace")
        else:
            dups = YaraRuleset.query.filter_by(email=email,
                                               namespace=namespace).count()
            if dups > 0:
                errors.append("The namespace you have provided has already "
                              "been used for that email address. Please "
                              "choose a new one.")

        source = request.form['source'].strip()
        if len(source) <= 0:
            errors.append("You must provide some Yara rules")

        try:
            if len(errors) <= 0:
                ruleset = YaraRuleset(email, namespace, source, True)
                db.session.add(ruleset)
                db.session.commit()

                # send email
                r = {'email': email, 'content': source, 'removal_code': ruleset.removal_code}
                sendmail(email,
                        'New YARA Rule Added! {}'.format(namespace),
                        render_template('email/yara_welcome.html', rule=r))

                flash("Rules successfully added!")
                return redirect(url_for('yara_index'))
        
        except yara.libyara_wrapper.YaraSyntaxError as e:
            errors.append("Syntax error in yara rule: {}".format(e))

    return render_template('yara/index.html', errors=errors)
Beispiel #6
0
def yara_scan_file_for_email(email, path):
    # TODO: filter for 'scan_on_upload==True' too
    # TODO: store compiled rules in database to avoid re-compiling?
    try:
        rulesets = YaraRuleset.query.filter_by(email=email).all()
        sources = {}

        for r in rulesets:
            sources[r.namespace] = r.source

        def matchcb(data):
            if data['matches']:
                yara_report_matches.apply_async(
                    args=(email, data['namespace'],
                          [path.split('/')[-1].split('.')[0]]),
                    countdown=10)
            return yara.CALLBACK_CONTINUE

        # TODO: these "except" clauses should be consolidated and should catch
        #       specific exceptions.
        try:
            rules = yara.compile(sources=sources)

            with gzip.open(path, 'rb') as f:
                try:
                    rules.match(data=f.read(), callback=matchcb)

                except:
                    # TODO: this should never happen, so this email should
                    #       go to the site admin
                    sendmail(
                        '*****@*****.**',
                        'YARA Livescan Results (error while reading file / matching ruleset!)',
                        render_template('email/yara_error.html'))

        except:
            # TODO: this should never happen, so this email should go to the site admin
            sendmail('*****@*****.**',
                     'YARA Scan Results (error in rule compilation!)',
                     render_template('email/yara_error.html'))

    except (sqlalchemy.exc.DatabaseError,
            sqlalchemy.exc.OperationalError) as exc:
        print("retrying....")
Beispiel #7
0
def yara_report_matches(email, namespace, hashes):
    print("got match for {0} - {1} !".format(email, namespace))

    now = datetime.datetime.now()
    matches = []
    
    for hash in hashes:
        record = Resource.query.filter(Resource.hash == hash).limit(app.config['MAX_PAGES_PER_HASH']).all() 
        urls = [{'webpage_url': r.pageview.url, 'webpage_hash': sha256(r.pageview.url), 'resource_url': r.url} for r in record]
        new_match = {'hash': hash, 'urls': urls}
        matches.append(new_match)

    rule = YaraRuleset.query.filter_by(email=email, namespace=namespace).first()
    
    if rule.email_tokens is None: 
        rule.email_tokens = 5
    
    print("email tokens = {}".format(rule.email_tokens))

    if rule.last_received_tokens is None:
        rule.last_received_tokens = now

    print("last rx tokens = {}".format(rule.last_received_tokens))

    rule.email_tokens -= 1
    rule.email_tokens = min(10, rule.email_tokens + (now - rule.last_received_tokens) // datetime.timedelta(minutes=1))
    rule.last_received_tokens = now

    print("at end, email tokens = {}".format(rule.email_tokens))

    sendmail(email, 
             'YARA Scan Results (success!): {}'.format(namespace), 
             render_template('email/yara_match.html', matches=matches, tokens=rule.email_tokens))

    if rule.email_tokens <= 0:
        print("removing rule!")
        sendmail(rule.email,
                 'YARA Rule Removed ({})'.format(rule.namespace),
                  render_template('email/yara_goodbye.html', rule=rule))

        db.session.delete(rule)

    db.session.commit()
Beispiel #8
0
def yara_scan_file_for_email(email, path):
    # TODO: filter for 'scan_on_upload==True' too
    # TODO: store compiled rules in database to avoid re-compiling?
    try:
        rulesets = YaraRuleset.query.filter_by(email=email).all()
        sources = {}
        
        for r in rulesets:
            sources[r.namespace] = r.source

        def matchcb(data):
            if data['matches']:
                yara_report_matches.apply_async(args=(email, 
                                                      data['namespace'], 
                                                      [path.split('/')[-1].split('.')[0]]), 
                                                countdown=10)
            return yara.CALLBACK_CONTINUE

        # TODO: these "except" clauses should be consolidated and should catch
        #       specific exceptions.
        try:
            rules = yara.compile(sources=sources)
            
            with gzip.open(path, 'rb') as f:
                try:
                    rules.match(data=f.read(), callback=matchcb)
    
                except:
                    # TODO: this should never happen, so this email should 
                    #       go to the site admin
                    sendmail('*****@*****.**',
                             'YARA Livescan Results (error while reading file / matching ruleset!)',
                             render_template('email/yara_error.html'))

        except:
            # TODO: this should never happen, so this email should go to the site admin
            sendmail('*****@*****.**', 
                     'YARA Scan Results (error in rule compilation!)', 
                     render_template('email/yara_error.html'))


    except (sqlalchemy.exc.DatabaseError, sqlalchemy.exc.OperationalError) as exc:
        print("retrying....")
Beispiel #9
0
def find_suspicious(pv_id):
    pageviews = Pageview.query.filter(Pageview.id == pv_id).all()
    suspicious = []

    if not pageviews:
        print('ERROR: no pageview found with id {}'.format(pv_id))
    elif len(pageviews) > 1:
        print('ERROR: >1 pageviews found with id {}'.format(pv_id))
    else:
        pageview = pageviews[0]
        hostname = urlparse(pageview.url).netloc
        root_domain = get_root_domain(hostname)

        domains = [
            get_root_domain(urlparse(rsc.url).netloc)
            for rsc in pageview.resources
        ]

        # dedup the list of domains and drop all empty values:
        domains = list(set(filter(lambda x: x, domains)))

        # remove any domains that are rooted in the WHITELIST or the current root_domain:
        suspicious = list(
            filter(
                lambda x: not any(
                    [x.endswith(y) for y in WHITELIST + [root_domain]]),
                domains))

        if suspicious:
            print("FOUND SUSPICIOUS DOMAINS: {}".format(suspicious))
            sendmail(
                '*****@*****.**',
                'Suspicious Resource Result! {}'.format(pageview.url),
                render_template('email/suspicious_resource_result.html',
                                url=pageview.url,
                                suspicious=suspicious,
                                domains=domains))
        else:
            print("Didn't find any suspicious domains")

    return suspicious
Beispiel #10
0
def yara_retroscan_for_rule(rule_id):
    rule = YaraRuleset.query.filter_by(id=rule_id).one()
    sources = {rule.namespace: rule.source}

    try:
        yara_rule = yara.compile(sources=sources)
    except:
        # TODO: this should never happen, so this email should go to the site admin
        # TODO: this should not catch any/all exceptions!
        sendmail('*****@*****.**',
                 'YARA Retroscan Results (error in rule compilation!)',
                 render_template('email/yara_error.html'))
        return

    os.nice(5)
    matches = []
    try:
        for subdir in os.listdir(app.config['RESOURCE_CONTENT_FOLDER']):
            try:
                subdir = os.path.join(app.config['RESOURCE_CONTENT_FOLDER'],
                                      subdir)
                for path in os.listdir(subdir):
                    with gzip.open(os.path.join(subdir, path), 'rb') as f:
                        if yara_rule.match(data=f.read()):
                            matches.append(path.split('.')[0])
                            if len(matches) > app.config['MAX_HASHES']:
                                break
            except NotADirectoryError:
                pass
    except:
        sendmail(rule.email,
                 'YARA Retroscan Error ({0})'.format(sys.exc_info()),
                 render_template('email/yara_error.html'))
    else:
        yara_report_matches.apply_async(args=(rule.email, rule.namespace,
                                              matches))

    os.nice(0)
Beispiel #11
0
def yara_remove():
    """ yara rule removal UI view """
    errors = []
    if request.method == 'POST':
        code = request.form['removal_code'].strip()
        if len(code) <= 0:
            errors.append("You must provide a removal code")
        else:
            rule = YaraRuleset.query.filter_by(removal_code=code).first()
            if not rule:
                errors.append("Could not find an active YARA rule with that "
                              "removal code.")
 
        if len(errors) == 0:
            sendmail(rule.email,
                    'YARA Rule Removed ({})'.format(rule.namespace),
                    render_template('email/yara_goodbye.html', rule=rule))
            
            db.session.delete(rule)
            db.session.commit()

            flash("Rules successfully removed")
    
    return render_template('yara/remove.html', errors=errors)
Beispiel #12
0
def yara_remove():
    """ yara rule removal UI view """
    errors = []
    if request.method == 'POST':
        code = request.form['removal_code'].strip()
        if len(code) <= 0:
            errors.append("You must provide a removal code")
        else:
            rule = YaraRuleset.query.filter_by(removal_code=code).first()
            if not rule:
                errors.append("Could not find an active YARA rule with that "
                              "removal code.")

        if len(errors) == 0:
            sendmail(rule.email,
                     'YARA Rule Removed ({})'.format(rule.namespace),
                     render_template('email/yara_goodbye.html', rule=rule))

            db.session.delete(rule)
            db.session.commit()

            flash("Rules successfully removed")

    return render_template('yara/remove.html', errors=errors)