Esempio n. 1
0
def create():
    error = None
    form = URLScrape(request.form)
    if request.method == 'POST':
        try:
            url = request.form['url']
            cleanget = cleangetUrl(url)
            filename = downloadedFilename(cleanget)
            _call_downloaded_url = geturlResource(cleanget)
            _call_create_txt_file = texttoFile(filename)
            txtfile = filename + ".txt"
            csvfile = filename + ".csv"
            cleaned_url = cleanUrl(cleanget)
            ktags = match_keyword(txtfile)
        except:
            error = "Unable to scrape that url. Check the address and that you're connected to the internet."
            return render_template(
                'one_scrape_form.html',
                username=session['name'],
                error=error,
                form=form,
            )
        error = []
        if form.N_BI_checkbx.data == True:
            nbi_out = iocExtractor(cleanget)
            if nbi_out == None:
                nbi_out = []
            result = Network_Bro_Intel_dt(
                source=cleaned_url,
                type_hash=hash_type(NBI_HASH_STRING, cleaned_url),
                priority=form.priority.data,
                bro_intel_indicators=nbi_out,
                created_by=session['name'],
                localfile=filename,
                localtxtfile=txtfile,
                localcsvfile=csvfile,
                tags=[],
            )
            result.str_tags = ktags
            try:
                db.session.add(result)
                db.session.commit()
                flash(
                    'successfully created network-bro_intel detection object')
            except IntegrityError:
                db.session.rollback()
                error_nbi = "source and network-bro_intel detection type object already exists."
                error.append(error_nbi)

        if form.N_S_checkbx.data == True:
            result = Network_Snort_Suricata_dt(
                source=cleaned_url,
                type_hash=hash_type(NS_HASH_STRING, cleaned_url),
                priority=form.priority.data,
                created_by=session['name'],
                localfile=filename,
                localtxtfile=txtfile,
                localcsvfile=csvfile,
                tags=[],
                snort_suricata_indicators=[],
            )
            result.str_tags = ktags
            try:
                db.session.add(result)
                db.session.commit()
                flash(
                    'successfully created network-snort_suricata detection object'
                )
            except IntegrityError:
                db.session.rollback()
                error_ns = "source and network-snort_suricata detection type object already exists."
                error.append(error_ns)
        if form.B_Y_checkbx.data == True:
            result = Binary_Yara_dt(
                source=cleaned_url,
                type_hash=hash_type(BY_HASH_STRING, cleaned_url),
                priority=form.priority.data,
                created_by=session['name'],
                localfile=filename,
                localtxtfile=txtfile,
                localcsvfile=csvfile,
                tags=[],
                bin_yara_indicators=[],
            )
            result.str_tags = ktags
            try:
                db.session.add(result)
                db.session.commit()
                flash('successfully created binary-yara detection object')
            except IntegrityError:
                db.session.rollback()
                error_by = "source and binary-yara detection type object already exists."
                error.append(error_by)
        if form.M_Y_checkbx.data == True:
            result = Memory_Yara_dt(
                source=cleaned_url,
                type_hash=hash_type(MY_HASH_STRING, cleaned_url),
                priority=form.priority.data,
                created_by=session['name'],
                localfile=filename,
                localtxtfile=txtfile,
                localcsvfile=csvfile,
                tags=[],
                mem_yara_indicators=[],
            )
            result.str_tags = ktags
            try:
                db.session.add(result)
                db.session.commit()
                flash('successfully created memory-yara detection object')
            except IntegrityError:
                db.session.rollback()
                error_by = "source and memory-yara detection type object already exists."
                error.append(error_by)

        if form.N_BI_checkbx.data == True:
            return render_template(
                'one_scrape_form.html',
                username=session['name'],
                form=form,
                multiple_errors=error,
                ktags=ktags,
                nbi_out=nbi_out,
            )
        return render_template(
            'one_scrape_form.html',
            username=session['name'],
            form=form,
            multiple_errors=error,
            ktags=ktags,
        )
    if request.method == 'GET':
        return render_template(
            'one_scrape_form.html',
            username=session['name'],
            form=form,
            error=error,
        )
Esempio n. 2
0
def create():
    error = None
    form = URLScrape(request.form)
    if request.method == 'POST':
        try:
            url = request.form['url']
            cleanget = cleangetUrl(url)
            filename = downloadedFilename(cleanget)
            _call_downloaded_url = geturlResource(cleanget)
            _call_create_txt_file = texttoFile(filename)
            txtfile = filename + ".txt"
            csvfile = filename + ".csv"
            cleaned_url = cleanUrl(cleanget)
            ktags = match_keyword(txtfile)
        except:
            error = "Unable to scrape that url. Check the address and that you're connected to the internet."
            return render_template('one_scrape_form.html',
                username=session['name'], 
                error=error,
                form=form,
                )
        error = []
        if form.N_BI_checkbx.data == True:
            nbi_out = iocExtractor(cleanget)
            if nbi_out == None:
                nbi_out = []
            result = Network_Bro_Intel_dt(
            source = cleaned_url,
            type_hash = hash_type(NBI_HASH_STRING, cleaned_url),
            priority = form.priority.data,
            bro_intel_indicators = nbi_out,
            created_by = session['name'],
            localfile = filename,
            localtxtfile = txtfile,
            localcsvfile = csvfile,
            tags = [],
            )
            result.str_tags = ktags
            try:
                db.session.add(result)
                db.session.commit()
                flash('successfully created network-bro_intel detection object')
            except IntegrityError:
                db.session.rollback()
                error_nbi = "source and network-bro_intel detection type object already exists."
                error.append(error_nbi)

        if form.N_S_checkbx.data == True:
            result = Network_Snort_Suricata_dt(
            source = cleaned_url,
            type_hash = hash_type(NS_HASH_STRING, cleaned_url),
            priority = form.priority.data,
            created_by = session['name'],
            localfile = filename,
            localtxtfile = txtfile,
            localcsvfile = csvfile,
            tags = [],
            snort_suricata_indicators=[],
            )
            result.str_tags = ktags
            try:
                db.session.add(result)
                db.session.commit()
                flash('successfully created network-snort_suricata detection object')
            except IntegrityError:
                db.session.rollback()
                error_ns = "source and network-snort_suricata detection type object already exists."
                error.append(error_ns)
        if form.B_Y_checkbx.data == True:
            result = Binary_Yara_dt(
            source = cleaned_url,
            type_hash = hash_type(BY_HASH_STRING, cleaned_url),
            priority = form.priority.data,
            created_by = session['name'],
            localfile = filename,
            localtxtfile = txtfile,
            localcsvfile = csvfile,
            tags = [],
            bin_yara_indicators=[],
            )
            result.str_tags = ktags
            try:
                db.session.add(result)
                db.session.commit()
                flash('successfully created binary-yara detection object')
            except IntegrityError:
                db.session.rollback()
                error_by = "source and binary-yara detection type object already exists."
                error.append(error_by)
        if form.M_Y_checkbx.data == True:
            result = Memory_Yara_dt(
            source = cleaned_url,
            type_hash = hash_type(MY_HASH_STRING, cleaned_url),
            priority = form.priority.data,
            created_by = session['name'],
            localfile = filename,
            localtxtfile = txtfile,
            localcsvfile = csvfile,
            tags = [],
            mem_yara_indicators=[],
            )
            result.str_tags = ktags
            try:
                db.session.add(result)
                db.session.commit()
                flash('successfully created memory-yara detection object')
            except IntegrityError:
                db.session.rollback()
                error_by = "source and memory-yara detection type object already exists."
                error.append(error_by)

        if form.N_BI_checkbx.data == True:
            return render_template('one_scrape_form.html', 
                    username=session['name'], 
                    form=form, 
                    multiple_errors=error,
                    ktags=ktags,
                    nbi_out=nbi_out,
                    )      
        return render_template('one_scrape_form.html', 
                username=session['name'], 
                form=form, 
                multiple_errors=error,
                ktags=ktags,
                )            
    if request.method == 'GET':
        return render_template('one_scrape_form.html', 
                username=session['name'], 
                form=form, 
                error=error,
                )
Esempio n. 3
0
def upload_create():
    DOCS = 'app/documents/'
    error = None
    form = UploadScrape(request.form)
    if request.method == 'POST':
        file = request.files['uploadpath']
        if file:
            filename = secure_filename(file.filename)
            sourcename = form.source.data
            cleansourcename = cleanUrl(sourcename)
            cleansource = downloadedFilename(sourcename)
            txtfile = cleansource + ".txt"
            csvfile = cleansource + ".csv"
            if not os.path.exists(DOCS + cleansource):
                file.save(os.path.join(docs, filename))
                _call_create_txt_file = texttoFile(filename)
                oldfilepath = os.path.join(docs, filename)
                oldtxtfilepath = os.path.join(docs, filename + '.txt')
                newfilepath = os.path.join(docs, cleansource)
                newtxtfilepath = os.path.join(docs, txtfile)
                os.rename(oldtxtfilepath, newtxtfilepath)
                os.rename(oldfilepath, newfilepath)
                ktags = match_keyword(txtfile)
            else:
                flash('a file already exists for that source, omitting upload')
                ktags = match_keyword(txtfile)
                pass
            error = []
            if form.N_BI_checkbx.data == True:
                nbi_out = indicatorListofDicts(cleansource)
                if nbi_out == None:
                    nbi_out = []
                result = Network_Bro_Intel_dt(
                    source=cleansourcename,
                    type_hash=hash_type(NBI_HASH_STRING, cleansourcename),
                    bro_intel_indicators=nbi_out,
                    priority=form.priority.data,
                    created_by=session['name'],
                    localfile=cleansource,
                    localtxtfile=txtfile,
                    localcsvfile=csvfile,
                    tags=[],
                )
                result.str_tags = ktags
                try:
                    db.session.add(result)
                    db.session.commit()
                    flash(
                        'successfully created network-bro_intel detection object'
                    )
                except IntegrityError:
                    db.session.rollback()
                    error_nbi = "source and network-bro_intel detection type object already exists."
                    error.append(error_nbi)

            if form.N_S_checkbx.data == True:
                result = Network_Snort_Suricata_dt(
                    source=cleansourcename,
                    type_hash=hash_type(NS_HASH_STRING, cleansourcename),
                    priority=form.priority.data,
                    created_by=session['name'],
                    localfile=cleansource,
                    localtxtfile=txtfile,
                    localcsvfile=csvfile,
                    tags=[],
                    snort_suricata_indicators=[],
                )
                result.str_tags = ktags
                try:
                    db.session.add(result)
                    db.session.commit()
                    flash(
                        'successfully created network-snort_suricata detection object'
                    )
                except IntegrityError:
                    db.session.rollback()
                    error_ns = "source and network-snort_suricata detection type object already exists."
                    error.append(error_ns)
            if form.B_Y_checkbx.data == True:
                result = Binary_Yara_dt(source=cleansourcename,
                                        type_hash=hash_type(
                                            BY_HASH_STRING, cleansourcename),
                                        priority=form.priority.data,
                                        created_by=session['name'],
                                        localfile=cleansource,
                                        localtxtfile=txtfile,
                                        localcsvfile=csvfile,
                                        tags=[],
                                        bin_yara_indicators=[])
                result.str_tags = ktags
                try:
                    db.session.add(result)
                    db.session.commit()
                    flash('successfully created binary-yara detection object')
                except IntegrityError:
                    db.session.rollback()
                    error_by = "source and binary-yara detection type object already exists."
                    error.append(error_by)
            if form.M_Y_checkbx.data == True:
                result = Memory_Yara_dt(source=cleansourcename,
                                        type_hash=hash_type(
                                            MY_HASH_STRING, cleansourcename),
                                        priority=form.priority.data,
                                        created_by=session['name'],
                                        localfile=cleansource,
                                        localtxtfile=txtfile,
                                        localcsvfile=csvfile,
                                        tags=[],
                                        mem_yara_indicators=[])
                result.str_tags = ktags
                try:
                    db.session.add(result)
                    db.session.commit()
                    flash('successfully created memory-yara detection object')
                except IntegrityError:
                    db.session.rollback()
                    error_by = "source and memory-yara detection type object already exists."
                    error.append(error_by)

            if form.N_BI_checkbx.data == True:
                return render_template(
                    'upload_scrape_form.html',
                    username=session['name'],
                    form=form,
                    multiple_errors=error,
                    ktags=ktags,
                    nbi_out=nbi_out,
                )
            return render_template(
                'upload_scrape_form.html',
                username=session['name'],
                form=form,
                multiple_errors=error,
                ktags=ktags,
            )
        else:
            error = "need to upload a file"
            return render_template(
                'upload_scrape_form.html',
                form=form,
                error=error,
                username=session['name'],
            )
    if request.method == 'GET':
        return render_template(
            'upload_scrape_form.html',
            form=form,
            error=error,
            username=session['name'],
        )
Esempio n. 4
0
def upload_create():
    DOCS = 'app/documents/'
    error = None
    form = UploadScrape(request.form)
    if request.method == 'POST':
        file = request.files['uploadpath']
        if file:
            filename = secure_filename(file.filename)
            sourcename = form.source.data
            cleansourcename = cleanUrl(sourcename)
            cleansource = downloadedFilename(sourcename)
            txtfile = cleansource + ".txt"
            csvfile = cleansource + ".csv"
            if not os.path.exists(DOCS + cleansource):
                file.save(os.path.join(docs, filename))
                _call_create_txt_file = texttoFile(filename)
                oldfilepath = os.path.join(docs, filename)
                oldtxtfilepath = os.path.join(docs, filename + '.txt')
                newfilepath = os.path.join(docs, cleansource)
                newtxtfilepath = os.path.join(docs, txtfile)
                os.rename(oldtxtfilepath, newtxtfilepath)
                os.rename(oldfilepath, newfilepath)
                ktags = match_keyword(txtfile)
            else:
                flash('a file already exists for that source, omitting upload')
                ktags = match_keyword(txtfile)
                pass
            error = []
            if form.N_BI_checkbx.data == True:                    
                nbi_out = indicatorListofDicts(cleansource)
                if nbi_out == None:
                    nbi_out = []
                result = Network_Bro_Intel_dt(
                    source = cleansourcename,
                    type_hash = hash_type(NBI_HASH_STRING, cleansourcename),
                    bro_intel_indicators = nbi_out,
                    priority = form.priority.data,
                    created_by = session['name'],
                    localfile = cleansource,
                    localtxtfile = txtfile,
                    localcsvfile = csvfile,
                    tags = [],
                    )
                result.str_tags = ktags
                try:
                    db.session.add(result)
                    db.session.commit()
                    flash('successfully created network-bro_intel detection object')
                except IntegrityError:
                    db.session.rollback()
                    error_nbi = "source and network-bro_intel detection type object already exists."
                    error.append(error_nbi)

            if form.N_S_checkbx.data == True:
                result = Network_Snort_Suricata_dt(
                source = cleansourcename,
                type_hash = hash_type(NS_HASH_STRING, cleansourcename),
                priority = form.priority.data,
                created_by = session['name'],
                localfile = cleansource,
                localtxtfile = txtfile,
                localcsvfile = csvfile,
                tags = [],
                snort_suricata_indicators=[],
                )
                result.str_tags = ktags
                try:
                    db.session.add(result)
                    db.session.commit()
                    flash('successfully created network-snort_suricata detection object')
                except IntegrityError:
                    db.session.rollback()
                    error_ns = "source and network-snort_suricata detection type object already exists."
                    error.append(error_ns)
            if form.B_Y_checkbx.data == True:
                result = Binary_Yara_dt(
                source = cleansourcename,
                type_hash = hash_type(BY_HASH_STRING, cleansourcename),
                priority = form.priority.data,
                created_by = session['name'],
                localfile = cleansource,
                localtxtfile = txtfile,
                localcsvfile = csvfile,
                tags = [],
                bin_yara_indicators=[]
                )
                result.str_tags = ktags
                try:
                    db.session.add(result)
                    db.session.commit()
                    flash('successfully created binary-yara detection object')
                except IntegrityError:
                    db.session.rollback()
                    error_by = "source and binary-yara detection type object already exists."
                    error.append(error_by)
            if form.M_Y_checkbx.data == True:
                result = Memory_Yara_dt(
                source = cleansourcename,
                type_hash = hash_type(MY_HASH_STRING, cleansourcename),
                priority = form.priority.data,
                created_by = session['name'],
                localfile = cleansource,
                localtxtfile = txtfile,
                localcsvfile = csvfile,
                tags = [],
                mem_yara_indicators=[]
                )
                result.str_tags = ktags
                try:
                    db.session.add(result)
                    db.session.commit()
                    flash('successfully created memory-yara detection object')
                except IntegrityError:
                    db.session.rollback()
                    error_by = "source and memory-yara detection type object already exists."
                    error.append(error_by)

            if form.N_BI_checkbx.data == True:
                return render_template('upload_scrape_form.html', 
                        username=session['name'], 
                        form=form, 
                        multiple_errors=error,
                        ktags=ktags,
                        nbi_out=nbi_out,
                        )                      
            return render_template('upload_scrape_form.html', 
                    username=session['name'], 
                    form=form, 
                    multiple_errors=error,
                    ktags=ktags,
                    ) 
        else:
            error = "need to upload a file"
            return render_template('upload_scrape_form.html',
                            form=form,
                            error=error,
                            username=session['name'],
                            )
    if request.method == 'GET':
        return render_template('upload_scrape_form.html',
                            form=form,
                            error=error,
                            username=session['name'],
                            )