def create():
error = None
form = URLScrape(request.form)
if request.method == 'POST':
try:
url = request.form['url']
cleanget = cleangetUrl(url)
filename = downloadedFilename(cleanget)
_call_downloaded_url = geturlResource(cleanget)
_call_create_txt_file = texttoFile(filename)
txtfile = filename + ".txt"
csvfile = filename + ".csv"
cleaned_url = cleanUrl(cleanget)
ktags = match_keyword(txtfile)
except:
error = "Unable to scrape that url. Check the address and that you're connected to the internet."
return render_template(
'one_scrape_form.html',
username=session['name'],
error=error,
form=form,
)
error = []
if form.N_BI_checkbx.data == True:
nbi_out = iocExtractor(cleanget)
if nbi_out == None:
nbi_out = []
result = Network_Bro_Intel_dt(
source=cleaned_url,
type_hash=hash_type(NBI_HASH_STRING, cleaned_url),
priority=form.priority.data,
bro_intel_indicators=nbi_out,
created_by=session['name'],
localfile=filename,
localtxtfile=txtfile,
localcsvfile=csvfile,
tags=[],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash(
'successfully created network-bro_intel detection object')
except IntegrityError:
db.session.rollback()
error_nbi = "source and network-bro_intel detection type object already exists."
error.append(error_nbi)
if form.N_S_checkbx.data == True:
result = Network_Snort_Suricata_dt(
source=cleaned_url,
type_hash=hash_type(NS_HASH_STRING, cleaned_url),
priority=form.priority.data,
created_by=session['name'],
localfile=filename,
localtxtfile=txtfile,
localcsvfile=csvfile,
tags=[],
snort_suricata_indicators=[],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash(
'successfully created network-snort_suricata detection object'
)
except IntegrityError:
db.session.rollback()
error_ns = "source and network-snort_suricata detection type object already exists."
error.append(error_ns)
if form.B_Y_checkbx.data == True:
result = Binary_Yara_dt(
source=cleaned_url,
type_hash=hash_type(BY_HASH_STRING, cleaned_url),
priority=form.priority.data,
created_by=session['name'],
localfile=filename,
localtxtfile=txtfile,
localcsvfile=csvfile,
tags=[],
bin_yara_indicators=[],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created binary-yara detection object')
except IntegrityError:
db.session.rollback()
error_by = "source and binary-yara detection type object already exists."
error.append(error_by)
if form.M_Y_checkbx.data == True:
result = Memory_Yara_dt(
source=cleaned_url,
type_hash=hash_type(MY_HASH_STRING, cleaned_url),
priority=form.priority.data,
created_by=session['name'],
localfile=filename,
localtxtfile=txtfile,
localcsvfile=csvfile,
tags=[],
mem_yara_indicators=[],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created memory-yara detection object')
except IntegrityError:
db.session.rollback()
error_by = "source and memory-yara detection type object already exists."
error.append(error_by)
if form.N_BI_checkbx.data == True:
return render_template(
'one_scrape_form.html',
username=session['name'],
form=form,
multiple_errors=error,
ktags=ktags,
nbi_out=nbi_out,
)
return render_template(
'one_scrape_form.html',
username=session['name'],
form=form,
multiple_errors=error,
ktags=ktags,
)
if request.method == 'GET':
return render_template(
'one_scrape_form.html',
username=session['name'],
form=form,
error=error,
)
def create():
error = None
form = URLScrape(request.form)
if request.method == 'POST':
try:
url = request.form['url']
cleanget = cleangetUrl(url)
filename = downloadedFilename(cleanget)
_call_downloaded_url = geturlResource(cleanget)
_call_create_txt_file = texttoFile(filename)
txtfile = filename + ".txt"
csvfile = filename + ".csv"
cleaned_url = cleanUrl(cleanget)
ktags = match_keyword(txtfile)
except:
error = "Unable to scrape that url. Check the address and that you're connected to the internet."
return render_template('one_scrape_form.html',
username=session['name'],
error=error,
form=form,
)
error = []
if form.N_BI_checkbx.data == True:
nbi_out = iocExtractor(cleanget)
if nbi_out == None:
nbi_out = []
result = Network_Bro_Intel_dt(
source = cleaned_url,
type_hash = hash_type(NBI_HASH_STRING, cleaned_url),
priority = form.priority.data,
bro_intel_indicators = nbi_out,
created_by = session['name'],
localfile = filename,
localtxtfile = txtfile,
localcsvfile = csvfile,
tags = [],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created network-bro_intel detection object')
except IntegrityError:
db.session.rollback()
error_nbi = "source and network-bro_intel detection type object already exists."
error.append(error_nbi)
if form.N_S_checkbx.data == True:
result = Network_Snort_Suricata_dt(
source = cleaned_url,
type_hash = hash_type(NS_HASH_STRING, cleaned_url),
priority = form.priority.data,
created_by = session['name'],
localfile = filename,
localtxtfile = txtfile,
localcsvfile = csvfile,
tags = [],
snort_suricata_indicators=[],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created network-snort_suricata detection object')
except IntegrityError:
db.session.rollback()
error_ns = "source and network-snort_suricata detection type object already exists."
error.append(error_ns)
if form.B_Y_checkbx.data == True:
result = Binary_Yara_dt(
source = cleaned_url,
type_hash = hash_type(BY_HASH_STRING, cleaned_url),
priority = form.priority.data,
created_by = session['name'],
localfile = filename,
localtxtfile = txtfile,
localcsvfile = csvfile,
tags = [],
bin_yara_indicators=[],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created binary-yara detection object')
except IntegrityError:
db.session.rollback()
error_by = "source and binary-yara detection type object already exists."
error.append(error_by)
if form.M_Y_checkbx.data == True:
result = Memory_Yara_dt(
source = cleaned_url,
type_hash = hash_type(MY_HASH_STRING, cleaned_url),
priority = form.priority.data,
created_by = session['name'],
localfile = filename,
localtxtfile = txtfile,
localcsvfile = csvfile,
tags = [],
mem_yara_indicators=[],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created memory-yara detection object')
except IntegrityError:
db.session.rollback()
error_by = "source and memory-yara detection type object already exists."
error.append(error_by)
if form.N_BI_checkbx.data == True:
return render_template('one_scrape_form.html',
username=session['name'],
form=form,
multiple_errors=error,
ktags=ktags,
nbi_out=nbi_out,
)
return render_template('one_scrape_form.html',
username=session['name'],
form=form,
multiple_errors=error,
ktags=ktags,
)
if request.method == 'GET':
return render_template('one_scrape_form.html',
username=session['name'],
form=form,
error=error,
)
def upload_create():
DOCS = 'app/documents/'
error = None
form = UploadScrape(request.form)
if request.method == 'POST':
file = request.files['uploadpath']
if file:
filename = secure_filename(file.filename)
sourcename = form.source.data
cleansourcename = cleanUrl(sourcename)
cleansource = downloadedFilename(sourcename)
txtfile = cleansource + ".txt"
csvfile = cleansource + ".csv"
if not os.path.exists(DOCS + cleansource):
file.save(os.path.join(docs, filename))
_call_create_txt_file = texttoFile(filename)
oldfilepath = os.path.join(docs, filename)
oldtxtfilepath = os.path.join(docs, filename + '.txt')
newfilepath = os.path.join(docs, cleansource)
newtxtfilepath = os.path.join(docs, txtfile)
os.rename(oldtxtfilepath, newtxtfilepath)
os.rename(oldfilepath, newfilepath)
ktags = match_keyword(txtfile)
else:
flash('a file already exists for that source, omitting upload')
ktags = match_keyword(txtfile)
pass
error = []
if form.N_BI_checkbx.data == True:
nbi_out = indicatorListofDicts(cleansource)
if nbi_out == None:
nbi_out = []
result = Network_Bro_Intel_dt(
source=cleansourcename,
type_hash=hash_type(NBI_HASH_STRING, cleansourcename),
bro_intel_indicators=nbi_out,
priority=form.priority.data,
created_by=session['name'],
localfile=cleansource,
localtxtfile=txtfile,
localcsvfile=csvfile,
tags=[],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash(
'successfully created network-bro_intel detection object'
)
except IntegrityError:
db.session.rollback()
error_nbi = "source and network-bro_intel detection type object already exists."
error.append(error_nbi)
if form.N_S_checkbx.data == True:
result = Network_Snort_Suricata_dt(
source=cleansourcename,
type_hash=hash_type(NS_HASH_STRING, cleansourcename),
priority=form.priority.data,
created_by=session['name'],
localfile=cleansource,
localtxtfile=txtfile,
localcsvfile=csvfile,
tags=[],
snort_suricata_indicators=[],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash(
'successfully created network-snort_suricata detection object'
)
except IntegrityError:
db.session.rollback()
error_ns = "source and network-snort_suricata detection type object already exists."
error.append(error_ns)
if form.B_Y_checkbx.data == True:
result = Binary_Yara_dt(source=cleansourcename,
type_hash=hash_type(
BY_HASH_STRING, cleansourcename),
priority=form.priority.data,
created_by=session['name'],
localfile=cleansource,
localtxtfile=txtfile,
localcsvfile=csvfile,
tags=[],
bin_yara_indicators=[])
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created binary-yara detection object')
except IntegrityError:
db.session.rollback()
error_by = "source and binary-yara detection type object already exists."
error.append(error_by)
if form.M_Y_checkbx.data == True:
result = Memory_Yara_dt(source=cleansourcename,
type_hash=hash_type(
MY_HASH_STRING, cleansourcename),
priority=form.priority.data,
created_by=session['name'],
localfile=cleansource,
localtxtfile=txtfile,
localcsvfile=csvfile,
tags=[],
mem_yara_indicators=[])
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created memory-yara detection object')
except IntegrityError:
db.session.rollback()
error_by = "source and memory-yara detection type object already exists."
error.append(error_by)
if form.N_BI_checkbx.data == True:
return render_template(
'upload_scrape_form.html',
username=session['name'],
form=form,
multiple_errors=error,
ktags=ktags,
nbi_out=nbi_out,
)
return render_template(
'upload_scrape_form.html',
username=session['name'],
form=form,
multiple_errors=error,
ktags=ktags,
)
else:
error = "need to upload a file"
return render_template(
'upload_scrape_form.html',
form=form,
error=error,
username=session['name'],
)
if request.method == 'GET':
return render_template(
'upload_scrape_form.html',
form=form,
error=error,
username=session['name'],
)
def upload_create():
DOCS = 'app/documents/'
error = None
form = UploadScrape(request.form)
if request.method == 'POST':
file = request.files['uploadpath']
if file:
filename = secure_filename(file.filename)
sourcename = form.source.data
cleansourcename = cleanUrl(sourcename)
cleansource = downloadedFilename(sourcename)
txtfile = cleansource + ".txt"
csvfile = cleansource + ".csv"
if not os.path.exists(DOCS + cleansource):
file.save(os.path.join(docs, filename))
_call_create_txt_file = texttoFile(filename)
oldfilepath = os.path.join(docs, filename)
oldtxtfilepath = os.path.join(docs, filename + '.txt')
newfilepath = os.path.join(docs, cleansource)
newtxtfilepath = os.path.join(docs, txtfile)
os.rename(oldtxtfilepath, newtxtfilepath)
os.rename(oldfilepath, newfilepath)
ktags = match_keyword(txtfile)
else:
flash('a file already exists for that source, omitting upload')
ktags = match_keyword(txtfile)
pass
error = []
if form.N_BI_checkbx.data == True:
nbi_out = indicatorListofDicts(cleansource)
if nbi_out == None:
nbi_out = []
result = Network_Bro_Intel_dt(
source = cleansourcename,
type_hash = hash_type(NBI_HASH_STRING, cleansourcename),
bro_intel_indicators = nbi_out,
priority = form.priority.data,
created_by = session['name'],
localfile = cleansource,
localtxtfile = txtfile,
localcsvfile = csvfile,
tags = [],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created network-bro_intel detection object')
except IntegrityError:
db.session.rollback()
error_nbi = "source and network-bro_intel detection type object already exists."
error.append(error_nbi)
if form.N_S_checkbx.data == True:
result = Network_Snort_Suricata_dt(
source = cleansourcename,
type_hash = hash_type(NS_HASH_STRING, cleansourcename),
priority = form.priority.data,
created_by = session['name'],
localfile = cleansource,
localtxtfile = txtfile,
localcsvfile = csvfile,
tags = [],
snort_suricata_indicators=[],
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created network-snort_suricata detection object')
except IntegrityError:
db.session.rollback()
error_ns = "source and network-snort_suricata detection type object already exists."
error.append(error_ns)
if form.B_Y_checkbx.data == True:
result = Binary_Yara_dt(
source = cleansourcename,
type_hash = hash_type(BY_HASH_STRING, cleansourcename),
priority = form.priority.data,
created_by = session['name'],
localfile = cleansource,
localtxtfile = txtfile,
localcsvfile = csvfile,
tags = [],
bin_yara_indicators=[]
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created binary-yara detection object')
except IntegrityError:
db.session.rollback()
error_by = "source and binary-yara detection type object already exists."
error.append(error_by)
if form.M_Y_checkbx.data == True:
result = Memory_Yara_dt(
source = cleansourcename,
type_hash = hash_type(MY_HASH_STRING, cleansourcename),
priority = form.priority.data,
created_by = session['name'],
localfile = cleansource,
localtxtfile = txtfile,
localcsvfile = csvfile,
tags = [],
mem_yara_indicators=[]
)
result.str_tags = ktags
try:
db.session.add(result)
db.session.commit()
flash('successfully created memory-yara detection object')
except IntegrityError:
db.session.rollback()
error_by = "source and memory-yara detection type object already exists."
error.append(error_by)
if form.N_BI_checkbx.data == True:
return render_template('upload_scrape_form.html',
username=session['name'],
form=form,
multiple_errors=error,
ktags=ktags,
nbi_out=nbi_out,
)
return render_template('upload_scrape_form.html',
username=session['name'],
form=form,
multiple_errors=error,
ktags=ktags,
)
else:
error = "need to upload a file"
return render_template('upload_scrape_form.html',
form=form,
error=error,
username=session['name'],
)
if request.method == 'GET':
return render_template('upload_scrape_form.html',
form=form,
error=error,
username=session['name'],
)