def system(request):
# get config model
model = SystemImporterFileCsvFormbasedConfigModel.objects.get(
system_importer_file_csv_formbased_config_name=
'SystemImporterFileCsvFormbasedConfig')
# form was valid to post
if request.method == "POST":
# call logger
debug_logger(str(request.user), " SYSTEM_IMPORTER_FILE_CSV_BEGAN")
# get systemcsv from request (no submitted file only relevant for tests, normally form enforces file submitting)
check_systemcsv = request.FILES.get('systemcsv', False)
# check request for systemcsv (file submitted)
if check_systemcsv:
# get text out of file (variable results from request object via file upload field)
systemcsv = TextIOWrapper(request.FILES['systemcsv'].file,
encoding=request.encoding)
# read rows out of csv
rows = csv.reader(systemcsv, quotechar="'")
# check file for csv respectively some kind of text file
file_check = check_file(request, rows)
# leave system_importer_file_csv if file check throws errors
if not file_check:
return redirect(reverse('system_list'))
# jump to begin of file again after iterating in file check
systemcsv.seek(0)
""" prepare and start loop """
# set row_counter (needed for logger)
row_counter = 1
# set systems_created_counter (needed for messages)
systems_created_counter = 0
# set systems_updated_counter (needed for messages)
systems_updated_counter = 0
# set systems_skipped_counter (needed for messages)
systems_skipped_counter = 0
# iterate over rows
for row in rows:
# skip first row in case of headline
if row_counter == 1 and model.csv_headline is True:
# autoincrement row counter
row_counter += 1
# leave loop for headline row
continue
# check row for valid system values
continue_system_importer_file_csv = check_row(
request, row, row_counter, model)
# leave loop for this row if there are invalid values
if continue_system_importer_file_csv:
# autoincrement row counter
row_counter += 1
continue
# get system name (decremented by one because index starts with zero: user provides 1 -> first column in CSV has index 0)
system_name = row[model.csv_column_system - 1]
# get all systems with this system_name
systemquery = System.objects.filter(system_name=system_name)
""" check how many systems were returned """
# if there is only one system
if len(systemquery) == 1:
# skip if system already exists (depending on CSV_SKIP_EXISTING_SYSTEM)
if model.csv_skip_existing_system:
# autoincrement counter
systems_skipped_counter += 1
# autoincrement row counter
row_counter += 1
# leave loop
continue
# modify existing system (depending on CSV_SKIP_EXISTING_SYSTEM)
elif not model.csv_skip_existing_system:
# get existing system object
system = System.objects.get(system_name=system_name)
# create form with request data
form = SystemImporterFileCsvFormbasedForm(
request.POST, request.FILES, instance=system)
# change system
if form.is_valid():
# don't save form yet
system = form.save(commit=False)
# change mandatory meta attributes
system.system_modify_time = timezone.now()
system.system_modified_by_user_id = request.user
# save object
system.save()
# change many2many (classic 'form.save_m2m()' would remove existing relationships regardless config)
system = case_attributes_form_based(
system, request.POST.getlist('case'), model)
system = company_attributes_form_based(
system, request.POST.getlist('company'), model)
system = tag_attributes_form_based(
system, request.POST.getlist('tag'), model)
# change ip addresses
if model.csv_choice_ip:
system = ip_attributes(system, request, row,
row_counter, model)
# autoincrement systems_updated_counter
systems_updated_counter += 1
# call logger
system.logger(
str(request.user),
" SYSTEM_IMPORTER_FILE_CSV_SYSTEM_MODIFIED")
# if there is more than one system
elif len(systemquery) > 1:
messages.error(
request, "System " + system_name +
" already exists multiple times. Nothing was changed for this system."
)
# if there is no system
else:
# create form with request data
form = SystemImporterFileCsvFormbasedForm(
request.POST, request.FILES)
# create system
if form.is_valid():
# create new system object
system = System()
# don't save form yet
system = form.save(commit=False)
# add system_name from csv
system.system_name = system_name
# add mandatory meta attributes
system.system_modify_time = timezone.now()
system.system_created_by_user_id = request.user
system.system_modified_by_user_id = request.user
# save object
system.save()
# add many2many
system = case_attributes_form_based(
system, request.POST.getlist('case'), model)
system = company_attributes_form_based(
system, request.POST.getlist('company'), model)
system = tag_attributes_form_based(
system, request.POST.getlist('tag'), model)
# add ip addresses
if model.csv_choice_ip:
system = ip_attributes(system, request, row,
row_counter, model)
# autoincrement systems_created_counter
systems_created_counter += 1
# call logger
system.logger(
str(request.user),
" SYSTEM_IMPORTER_FILE_CSV_SYSTEM_CREATED")
# autoincrement row counter
row_counter += 1
# check request for systemcsv (file not submitted)
else:
# get empty form with default values
form = SystemImporterFileCsvFormbasedForm(initial={
'systemstatus': 2,
'analysisstatus': 1,
})
# show form again
return render(
request,
'dfirtrack_main/system/system_importer_file_csv_form_based.html',
{
'form': form,
})
# call final messages
final_messages(request, systems_created_counter,
systems_updated_counter, systems_skipped_counter)
# call logger
debug_logger(str(request.user), " SYSTEM_IMPORTER_FILE_CSV_END")
return redirect(reverse('system_list'))
else:
# get config model
model = SystemImporterFileCsvFormbasedConfigModel.objects.get(
system_importer_file_csv_formbased_config_name=
'SystemImporterFileCsvFormbasedConfig')
# check config before showing form
stop_system_importer_file_csv = check_config(request, model)
# leave system_importer_file_csv if variables caused errors
if stop_system_importer_file_csv:
return redirect(reverse('system_list'))
# show warning if existing systems will be updated
if not model.csv_skip_existing_system:
messages.warning(request,
'WARNING: Existing systems will be updated!')
# get empty form with default values
form = SystemImporterFileCsvFormbasedForm(initial={
'systemstatus': 2,
'analysisstatus': 1,
})
# call logger
debug_logger(str(request.user), " SYSTEM_IMPORTER_FILE_CSV_ENTERED")
# show form
return render(
request,
'dfirtrack_main/system/system_importer_file_csv_form_based.html', {
'form': form,
})
def system_handler(request=None, uploadfile=False):
# get config model
model = SystemImporterFileCsvConfigModel.objects.get(
system_importer_file_csv_config_name='SystemImporterFileCsvConfig')
""" set username for logger and object """
# if function was called from 'system_instant' and 'system_upload'
if request:
# get user for object
csv_import_user = request.user
# get user for logger
logger_username = str(request.user)
# if function was called from 'system_cron'
else:
# get user for object
csv_import_user = model.csv_import_username
# get user for logger
logger_username = model.csv_import_username.username
""" start system importer """
# get starttime
starttime = timezone.now().strftime('%Y-%m-%d %H:%M:%S')
# call logger
debug_logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_START')
# create lock tags
create_lock_tags(model)
""" file handling """
# file was uploaded via form (called via 'system_upload')
if uploadfile:
# text object can not be passed as argument from 'system_upload'
systemcsv = TextIOWrapper(request.FILES['systemcsv'].file,
encoding=request.encoding)
# file was fetched from file system (called via 'system_instant' or 'system_cron')
else:
# build csv file path
csv_import_file = model.csv_import_path + '/' + model.csv_import_filename
# open file
systemcsv = open(csv_import_file, 'r')
# get field delimiter from config
if model.csv_field_delimiter == 'field_comma':
delimiter = ','
elif model.csv_field_delimiter == 'field_semicolon':
delimiter = ';'
# get text quotechar from config
if model.csv_text_quote == 'text_double_quotation_marks':
quotechar = '"'
elif model.csv_text_quote == 'text_single_quotation_marks':
quotechar = "'"
# read rows out of csv
rows = csv.reader(systemcsv, delimiter=delimiter, quotechar=quotechar)
""" check file """
# if function was called from 'system_instant' and 'system_upload'
if request:
# check file for csv respectively some kind of text file
file_check = check_content_file_type(rows, logger_username, request)
# if function was called from 'system_cron'
else:
# check file for csv respectively some kind of text file
file_check = check_content_file_type(rows, logger_username)
# leave system_importer_file_csv if file check throws errors
if not file_check:
# close file
systemcsv.close()
# return to calling function 'csv.system_cron' or 'csv.system_instant' or 'csv.system_upload'
return
# jump to begin of file again after iterating in file check
systemcsv.seek(0)
""" prepare loop """
# set row_counter (needed for logger)
row_counter = 1
# set systems_created_counter (needed for logger)
systems_created_counter = 0
# set systems_updated_counter (needed for logger)
systems_updated_counter = 0
# set systems_multiple_counter (needed for logger)
systems_multiple_counter = 0
# create empty list (needed for logger)
systems_multiple_list = []
# set systems_skipped_counter (needed for logger)
systems_skipped_counter = 0
""" start loop """
# iterate over rows
for row in rows:
""" skip headline if necessary """
# check for first row and headline condition
if row_counter == 1 and model.csv_headline:
# autoincrement row counter
row_counter += 1
# leave loop for headline row
continue
""" filter for systems """
# if function was called from 'system_instant' and 'system_upload'
if request:
# check system_name for valid value
stop_system_importer_file_csv = check_system_name(
model, row, row_counter, request)
# if function was called from 'system_cron'
else:
# check system_name for valid value
stop_system_importer_file_csv = check_system_name(
model, row, row_counter)
# leave loop if system_name caused errors
if stop_system_importer_file_csv:
# autoincrement counter
systems_skipped_counter += 1
# autoincrement row counter
row_counter += 1
# leave loop
continue
# get system name (for domain name comparison)
system_name = row[model.csv_column_system - 1]
# TODO: [logic] add option which attributes are used for filtering?
# TODO: [logic] (like domain, dnsname, company)
# TODO: [logic] e.g. 'csv_identification_dnsname'
# get all systems
systemquery = System.objects.filter(system_name=system_name, )
""" check how many systems were returned """
# if there is only one system -> modify system
if len(systemquery) == 1:
# skip if system already exists (depending on csv_skip_existing_system)
if model.csv_skip_existing_system:
# autoincrement counter
systems_skipped_counter += 1
# autoincrement row counter
row_counter += 1
# leave loop
continue
# TODO: [logic] add option which attributes are used for filtering?
# TODO: [logic] (like domain, dnsname, company)
# TODO: [logic] e.g. 'csv_identification_dnsname'
# get existing system object
system = System.objects.get(system_name=system_name, )
# change mandatory meta attributes
system.system_modify_time = timezone.now()
system.system_modified_by_user_id = csv_import_user
# set value for already existing system (modify system)
system_created = False
# if function was called from 'system_instant' and 'system_upload'
if request:
# add foreign key relationships to system
system = add_fk_attributes(system, system_created, model, row,
row_counter, request)
# if function was called from 'system_cron'
else:
# add foreign key relationships to system
system = add_fk_attributes(system, system_created, model, row,
row_counter)
# save object
system.save()
# if function was called from 'system_instant' and 'system_upload'
if request:
# add many2many relationships to system
system = add_many2many_attributes(system, system_created,
model, row, row_counter,
request)
# if function was called from 'system_cron'
else:
# add many2many relationships to system
system = add_many2many_attributes(system, system_created,
model, row, row_counter)
# autoincrement systems_updated_counter
systems_updated_counter += 1
# call logger
system.logger(logger_username,
' SYSTEM_IMPORTER_FILE_CSV_SYSTEM_MODIFIED')
# if there is more than one system
elif len(systemquery) > 1:
# add system name to list
systems_multiple_list.append(system_name)
# autoincrement systems_multiple_counter
systems_multiple_counter += 1
# call logger
warning_logger(
logger_username,
f' SYSTEM_IMPORTER_FILE_CSV_MULTIPLE_SYSTEMS System:{system_name}'
)
# if there is no system -> create system
else:
# create new system object
system = System()
# add system_name from csv
system.system_name = system_name
# add mandatory meta attributes
system.system_modify_time = timezone.now()
system.system_created_by_user_id = csv_import_user
system.system_modified_by_user_id = csv_import_user
# set value for new system (create system)
system_created = True
# if function was called from 'system_instant' and 'system_upload'
if request:
# add foreign key relationships to system
system = add_fk_attributes(system, system_created, model, row,
row_counter, request)
# if function was called from 'system_cron'
else:
# add foreign key relationships to system
system = add_fk_attributes(system, system_created, model, row,
row_counter)
# save object
system.save()
# if function was called from 'system_instant' and 'system_upload'
if request:
# add many2many relationships to system
system = add_many2many_attributes(system, system_created,
model, row, row_counter,
request)
# if function was called from 'system_cron'
else:
# add many2many relationships to system
system = add_many2many_attributes(system, system_created,
model, row, row_counter)
# autoincrement systems_created_counter
systems_created_counter += 1
# call logger
system.logger(logger_username,
' SYSTEM_IMPORTER_FILE_CSV_SYSTEM_CREATED')
# autoincrement row counter
row_counter += 1
# close file
systemcsv.close()
""" finish system importer """
# get endtime
endtime = timezone.now().strftime('%Y-%m-%d %H:%M:%S')
# if function was called from 'system_instant' and 'system_upload'
if request:
# call final messages
final_messages(
systems_created_counter,
systems_updated_counter,
systems_skipped_counter,
systems_multiple_counter,
systems_multiple_list,
request,
)
# if function was called from 'system_cron'
else:
# call final messages
final_messages_cron(
systems_created_counter,
systems_updated_counter,
systems_skipped_counter,
systems_multiple_counter,
systems_multiple_list,
starttime,
endtime,
)
# call logger
info_logger(
logger_username, f' SYSTEM_IMPORTER_FILE_CSV_STATUS'
f' created:{systems_created_counter}'
f'|updated:{systems_updated_counter}'
f'|skipped:{systems_skipped_counter}'
f'|multiple:{systems_multiple_counter}')
# call logger
debug_logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_END')
# return to calling function 'csv.system_cron' or 'csv.system_instant' or 'csv.system_upload'
return