def ips_save(request, system, lines):
# iterate over lines
for line in lines:
# skip empty lines
if line == '':
# call logger
warning_logger(str(request.user), ' SYSTEM_ADD_IP_EMPTY_LINE')
messages.error(request, 'Empty line instead of IP was provided')
continue
# check line for ip
try:
ipaddress.ip_address(line)
except ValueError:
# call logger
warning_logger(str(request.user), ' SYSTEM_ADD_IP_NO_IP')
messages.error(request, 'Provided string was no IP')
continue
# create ip
ip, created = Ip.objects.get_or_create(ip_ip=line)
# call logger
if created == True:
ip.logger(str(request.user), ' SYSTEM_ADD_IP_CREATED')
messages.success(request, 'IP created')
else:
messages.warning(request, 'IP already exists in database')
# save ip for system
system.ip.add(ip)
def system_creator_async(request_post, request_user):
""" function to create many systems at once """
# call logger
debug_logger(str(request_user), " SYSTEM_CREATOR_BEGIN")
# exctract lines from systemlist (list results from request object via large text area)
lines = request_post.get('systemlist').splitlines()
# iterate over lines
for line in lines:
# skip emtpy lines
if line == '':
warning_logger(str(request_user), " SYSTEM_CREATOR_ROW_EMPTY")
continue
# check line for length of string
if len(line) > 50:
warning_logger(str(request_user), " SYSTEM_CREATOR_LONG_STRING")
continue
# check for existence of system
system = System.objects.filter(system_name = line)
if system.count() > 0:
# call logger
error_logger(str(request_user), " SYSTEM_CREATOR_SYSTEM_EXISTS " + "system_name:" + line)
# leave this loop because system with this systemname already exists
continue
# create form with request data
form = SystemCreatorForm(request_post)
# create system
if form.is_valid():
# don't save form yet
system = form.save(commit=False)
# set system_name
system.system_name = line
# set auto values
system.system_created_by_user_id = request_user
system.system_modified_by_user_id = request_user
system.system_modify_time = timezone.now()
# save object
system.save()
# save manytomany
form.save_m2m()
# call logger
system.logger(str(request_user), ' SYSTEM_CREATOR_EXECUTED')
# call logger
debug_logger(str(request_user), " SYSTEM_CREATOR_END")
def check_and_create_ip(ip_ip, model, row_counter, request=None):
""" check IPs for valid values """
""" set username for logger """
# if function was called from 'system_instant' and 'system_upload'
if request:
logger_username = str(request.user)
# if function was called from 'system_cron'
else:
logger_username = model.csv_import_username.username
""" perform checks """
# value is an IP
try:
# check ip column for IP(s)
ipaddress.ip_address(ip_ip)
# create ip
ip, created = Ip.objects.get_or_create(ip_ip=ip_ip)
# IP was created
if created:
# call logger
ip.logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_IP_CREATED')
# return IP to 'csv_attributes_add.add_many2many_attributes'
return ip
# value is not an IP
except ValueError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(
request,
f'Value for IP address in row {row_counter} was not a valid IP address.'
)
# call logger
warning_logger(
logger_username,
f' SYSTEM_IMPORTER_FILE_CSV_IP_COLUMN row_{row_counter}:invalid_ip'
)
# return nothing to 'csv_attributes_add.add_many2many_attributes'
return None
def check_and_create_ip(column_ip, request, row_counter):
# check ip column for ip
try:
ipaddress.ip_address(column_ip)
except ValueError:
messages.error(request, "Value for ip address in row " + str(row_counter) + " was not a valid IP address.")
# call logger
warning_logger(str(request.user), " SYSTEM_IMPORTER_FILE_CSV_IP_COLUMN " + "row_" + str(row_counter) + ":invalid_ip")
return None
# create ip
ip, created = Ip.objects.get_or_create(ip_ip=column_ip)
if created == True:
ip.logger(str(request.user), " SYSTEM_IMPORTER_FILE_CSV_IP_CREATED")
return ip
def read_or_create_mkdocs_yml(username, mkdconfpath):
# open mkdocs.yml for reading
try:
mkdconffile = open(mkdconfpath)
# read YAML to dict
mkdconfdict = yaml.safe_load(mkdconffile)
# close mkdocs.yml
mkdconffile.close()
except FileNotFoundError:
# call logger
warning_logger(username, " SYSTEM_EXPORTER_MARKDOWN no mkdocs.yml found")
# set empty dummydict for non-existing file to make code work as usual (just for the first execution because afterwards 'mkdocs.yml' will be there)
mkdconfdict = {'pages': [{'Systems': []}]}
return mkdconfdict
def check_row(request, row, row_counter, model):
""" check some values of csv rows """
# reset continue condition
continue_system_importer_file_csv = False
# check system column for empty string
if not row[model.csv_column_system - 1]:
messages.error(request, "Value for system in row " + str(row_counter) + " was an empty string. System not created.")
# call logger
warning_logger(str(request.user), " SYSTEM_IMPORTER_FILE_CSV_SYSTEM_COLUMN " + "row_" + str(row_counter) + ":empty_column")
continue_system_importer_file_csv = True
# check system column for length of string
if len(row[model.csv_column_system - 1]) > 50:
messages.error(request, "Value for system in row " + str(row_counter) + " was too long. System not created.")
# call logger
warning_logger(str(request.user), " SYSTEM_IMPORTER_FILE_CSV_SYSTEM_COLUMN " + "row_" + str(row_counter) + ":long_string")
continue_system_importer_file_csv = True
return continue_system_importer_file_csv
def check_config(request, model):
""" check variables of dfirtrack.config """
# reset stop condition
stop_system_importer_file_csv = False
# check CSV_COLUMN_SYSTEM for value
if not 1<= model.csv_column_system <= 256:
messages.error(request, "`CSV_COLUMN_SYSTEM` is outside the allowed range. Check config!")
# call logger
warning_logger(str(request.user), " SYSTEM_IMPORTER_FILE_CSV variable CSV_COLUMN_SYSTEM out of range")
stop_system_importer_file_csv = True
# check CSV_COLUMN_IP for value
if not 1<= model.csv_column_ip <= 256:
messages.error(request, "`CSV_COLUMN_IP` is outside the allowed range. Check config!")
# call logger
warning_logger(str(request.user), " SYSTEM_IMPORTER_FILE_CSV variable CSV_COLUMN_IP out of range")
stop_system_importer_file_csv = True
# create final message and log
if stop_system_importer_file_csv:
messages.warning(request, "Nothing was changed.")
# call logger
warning_logger(str(request.user), " SYSTEM_IMPORTER_FILE_CSV_ENDED_WITH_ERRORS")
return stop_system_importer_file_csv
def check_config(request):
""" check variables in config """
# get config model
model = SystemExporterMarkdownConfigModel.objects.get(
system_exporter_markdown_config_name='SystemExporterMarkdownConfig')
# reset stop condition
stop_exporter_markdown = False
# check MARKDOWN_PATH for empty string
if not model.markdown_path:
messages.error(
request, "`MARKDOWN_PATH` contains an emtpy string. Check config!")
# call logger
warning_logger(
str(request.user),
" EXPORTER_MARKDOWN variable MARKDOWN_PATH empty string")
stop_exporter_markdown = True
# check MARKDOWN_PATH for existence in file system (check only if it actually is a non-empty string)
if model.markdown_path:
if not os.path.isdir(model.markdown_path):
messages.error(
request,
"`MARKDOWN_PATH` does not exist in file system. Check config or filesystem!"
)
# call logger
warning_logger(
str(request.user),
" EXPORTER_MARKDOWN path MARKDOWN_PATH not existing")
stop_exporter_markdown = True
# check MARKDOWN_PATH for write permission (check only if it actually is a non-empty string)
if model.markdown_path:
if not os.access(model.markdown_path, os.W_OK):
messages.error(
request,
"`MARKDOWN_PATH` is not writeable. Check config or filesystem!"
)
# call logger
warning_logger(
str(request.user),
" EXPORTER_MARKDOWN path MARKDOWN_PATH not writeable")
stop_exporter_markdown = True
return stop_exporter_markdown
def add_fk_attributes(system, system_created, model, row, row_counter, request=None):
""" add foreign key relationships to system """
""" set username for logger """
# if function was called from 'system_instant' and 'system_upload'
if request:
logger_username = str(request.user)
# if function was called from 'system_cron'
else:
logger_username = model.csv_import_username.username
""" systemstatus (tagfree is set with tags in 'csv_attributes_add.add_many2many_attributes()') """
# set systemstatus for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_systemstatus):
# set default systemstatus for new system
if system_created:
# set systemstatus for new system
system.systemstatus = model.csv_default_systemstatus
# change systemstatus for existing system if not locked
else:
# get lockstatus
tag_lock_systemstatus = Tag.objects.get(tag_name = model.csv_tag_lock_systemstatus)
# check for lockstatus in all tags of system
if tag_lock_systemstatus not in system.tag.all():
# change to default systemstatus for existing system
system.systemstatus = model.csv_default_systemstatus
""" analysisstatus (tagfree is set with tags in 'csv_attributes_add.add_many2many_attributes()') """
# set analysisstatus for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_analysisstatus):
# set default analysisstatus for new system
if system_created:
# set analysisstatus for new system
system.analysisstatus = model.csv_default_analysisstatus
# change analysisstatus for existing system if not locked
else:
# get lockstatus
tag_lock_analysisstatus = Tag.objects.get(tag_name = model.csv_tag_lock_analysisstatus)
# check for lockstatus in all tags of system
if tag_lock_analysisstatus not in system.tag.all():
# change to default analysisstatus for existing system
system.analysisstatus = model.csv_default_analysisstatus
""" dnsname """
# set dnsname for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_dnsname):
# get dnsname from CSV
if model.csv_choice_dnsname:
# check for index error
try:
# get dnsname from CSV column
dnsname_name = row[model.csv_column_dnsname - 1]
# check for empty string
if dnsname_name:
# value is valid
try:
# get or create dnsname
dnsname, created = Dnsname.objects.get_or_create(dnsname_name = dnsname_name)
# call logger if created
if created:
dnsname.logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_DNSNAME_CREATED')
# value is not valid
except DataError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Value for DNS name in row {row_counter} was not a valid value.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_DNSNAME_COLUMN row_{row_counter}:invalid_dnsname')
# set empty value
dnsname = None
# string was empty
else:
# set empty value (field is empty)
dnsname = None
# index out of range
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Index for DNS name in row {row_counter} was out of range.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_DNSNAME_COLUMN row_{row_counter}:out_of_range')
# set empty value
dnsname = None
# get dnsname from DB
elif model.csv_default_dnsname:
dnsname = model.csv_default_dnsname
# set empty value (removes for existing system if neither CSV nor DB is chosen, does nothing for new system)
else:
dnsname = None
# set dnsname for system
system.dnsname = dnsname
""" domain """
# set domain for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_domain):
# get domain from CSV
if model.csv_choice_domain:
# check for index error
try:
# get domain from CSV column
domain_name = row[model.csv_column_domain - 1]
# check for empty string and compare to system name (when queried with local account, hostname is returned under some circumstances depending on tool)
if domain_name and domain_name != system.system_name:
# value is valid
try:
# get or create domain
domain, created = Domain.objects.get_or_create(domain_name = domain_name)
# call logger if created
if created:
domain.logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_DOMAIN_CREATED')
# value is not valid
except DataError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Value for domain in row {row_counter} was not a valid value.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_DOMAIN_COLUMN row_{row_counter}:invalid_domain')
# set empty value
domain = None
# string was empty or same as system_name
else:
# set empty value (field is empty)
domain = None
# index out of range
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Index for domain in row {row_counter} was out of range.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_DOMAIN_COLUMN row_{row_counter}:out_of_range')
# set empty value
domain = None
# get domain from DB
elif model.csv_default_domain:
domain = model.csv_default_domain
# set empty value (removes for existing system if neither CSV nor DB is chosen, does nothing for new system)
else:
domain = None
# set domain for system
system.domain = domain
""" location """
# set location for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_location):
# get location from CSV
if model.csv_choice_location:
# check for index error
try:
# get location from CSV column
location_name = row[model.csv_column_location - 1]
# check for empty string
if location_name:
# value is valid
try:
# get or create location
location, created = Location.objects.get_or_create(location_name = location_name)
# call logger if created
if created:
location.logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_LOCATION_CREATED')
# value is not valid
except DataError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Value for location in row {row_counter} was not a valid value.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_LOCATION_COLUMN row_{row_counter}:invalid_location')
# set empty value
location = None
# string was empty
else:
# set empty value (field is empty)
location = None
# index out of range
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Index for location in row {row_counter} was out of range.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_LOCATION_COLUMN row_{row_counter}:out_of_range')
# set empty value
location = None
# get location from DB
elif model.csv_default_location:
location = model.csv_default_location
# set empty value (removes for existing system if neither CSV nor DB is chosen, does nothing for new system)
else:
location = None
# set location for system
system.location = location
""" os """
# set os for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_os):
# get os from CSV
if model.csv_choice_os:
# check for index error
try:
# get os from CSV column
os_name = row[model.csv_column_os - 1]
# check for empty string
if os_name:
# value is valid
try:
# get or create os
os, created = Os.objects.get_or_create(os_name = os_name)
# call logger if created
if created:
os.logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_OS_CREATED')
# value is not valid
except DataError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Value for OS in row {row_counter} was not a valid value.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_OS_COLUMN row_{row_counter}:invalid_os')
# set empty value
os = None
# string was empty
else:
# set empty value (field is empty)
os = None
# index out of range
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Index for OS in row {row_counter} was out of range.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_OS_COLUMN row_{row_counter}:out_of_range')
# set empty value
os = None
# get os from DB
elif model.csv_default_os:
os = model.csv_default_os
# set empty value (removes for existing system if neither CSV nor DB is chosen, does nothing for new system)
else:
os = None
# set os for system
system.os = os
""" reason """
# set reason for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_reason):
# get reason from CSV
if model.csv_choice_reason:
# check for index error
try:
# get reason from CSV column
reason_name = row[model.csv_column_reason - 1]
# check for empty string
if reason_name:
# value is valid
try:
# get or create reason
reason, created = Reason.objects.get_or_create(reason_name = reason_name)
# call logger if created
if created:
reason.logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_REASON_CREATED')
# value is not valid
except DataError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Value for reason in row {row_counter} was not a valid value.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_REASON_COLUMN row_{row_counter}:invalid_reason')
# set empty value
reason = None
# string was empty
else:
# set empty value (field is empty)
reason = None
# index out of range
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Index for reason in row {row_counter} was out of range.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_REASON_COLUMN row_{row_counter}:out_of_range')
# set empty value
reason = None
# get reason from DB
elif model.csv_default_reason:
reason = model.csv_default_reason
# set empty value (removes for existing system if neither CSV nor DB is chosen, does nothing for new system)
else:
reason = None
# set reason for system
system.reason = reason
""" recommendation """
# set recommendation for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_recommendation):
# get recommendation from CSV
if model.csv_choice_recommendation:
# check for index error
try:
# get recommendation from CSV column
recommendation_name = row[model.csv_column_recommendation - 1]
# check for empty string
if recommendation_name:
# value is valid
try:
# get or create recommendation
recommendation, created = Recommendation.objects.get_or_create(recommendation_name = recommendation_name)
# call logger if created
if created:
recommendation.logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_RECOMMENDATION_CREATED')
# value is not valid
except DataError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Value for recommendation in row {row_counter} was not a valid value.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_RECOMMENDATION_COLUMN row_{row_counter}:invalid_recommendation')
# set empty value
recommendation = None
# string was empty
else:
# set empty value (field is empty)
recommendation = None
# index out of range
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Index for recommendation in row {row_counter} was out of range.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_RECOMMENDATION_COLUMN row_{row_counter}:out_of_range')
# set empty value
recommendation = None
# get recommendation from DB
elif model.csv_default_recommendation:
recommendation = model.csv_default_recommendation
# set empty value (removes for existing system if neither CSV nor DB is chosen, does nothing for new system)
else:
recommendation = None
# set recommendation for system
system.recommendation = recommendation
""" serviceprovider """
# set serviceprovider for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_serviceprovider):
# get serviceprovider from CSV
if model.csv_choice_serviceprovider:
# check for index error
try:
# get serviceprovider from CSV column
serviceprovider_name = row[model.csv_column_serviceprovider - 1]
# check for empty string
if serviceprovider_name:
# value is valid
try:
# get or create serviceprovider
serviceprovider, created = Serviceprovider.objects.get_or_create(serviceprovider_name = serviceprovider_name)
# call logger if created
if created:
serviceprovider.logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_SERVICEPROVIDER_CREATED')
# value is not valid
except DataError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Value for serviceprovider in row {row_counter} was not a valid value.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_SERVICEPROVIDER_COLUMN row_{row_counter}:invalid_serviceprovider')
# set empty value
serviceprovider = None
# string was empty
else:
# set empty value (field is empty)
serviceprovider = None
# index out of range
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Index for serviceprovider in row {row_counter} was out of range.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_SERVICEPROVIDER_COLUMN row_{row_counter}:out_of_range')
# set empty value
serviceprovider = None
# get serviceprovider from DB
elif model.csv_default_serviceprovider:
serviceprovider = model.csv_default_serviceprovider
# set empty value (removes for existing system if neither CSV nor DB is chosen, does nothing for new system)
else:
serviceprovider = None
# set serviceprovider for system
system.serviceprovider = serviceprovider
""" systemtype """
# set systemtype for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_systemtype):
# get systemtype from CSV
if model.csv_choice_systemtype:
# check for index error
try:
# get systemtype from CSV column
systemtype_name = row[model.csv_column_systemtype - 1]
# check for empty string
if systemtype_name:
# value is valid
try:
# get or create systemtype
systemtype, created = Systemtype.objects.get_or_create(systemtype_name = systemtype_name)
# call logger if created
if created:
systemtype.logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_SYSTEMTYPE_CREATED')
# value is not valid
except DataError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Value for systemtype in row {row_counter} was not a valid value.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_SYSTEMTYPE_COLUMN row_{row_counter}:invalid_systemtype')
# set empty value
systemtype = None
# string was empty
else:
# set empty value (field is empty)
systemtype = None
# index out of range
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Index for systemtype in row {row_counter} was out of range.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_SYSTEMTYPE_COLUMN row_{row_counter}:out_of_range')
# set empty value
systemtype = None
# get systemtype from DB
elif model.csv_default_systemtype:
systemtype = model.csv_default_systemtype
# set empty value (removes for existing system if neither CSV nor DB is chosen, does nothing for new system)
else:
systemtype = None
# set systemtype for system
system.systemtype = systemtype
# return system with foreign key relations to 'csv_main.system_handler'
return system
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_CASE_COLUMN row_{row_counter}:invalid_case')
# set empty value
case = None
# only add case to system if one of the previous checks was successful
if case:
# set case for system
system.case.add(case)
# index out of range
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Index for case in row {row_counter} was out of range.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_CASE_COLUMN row_{row_counter}:out_of_range')
# get case from DB
elif model.csv_default_case:
cases = model.csv_default_case
for case in cases.all():
# add case to system
system.case.add(case)
""" company """
# set company for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_company):
# remove companies if not new system
if not system_created:
def check_system_name(model, row, row_counter, request=None):
""" check system name for valid value """
# reset continue condition
stop_system_importer_file_csv = False
""" set username for logger """
# if function was called from 'system_instant' and 'system_upload'
if request:
logger_username = str(request.user)
# if function was called from 'system_cron'
else:
logger_username = model.csv_import_username.username
""" perform checks (for system_name, return immediately) """
# check for index error
try:
system_name = row[model.csv_column_system - 1]
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(
request,
f'Index for system in row {row_counter} was out of range. System not created.'
)
# call logger
warning_logger(
logger_username,
f' SYSTEM_IMPORTER_FILE_CSV_SYSTEM_COLUMN row_{row_counter}:out_of_range'
)
# set stop condition
stop_system_importer_file_csv = True
# return stop condition to 'csv_main.system_handler'
return stop_system_importer_file_csv
""" perform checks (remaining checks, need system_name) """
# check system column for empty string
if not system_name:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(
request,
f'Value for system in row {row_counter} was an empty string. System not created.'
)
# call logger
warning_logger(
logger_username,
f' SYSTEM_IMPORTER_FILE_CSV_SYSTEM_COLUMN row_{row_counter}:empty_column'
)
# set stop condition
stop_system_importer_file_csv = True
# check system column for length of string
if len(system_name) > 50:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(
request,
f'Value for system in row {row_counter} was too long. System not created.'
)
# call logger
warning_logger(
logger_username,
f' SYSTEM_IMPORTER_FILE_CSV_SYSTEM_COLUMN row_{row_counter}:long_string'
)
# set stop condition
stop_system_importer_file_csv = True
# return stop condition to 'csv_main.system_handler'
return stop_system_importer_file_csv
def reportitems(request):
""" this function checks for every system the existence of a markdown file with information about the and imports the content of this file as reportitem for the corresponding system """
# call logger
debug_logger(str(request.user), " REPORTITEM_FILESYSTEM_IMPORTER_BEGIN")
# check whether REPORTITEMS_FILESYSTEMPATH is defined in `dfirtrack.config`
if reportitems_filesystempath == '':
# call logger
error_logger(str(request.user),
" REPORTITEMS_FILESYSTEMPATH_VARIABLE_UNDEFINED")
messages.error(
request,
"The variable REPORTITEMS_FILESYSTEMPATH seems to be undefined. Check `dfirtrack.config`!"
)
# leave importer
return redirect('/systems/')
# check whether REPORTITEMS_FILESYSTEMPATH points to non-existing directory
if not os.path.isdir(reportitems_filesystempath):
# call logger
error_logger(str(request.user),
" REPORTITEMS_FILESYSTEM_IMPORTER_WRONG_PATH")
messages.error(
request,
"The variable REPORTITEMS_FILESYSTEMPATH points to a non-existing directory. Check `dfirtrack.config`!"
)
# leave importer
return redirect('/systems/')
# check whether REPORTITEMS_HEADLINE is defined in `dfirtrack.config`
if reportitems_headline == '':
# call logger
error_logger(str(request.user),
" REPORTITEMS_HEADLINE_VARIABLE_UNDEFINED")
messages.error(
request,
"The variable REPORTITEMS_HEADLINE seems to be undefined. Check `dfirtrack.config`!"
)
# leave importer
return redirect('/systems/')
# check whether REPORTITEMS_SUBHEADLINE is defined in `dfirtrack.config`
if reportitems_subheadline == '':
# call logger
error_logger(str(request.user),
" REPORTITEMS_SUBHEADLINE_VARIABLE_UNDEFINED")
messages.error(
request,
"The variable REPORTITEMS_SUBHEADLINE seems to be undefined. Check `dfirtrack.config`!"
)
# leave importer
return redirect('/systems/')
# check whether REPORTITEMS_DELETE is defined in `dfirtrack.config`
if not isinstance(reportitems_delete, bool):
# call logger
error_logger(str(request.user),
" REPORTITEMS_DELETE_VARIABLE_UNDEFINED")
messages.error(
request,
"The variable REPORTITEMS_DELETE seems to be undefined or not a boolean. Check `dfirtrack.config`!"
)
# leave importer
return redirect('/systems/')
# get all system objects
systems = System.objects.all()
# create headline if it does not exist
headline, created = Headline.objects.get_or_create(
headline_name=reportitems_headline)
if created == True:
# call logger
headline.logger(str(request.user),
" REPORTITEMS_FILESYSTEM_IMPORTER_HEADLINE_CREATED")
# set counter for non-existing files (needed for messages)
nofile_found_counter = 0
# set counter for created reportitems (needed for messages)
reportitems_created_counter = 0
# set counter for modified reportitems (needed for messages)
reportitems_modified_counter = 0
# set counter for deleted reportitems (needed for messages)
reportitems_deleted_counter = 0
# iterate over systems
for system in systems:
# create path for reportfile
reportpath = reportitems_filesystempath + "/" + system.system_name + ".md"
# check whether a file is existing for this system
if not os.path.isfile(reportpath):
# call logger
warning_logger(
str(request.user),
" REPORTITEMS_FILESYSTEM_IMPORTER_NO_FILE system_name:" +
system.system_name)
# autoincrement counter
nofile_found_counter += 1
# check whether already existing reportitem for this system should be deleted if no file was provided
if reportitems_delete:
# delete already existing reportitem for this system if no file was provided
try:
reportitem = Reportitem.objects.get(
system=system,
headline=headline,
reportitem_subheadline=reportitems_subheadline)
# call logger (before deleting instance)
reportitem.logger(
str(request.user),
" REPORTITEMS_FILESYSTEM_IMPORTER_REPORTITEM_DELETED")
reportitem.delete()
# autoincrement counter
reportitems_deleted_counter += 1
except Reportitem.DoesNotExist:
pass
# continue with next system
continue
# create reportitem if it does not exist (get_or_create won't work in this context because of needed user objects for saving)
try:
reportitem = Reportitem.objects.get(
system=system,
headline=headline,
reportitem_subheadline=reportitems_subheadline)
reportitems_modified_counter += 1
except Reportitem.DoesNotExist:
reportitem = Reportitem()
reportitems_created_counter += 1
reportitem.system = system
reportitem.headline = headline
reportitem.reportitem_subheadline = reportitems_subheadline
reportitem.reportitem_created_by_user_id = request.user
# open file
reportfile = open(reportpath, "r")
# add changing values (existing reportitem_note will be overwritten)
reportitem.reportitem_note = reportfile.read()
# close file
reportfile.close()
reportitem.reportitem_modified_by_user_id = request.user
reportitem.save()
# call logger
reportitem.logger(
str(request.user),
" REPORTITEMS_FILESYSTEM_IMPORTER_REPORTITEM_CREATED_OR_MODIFIED")
# call final messages
if nofile_found_counter > 0:
if nofile_found_counter == 1:
messages.warning(
request, "No file was found for " + str(nofile_found_counter) +
" system.")
else:
messages.warning(
request, "No files were found for " +
str(nofile_found_counter) + " systems.")
if reportitems_created_counter > 0:
if reportitems_created_counter == 1:
messages.success(
request,
str(reportitems_created_counter) + ' reportitem was created.')
else:
messages.success(
request,
str(reportitems_created_counter) +
' reportitems were created.')
if reportitems_modified_counter > 0:
if reportitems_modified_counter == 1:
messages.success(
request,
str(reportitems_modified_counter) +
' reportitem was modified.')
else:
messages.success(
request,
str(reportitems_modified_counter) +
' reportitems were modified.')
if reportitems_deleted_counter > 0:
if reportitems_deleted_counter == 1:
messages.success(
request,
str(reportitems_deleted_counter) + ' reportitem was deleted.')
else:
messages.success(
request,
str(reportitems_deleted_counter) +
' reportitems were deleted.')
return redirect('/systems/')
def system_creator_async(request_post, request_user):
""" function to create many systems at once """
# call logger
debug_logger(str(request_user), ' SYSTEM_CREATOR_BEGIN')
# exctract lines from systemlist (list results from request object via large text area)
lines = request_post.get('systemlist').splitlines()
# count lines (needed for messages)
number_of_lines = len(lines)
# set systems_created_counter (needed for messages)
systems_created_counter = 0
# set systems_skipped_counter (needed for messages)
systems_skipped_counter = 0
# set lines_faulty_counter (needed for messages)
lines_faulty_counter = 0
# create empty list (needed for messages)
skipped_systems = []
# iterate over lines
for line in lines:
# skip emtpy lines
if line == '':
# autoincrement counter
lines_faulty_counter += 1
# call logger
warning_logger(str(request_user), ' SYSTEM_CREATOR_ROW_EMPTY')
continue
# check line for length of string
if len(line) > 50:
# autoincrement counter
lines_faulty_counter += 1
# call logger
warning_logger(str(request_user), ' SYSTEM_CREATOR_LONG_STRING')
continue
# check for existence of system
system = System.objects.filter(system_name=line)
""" already existing system """
# in case of existing system
if system.count() > 0:
# autoincrement counter
systems_skipped_counter += 1
# add system name to list of skipped systems
skipped_systems.append(line)
# call logger
error_logger(
str(request_user),
' SYSTEM_CREATOR_SYSTEM_EXISTS ' + 'system_name:' + line)
# leave this loop because system with this systemname already exists
continue
""" new system """
# create form with request data
form = SystemCreatorForm(request_post)
# create system
if form.is_valid():
""" object creation """
# don't save form yet
system = form.save(commit=False)
# set system_name
system.system_name = line
# set auto values
system.system_created_by_user_id = request_user
system.system_modified_by_user_id = request_user
system.system_modify_time = timezone.now()
# save object
system.save()
# save manytomany
form.save_m2m()
""" object counter / log """
# autoincrement counter
systems_created_counter += 1
# call logger
system.logger(str(request_user), ' SYSTEM_CREATOR_EXECUTED')
""" call final messages """
# finish message
message_user(request_user, 'System creator finished', constants.SUCCESS)
# number messages
if systems_created_counter > 0:
if systems_created_counter == 1:
message_user(request_user,
str(systems_created_counter) + ' system was created.',
constants.SUCCESS)
else:
message_user(
request_user,
str(systems_created_counter) + ' systems were created.',
constants.SUCCESS)
if systems_skipped_counter > 0:
if systems_skipped_counter == 1:
message_user(
request_user,
str(systems_skipped_counter) + ' system was skipped. ' +
str(skipped_systems), constants.ERROR)
else:
message_user(
request_user,
str(systems_skipped_counter) + ' systems were skipped. ' +
str(skipped_systems), constants.ERROR)
if lines_faulty_counter > 0:
if lines_faulty_counter == 1:
message_user(
request_user,
str(lines_faulty_counter) + ' line out of ' +
str(number_of_lines) +
' lines was faulty (see log file for details).',
constants.WARNING)
else:
message_user(
request_user,
str(lines_faulty_counter) + ' lines out of ' +
str(number_of_lines) +
' lines were faulty (see log file for details).',
constants.WARNING)
# call logger
info_logger(
str(request_user),
' SYSTEM_CREATOR_STATUS ' + 'created:' + str(systems_created_counter) +
'|' + 'skipped:' + str(systems_skipped_counter) + '|' +
'faulty_lines:' + str(lines_faulty_counter))
# call logger
debug_logger(str(request_user), ' SYSTEM_CREATOR_END')
def system_handler(request=None, uploadfile=False):
# get config model
model = SystemImporterFileCsvConfigModel.objects.get(
system_importer_file_csv_config_name='SystemImporterFileCsvConfig')
""" set username for logger and object """
# if function was called from 'system_instant' and 'system_upload'
if request:
# get user for object
csv_import_user = request.user
# get user for logger
logger_username = str(request.user)
# if function was called from 'system_cron'
else:
# get user for object
csv_import_user = model.csv_import_username
# get user for logger
logger_username = model.csv_import_username.username
""" start system importer """
# get starttime
starttime = timezone.now().strftime('%Y-%m-%d %H:%M:%S')
# call logger
debug_logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_START')
# create lock tags
create_lock_tags(model)
""" file handling """
# file was uploaded via form (called via 'system_upload')
if uploadfile:
# text object can not be passed as argument from 'system_upload'
systemcsv = TextIOWrapper(request.FILES['systemcsv'].file,
encoding=request.encoding)
# file was fetched from file system (called via 'system_instant' or 'system_cron')
else:
# build csv file path
csv_import_file = model.csv_import_path + '/' + model.csv_import_filename
# open file
systemcsv = open(csv_import_file, 'r')
# get field delimiter from config
if model.csv_field_delimiter == 'field_comma':
delimiter = ','
elif model.csv_field_delimiter == 'field_semicolon':
delimiter = ';'
# get text quotechar from config
if model.csv_text_quote == 'text_double_quotation_marks':
quotechar = '"'
elif model.csv_text_quote == 'text_single_quotation_marks':
quotechar = "'"
# read rows out of csv
rows = csv.reader(systemcsv, delimiter=delimiter, quotechar=quotechar)
""" check file """
# if function was called from 'system_instant' and 'system_upload'
if request:
# check file for csv respectively some kind of text file
file_check = check_content_file_type(rows, logger_username, request)
# if function was called from 'system_cron'
else:
# check file for csv respectively some kind of text file
file_check = check_content_file_type(rows, logger_username)
# leave system_importer_file_csv if file check throws errors
if not file_check:
# close file
systemcsv.close()
# return to calling function 'csv.system_cron' or 'csv.system_instant' or 'csv.system_upload'
return
# jump to begin of file again after iterating in file check
systemcsv.seek(0)
""" prepare loop """
# set row_counter (needed for logger)
row_counter = 1
# set systems_created_counter (needed for logger)
systems_created_counter = 0
# set systems_updated_counter (needed for logger)
systems_updated_counter = 0
# set systems_multiple_counter (needed for logger)
systems_multiple_counter = 0
# create empty list (needed for logger)
systems_multiple_list = []
# set systems_skipped_counter (needed for logger)
systems_skipped_counter = 0
""" start loop """
# iterate over rows
for row in rows:
""" skip headline if necessary """
# check for first row and headline condition
if row_counter == 1 and model.csv_headline:
# autoincrement row counter
row_counter += 1
# leave loop for headline row
continue
""" filter for systems """
# if function was called from 'system_instant' and 'system_upload'
if request:
# check system_name for valid value
stop_system_importer_file_csv = check_system_name(
model, row, row_counter, request)
# if function was called from 'system_cron'
else:
# check system_name for valid value
stop_system_importer_file_csv = check_system_name(
model, row, row_counter)
# leave loop if system_name caused errors
if stop_system_importer_file_csv:
# autoincrement counter
systems_skipped_counter += 1
# autoincrement row counter
row_counter += 1
# leave loop
continue
# get system name (for domain name comparison)
system_name = row[model.csv_column_system - 1]
# TODO: [logic] add option which attributes are used for filtering?
# TODO: [logic] (like domain, dnsname, company)
# TODO: [logic] e.g. 'csv_identification_dnsname'
# get all systems
systemquery = System.objects.filter(system_name=system_name, )
""" check how many systems were returned """
# if there is only one system -> modify system
if len(systemquery) == 1:
# skip if system already exists (depending on csv_skip_existing_system)
if model.csv_skip_existing_system:
# autoincrement counter
systems_skipped_counter += 1
# autoincrement row counter
row_counter += 1
# leave loop
continue
# TODO: [logic] add option which attributes are used for filtering?
# TODO: [logic] (like domain, dnsname, company)
# TODO: [logic] e.g. 'csv_identification_dnsname'
# get existing system object
system = System.objects.get(system_name=system_name, )
# change mandatory meta attributes
system.system_modify_time = timezone.now()
system.system_modified_by_user_id = csv_import_user
# set value for already existing system (modify system)
system_created = False
# if function was called from 'system_instant' and 'system_upload'
if request:
# add foreign key relationships to system
system = add_fk_attributes(system, system_created, model, row,
row_counter, request)
# if function was called from 'system_cron'
else:
# add foreign key relationships to system
system = add_fk_attributes(system, system_created, model, row,
row_counter)
# save object
system.save()
# if function was called from 'system_instant' and 'system_upload'
if request:
# add many2many relationships to system
system = add_many2many_attributes(system, system_created,
model, row, row_counter,
request)
# if function was called from 'system_cron'
else:
# add many2many relationships to system
system = add_many2many_attributes(system, system_created,
model, row, row_counter)
# autoincrement systems_updated_counter
systems_updated_counter += 1
# call logger
system.logger(logger_username,
' SYSTEM_IMPORTER_FILE_CSV_SYSTEM_MODIFIED')
# if there is more than one system
elif len(systemquery) > 1:
# add system name to list
systems_multiple_list.append(system_name)
# autoincrement systems_multiple_counter
systems_multiple_counter += 1
# call logger
warning_logger(
logger_username,
f' SYSTEM_IMPORTER_FILE_CSV_MULTIPLE_SYSTEMS System:{system_name}'
)
# if there is no system -> create system
else:
# create new system object
system = System()
# add system_name from csv
system.system_name = system_name
# add mandatory meta attributes
system.system_modify_time = timezone.now()
system.system_created_by_user_id = csv_import_user
system.system_modified_by_user_id = csv_import_user
# set value for new system (create system)
system_created = True
# if function was called from 'system_instant' and 'system_upload'
if request:
# add foreign key relationships to system
system = add_fk_attributes(system, system_created, model, row,
row_counter, request)
# if function was called from 'system_cron'
else:
# add foreign key relationships to system
system = add_fk_attributes(system, system_created, model, row,
row_counter)
# save object
system.save()
# if function was called from 'system_instant' and 'system_upload'
if request:
# add many2many relationships to system
system = add_many2many_attributes(system, system_created,
model, row, row_counter,
request)
# if function was called from 'system_cron'
else:
# add many2many relationships to system
system = add_many2many_attributes(system, system_created,
model, row, row_counter)
# autoincrement systems_created_counter
systems_created_counter += 1
# call logger
system.logger(logger_username,
' SYSTEM_IMPORTER_FILE_CSV_SYSTEM_CREATED')
# autoincrement row counter
row_counter += 1
# close file
systemcsv.close()
""" finish system importer """
# get endtime
endtime = timezone.now().strftime('%Y-%m-%d %H:%M:%S')
# if function was called from 'system_instant' and 'system_upload'
if request:
# call final messages
final_messages(
systems_created_counter,
systems_updated_counter,
systems_skipped_counter,
systems_multiple_counter,
systems_multiple_list,
request,
)
# if function was called from 'system_cron'
else:
# call final messages
final_messages_cron(
systems_created_counter,
systems_updated_counter,
systems_skipped_counter,
systems_multiple_counter,
systems_multiple_list,
starttime,
endtime,
)
# call logger
info_logger(
logger_username, f' SYSTEM_IMPORTER_FILE_CSV_STATUS'
f' created:{systems_created_counter}'
f'|updated:{systems_updated_counter}'
f'|skipped:{systems_skipped_counter}'
f'|multiple:{systems_multiple_counter}')
# call logger
debug_logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_END')
# return to calling function 'csv.system_cron' or 'csv.system_instant' or 'csv.system_upload'
return
def reportitem(request):
""" this function checks for every system the existence of a markdown file with information about system the and imports the content of this file as reportitem for the corresponding system """
# call logger
debug_logger(str(request.user), " REPORTITEM_IMPORTER_FILE_FILESYSTEM_BEGAN")
# check config before continuing
stop_reportitem_importer_file_filesystem = check_config(request)
# leave reportitem_importer_file_filesystem if variables caused errors
if stop_reportitem_importer_file_filesystem:
return redirect(reverse('system_list'))
# get all system objects
systems = System.objects.all()
# create headline if it does not exist
headline, created = Headline.objects.get_or_create(headline_name=dfirtrack_config.REPORTITEM_HEADLINE)
if created == True:
# call logger
headline.logger(str(request.user), " REPORTITEM_IMPORTER_FILE_FILESYSTEM_HEADLINE_CREATED")
# set counter for non-existing files (needed for messages)
nofile_found_counter = 0
# set counter for created reportitems (needed for messages)
reportitems_created_counter = 0
# set counter for modified reportitems (needed for messages)
reportitems_modified_counter = 0
# set counter for deleted reportitems (needed for messages)
reportitems_deleted_counter = 0
# iterate over systems
for system in systems:
# create path for reportfile
reportpath = dfirtrack_config.REPORTITEM_FILESYSTEMPATH + "/" + system.system_name + ".md"
# check whether a file is existing for this system
if not os.path.isfile(reportpath):
# call logger
warning_logger(str(request.user), " REPORTITEM_IMPORTER_FILE_FILESYSTEM_NO_FILE system_name:" + system.system_name)
# autoincrement counter
nofile_found_counter += 1
# check whether already existing reportitem for this system should be deleted if no file was provided
if dfirtrack_config.REPORTITEM_DELETE:
# delete already existing reportitem for this system if no file was provided
try:
reportitem = Reportitem.objects.get(system = system, headline = headline, reportitem_subheadline = dfirtrack_config.REPORTITEM_SUBHEADLINE)
# call logger (before deleting instance)
reportitem.logger(str(request.user), " REPORTITEM_IMPORTER_FILE_FILESYSTEM_REPORTITEM_DELETED")
reportitem.delete()
# autoincrement counter
reportitems_deleted_counter += 1
except Reportitem.DoesNotExist:
pass
# continue with next system
continue
# create reportitem if it does not exist (get_or_create won't work in this context because of needed user objects for saving)
try:
reportitem = Reportitem.objects.get(system = system, headline = headline, reportitem_subheadline = dfirtrack_config.REPORTITEM_SUBHEADLINE)
reportitems_modified_counter += 1
except Reportitem.DoesNotExist:
reportitem = Reportitem()
reportitems_created_counter += 1
reportitem.system = system
reportitem.headline = headline
reportitem.reportitem_subheadline = dfirtrack_config.REPORTITEM_SUBHEADLINE
reportitem.reportitem_created_by_user_id = request.user
# open file
reportfile = open(reportpath, "r")
# add changing values (existing reportitem_note will be overwritten)
reportitem.reportitem_note = reportfile.read()
# close file
reportfile.close()
reportitem.reportitem_modified_by_user_id = request.user
reportitem.save()
# call logger
reportitem.logger(str(request.user), " REPORTITEM_IMPORTER_FILE_FILESYSTEM_REPORTITEM_CREATED_OR_MODIFIED")
# call final messages
if nofile_found_counter > 0:
if nofile_found_counter == 1:
messages.warning(request, "No file was found for " + str(nofile_found_counter) + " system.")
else:
messages.warning(request, "No files were found for " + str(nofile_found_counter) + " systems.")
if reportitems_created_counter > 0:
if reportitems_created_counter == 1:
messages.success(request, str(reportitems_created_counter) + ' reportitem was created.')
else:
messages.success(request, str(reportitems_created_counter) + ' reportitems were created.')
if reportitems_modified_counter > 0:
if reportitems_modified_counter == 1:
messages.success(request, str(reportitems_modified_counter) + ' reportitem was modified.')
else:
messages.success(request, str(reportitems_modified_counter) + ' reportitems were modified.')
if reportitems_deleted_counter > 0:
if reportitems_deleted_counter == 1:
messages.success(request, str(reportitems_deleted_counter) + ' reportitem was deleted.')
else:
messages.success(request, str(reportitems_deleted_counter) + ' reportitems were deleted.')
return redirect(reverse('system_list'))
def check_config(request): # coverage: ignore branch
""" check variables in `dfirtrack.config` """
# reset stop condition
stop_importer_api_giraf = False
# check GIRAF_URL for string
if not isinstance(dfirtrack_config.GIRAF_URL, str):
messages.error(request, "`GIRAF_URL` is not a string. Check `dfirtrack.config`!")
# call logger
warning_logger(str(request.user), " IMPORTER_API_GIRAF variable GIRAF_URL no string")
stop_importer_api_giraf = True
# check GIRAF_URL for empty string (check only if it actually is a string)
if isinstance(dfirtrack_config.GIRAF_URL, str) and not dfirtrack_config.GIRAF_URL:
messages.error(request, "`GIRAF_URL` contains an empty string. Check `dfirtrack.config`!")
# call logger
warning_logger(str(request.user), " IMPORTER_API_GIRAF variable GIRAF_URL empty string")
stop_importer_api_giraf = True
# check GIRAF_USER for string
if not isinstance(dfirtrack_config.GIRAF_USER, str):
messages.error(request, "`GIRAF_USER` is not a string. Check `dfirtrack.config`!")
# call logger
warning_logger(str(request.user), " IMPORTER_API_GIRAF variable GIRAF_USER no string")
stop_importer_api_giraf = True
# check GIRAF_USER for empty string (check only if it actually is a string)
if isinstance(dfirtrack_config.GIRAF_USER, str) and not dfirtrack_config.GIRAF_USER:
messages.error(request, "`GIRAF_USER` contains an empty string. Check `dfirtrack.config`!")
# call logger
warning_logger(str(request.user), " IMPORTER_API_GIRAF variable GIRAF_USER empty string")
stop_importer_api_giraf = True
# check GIRAF_PASS for string
if not isinstance(dfirtrack_config.GIRAF_PASS, str):
messages.error(request, "`GIRAF_PASS` is not a string. Check `dfirtrack.config`!")
# call logger
warning_logger(str(request.user), " IMPORTER_API_GIRAF variable GIRAF_PASS no string")
stop_importer_api_giraf = True
# check GIRAF_PASS for empty string (check only if it actually is a string)
if isinstance(dfirtrack_config.GIRAF_PASS, str) and not dfirtrack_config.GIRAF_PASS:
messages.error(request, "`GIRAF_PASS` contains an empty string. Check `dfirtrack.config`!")
# call logger
warning_logger(str(request.user), " IMPORTER_API_GIRAF variable GIRAF_PASS empty string")
stop_importer_api_giraf = True
return stop_importer_api_giraf
def system_modificator(request):
"""function to modify many systems at once (helper function to call the real function)"""
# form was valid to post
if request.method == "POST":
try:
request.POST['systemlist']
except:
# call logger
warning_logger(str(request.user),
' SYSTEM_MODIFICATOR_NO_SYSTEMS_PROVIDED')
# show immediate message for user
messages.warning(request, 'No systems were provided.')
# return directly to system list
return redirect(reverse('system_list'))
# get objects from request object
request_post = request.POST
request_user = request.user
# show immediate message for user
messages.success(request, 'System modificator started')
# call async function
async_task(
"dfirtrack_main.modificator.system_modificator.system_modificator_async",
request_post,
request_user,
)
# return directly to system list
return redirect(reverse('system_list'))
# show empty form
else:
show_systemlist = bool(int(request.GET.get('systemlist', 0)))
# workflows
workflows = Workflow.objects.all()
# show empty form with default values for convenience and speed reasons
form = SystemModificatorForm(
initial={
'systemstatus_choice': 'keep_status',
'analysisstatus_choice': 'keep_status',
'company_delete': 'keep_not_add',
'tag_delete': 'keep_not_add',
'contact_delete': 'keep_existing',
'location_delete': 'keep_existing',
'serviceprovider_delete': 'keep_existing',
},
use_system_charfield=show_systemlist,
)
# call logger
debug_logger(str(request.user), ' SYSTEM_MODIFICATOR_ENTERED')
return render(
request,
'dfirtrack_main/system/system_modificator.html',
{
'form': form,
'workflows': workflows
},
)
def system_modificator_async(request_post, request_user):
""" function to modify many systems at once """
# call logger
debug_logger(str(request_user), " SYSTEM_MODIFICATOR_BEGIN")
# exctract lines from systemlist (list results from request object via large text area)
lines = request_post.get('systemlist').splitlines()
# iterate over lines
for line in lines:
# skip emtpy lines
if line == '':
warning_logger(str(request_user), " SYSTEM_MODIFICATOR_ROW_EMPTY")
continue
# check line for string
if not isinstance(line, str):
warning_logger(str(request_user), " SYSTEM_MODIFICATOR_NO_STRING")
continue
# check line for length of string
if len(line) > 50:
warning_logger(str(request_user), " SYSTEM_MODIFICATOR_LONG_STRING")
continue
# check for existence of system
system = System.objects.filter(system_name = line)
if system.count() == 0:
# call logger
error_logger(str(request_user), " SYSTEM_MODIFICATOR_SYSTEM_DOES_NOT_EXISTS " + "system_name:" + line)
# leave this loop because system with this systemname does not exist
continue
elif system.count() > 1:
# call logger
error_logger(str(request_user), " SYSTEM_MODIFICATOR_SYSTEM_NOT_DISTINCT " + "system_name:" + line)
# leave this loop because system with this systemname is not distinct
continue
# get existing system
system = System.objects.get(system_name = line)
# create form with request data
form = SystemModificatorForm(request_post, instance = system)
# extract tags (list results from request object via multiple choice field)
tags = request_post.getlist('tag')
# modify system
if form.is_valid():
# don't save form yet
system = form.save(commit=False)
# set system_name
system.system_name = line
# set auto values
system.system_modified_by_user_id = request_user
system.system_modify_time = timezone.now()
# save object
system.save()
# call logger
system.logger(str(request_user), ' SYSTEM_MODIFICATOR_EXECUTED')
# TODO: add check for empty list
# add tags (using save_m2m would replace existing tags)
for tag_id in tags:
# get object
tag = Tag.objects.get(tag_id=tag_id)
# add tag to system
system.tag.add(tag)
# call logger
debug_logger(str(request_user), " SYSTEM_MODIFICATOR_END")
def entry(request):
""" this form parses a file and tries to get entries for a single system """
# form was valid to post
if request.method == "POST":
# call logger
debug_logger(str(request.user), " ENTRY_TXT_IMPORTER_BEGIN")
# get text out of file (variable results from request object via file upload field)
entryfile = TextIOWrapper(request.FILES['entryfile'].file,
encoding=request.encoding)
# set row counter (needed for logger)
i = 0
# iterate over rows in file
for row in entryfile:
# autoincrement row counter
i += 1
# skip first two lines # TODO: remove first two lines from parsing script
if i == 1 or i == 2:
continue
# split line from markdown table format to single values
column = row.split("|")
# check row for empty value
if len(column) < 6:
warning_logger(
str(request.user), " ENTRY_TXT_IMPORTER_SYSTEM_COLUMN " +
"row_" + str(i) + ":empty_row")
continue
# get values of former markdown tables
entry_date = column[1]
entry_utc = column[2]
entry_system = column[3]
entry_type = column[4]
entry_content = column[5]
# remove trailing and leading whitespaces
entry_date = entry_date.strip()
entry_utc = entry_utc.strip()
entry_system = entry_system.strip()
entry_type = entry_type.strip()
entry_content = entry_content.strip()
# concatenate all relevant entry values to a string for hashing
entry_string = entry_date + entry_utc + entry_system + entry_type + entry_content
# calculate hash from entry values
entry_hash = hashlib.sha1(entry_string.encode('utf8'))
entry_sha1 = entry_hash.hexdigest()
# get system_id as string from POST object
system = request.POST['system']
# check for existing entry_sha1 for this system and skip if it already exist
try:
check_entry = Entry.objects.get(system=system,
entry_sha1=entry_sha1)
warning_logger(
str(request.user), " ENTRY_TXT_IMPORTER_ENTRY_EXISTS " +
"row_" + str(i) + ":entry_exists")
continue
except:
pass
# convert timing information to datetime object
entry_time = parse(entry_date + " " + entry_utc + "+00:00")
# create form with request data
form = EntryFileImport(request.POST, request.FILES)
# create entry
if form.is_valid():
pass
# don't save form yet
entry = form.save(commit=False)
# set values from file (row / column)
entry.entry_time = entry_time
entry.entry_sha1 = entry_sha1
entry.entry_date = entry_date
entry.entry_utc = entry_utc
entry.entry_system = entry_system
entry.entry_type = entry_type
entry.entry_content = entry_content
# set auto values
entry.entry_created_by_user_id = request.user
entry.entry_modified_by_user_id = request.user
# save object
entry.save()
# call logger
entry.logger(str(request.user), ' ENTRY_TXT_IMPORTER_EXECUTED')
# call logger
debug_logger(str(request.user), " ENTRY_TXT_IMPORTER_END")
return redirect(reverse('system_detail', args=(system, )))
else:
# show empty form with preselected system
if request.method == 'GET' and 'system' in request.GET:
system = request.GET['system']
form = EntryFileImport(initial={
'system': system,
})
else:
# show empty form
form = EntryFileImport()
# call logger
debug_logger(str(request.user), " ENTRY_TXT_IMPORTER_ENTERED")
return render(request, 'dfirtrack_main/entry/entry_file_importer.html',
{'form': form})
def check_config(request):
""" check variables of dfirtrack.config """
# reset stop condition
stop_reportitem_importer_file_filesystem = False
# check REPORTITEM_FILESYSTEMPATH for string
if not isinstance(dfirtrack_config.REPORTITEM_FILESYSTEMPATH, str):
messages.error(
request,
"`REPORTITEM_FILESYSTEMPATH` is not a string. Check `dfirtrack.config`!"
)
# call logger
warning_logger(
str(request.user),
" REPORTITEM_IMPORTER_FILE_FILESYSTEM variable REPORTITEM_FILESYSTEMPATH no string"
)
stop_reportitem_importer_file_filesystem = True
# check REPORTITEM_FILESYSTEMPATH for empty string (check only if it is a string)
if isinstance(dfirtrack_config.REPORTITEM_FILESYSTEMPATH,
str) and not dfirtrack_config.REPORTITEM_FILESYSTEMPATH:
messages.error(
request,
"`REPORTITEM_FILESYSTEMPATH` contains an empty string. Check `dfirtrack.config`!"
)
# call logger
warning_logger(
str(request.user),
" REPORTITEM_IMPORTER_FILE_FILESYSTEM variable REPORTITEM_FILESYSTEMPATH empty string"
)
stop_reportitem_importer_file_filesystem = True
# check REPORTITEM_FILESYSTEMPATH existing directory (check only if REPORTITEM_FILESYSTEMPATH is a string and not an empty string)
if isinstance(dfirtrack_config.REPORTITEM_FILESYSTEMPATH,
str) and dfirtrack_config.REPORTITEM_FILESYSTEMPATH:
if not os.path.isdir(dfirtrack_config.REPORTITEM_FILESYSTEMPATH):
messages.error(
request,
"`REPORTITEM_FILESYSTEMPATH` points to a non-existing directory. Check `dfirtrack.config`!"
)
# call logger
warning_logger(
str(request.user),
" REPORTITEM_IMPORTER_FILE_FILESYSTEM variable REPORTITEM_FILESYSTEMPATH path does not exist"
)
stop_reportitem_importer_file_filesystem = True
# check REPORTITEM_HEADLINE for string
if not isinstance(dfirtrack_config.REPORTITEM_HEADLINE, str):
messages.error(
request,
"`REPORTITEM_HEADLINE` is not a string. Check `dfirtrack.config`!")
# call logger
warning_logger(
str(request.user),
" REPORTITEM_IMPORTER_FILE_FILESYSTEM variable REPORTITEM_HEADLINE no string"
)
stop_reportitem_importer_file_filesystem = True
# check REPORTITEM_HEADLINE for empty string (check only if it is a string)
if isinstance(dfirtrack_config.REPORTITEM_HEADLINE,
str) and not dfirtrack_config.REPORTITEM_HEADLINE:
messages.error(
request,
"`REPORTITEM_HEADLINE` contains an empty string. Check `dfirtrack.config`!"
)
# call logger
warning_logger(
str(request.user),
" REPORTITEM_IMPORTER_FILE_FILESYSTEM variable REPORTITEM_HEADLINE empty string"
)
stop_reportitem_importer_file_filesystem = True
# check REPORTITEM_SUBHEADLINE for string
if not isinstance(dfirtrack_config.REPORTITEM_SUBHEADLINE, str):
messages.error(
request,
"`REPORTITEM_SUBHEADLINE` is not a string. Check `dfirtrack.config`!"
)
# call logger
warning_logger(
str(request.user),
" REPORTITEM_IMPORTER_FILE_FILESYSTEM variable REPORTITEM_SUBHEADLINE no string"
)
stop_reportitem_importer_file_filesystem = True
# check REPORTITEM_SUBHEADLINE for empty string (check only if it is a string)
if isinstance(dfirtrack_config.REPORTITEM_SUBHEADLINE,
str) and not dfirtrack_config.REPORTITEM_SUBHEADLINE:
messages.error(
request,
"`REPORTITEM_SUBHEADLINE` contains an empty string. Check `dfirtrack.config`!"
)
# call logger
warning_logger(
str(request.user),
" REPORTITEM_IMPORTER_FILE_FILESYSTEM variable REPORTITEM_SUBHEADLINE empty string"
)
stop_reportitem_importer_file_filesystem = True
# check REPORTITEM_DELETE for bool
if not isinstance(dfirtrack_config.REPORTITEM_DELETE, bool):
messages.error(
request,
"`REPORTITEM_DELETE` is not boolean. Check `dfirtrack.config`!")
# call logger
warning_logger(
str(request.user),
" REPORTITEM_IMPORTER_FILE_FILESYSTEM variable REPORTITEM_DELETE not boolean"
)
stop_reportitem_importer_file_filesystem = True
# create final message and log
if stop_reportitem_importer_file_filesystem:
messages.warning(request, "Nothing was changed.")
# call logger
warning_logger(
str(request.user),
" REPORTITEM_IMPORTER_FILE_FILESYSTEM_ENDED_WITH_ERRORS")
return stop_reportitem_importer_file_filesystem
def system_modificator_async(request_post, request_user):
"""function to modify many systems at once"""
# call logger
debug_logger(str(request_user), ' SYSTEM_MODIFICATOR_START')
# extract lines from systemlist (list results either from request object via multiline selector or via large text area)
lines = request_post.getlist('systemlist')
system_char_field_used = False
# if large text area was used, the list contains only one entry with (one or more) line breaks
if len(lines) == 1 and ("\r\n" in lines[0] or not lines[0].isdigit()):
system_char_field_used = True
lines = lines[0].splitlines()
""" prepare and start loop """
# count lines (needed for messages)
number_of_lines = len(lines)
# set systems_modified_counter (needed for messages)
systems_modified_counter = 0
# set systems_skipped_counter (needed for messages)
systems_skipped_counter = 0
# set lines_faulty_counter (needed for messages)
lines_faulty_counter = 0
# create empty list (needed for messages)
skipped_systems = []
# workflows to apply
workflows = request_post.getlist("workflow")
# set workflows_applied (needed for messages)
if workflows:
workflow_count = len(workflows)
else:
workflow_count = 0
workflows_applied = 0
# iterate over lines
for line in lines:
# skip empty lines
if line == '':
# autoincrement counter
lines_faulty_counter += 1
# call logger
warning_logger(str(request_user), ' SYSTEM_MODIFICATOR_ROW_EMPTY')
continue
# check line for string
if not isinstance(line, str): # coverage: ignore branch
# autoincrement counter
lines_faulty_counter += 1
# call logger
warning_logger(str(request_user), ' SYSTEM_MODIFICATOR_NO_STRING')
continue
# check line for length of string
if len(line) > 50:
# autoincrement counter
lines_faulty_counter += 1
# call logger
warning_logger(str(request_user),
' SYSTEM_MODIFICATOR_LONG_STRING')
continue
# check for existence of system
if system_char_field_used:
system = System.objects.filter(system_name=line)
else:
system = System.objects.filter(system_id=line)
""" handling non-existing (== 0) or non-unique (> 1) systems """
# system does not exist
if system.count() == 0:
# autoincrement counter
systems_skipped_counter += 1
# add system name to list of skipped systems
skipped_systems.append(line)
# call logger
warning_logger(
str(request_user),
f' SYSTEM_MODIFICATOR_SYSTEM_DOES_NOT_EXISTS system_id/system_name:{line}',
)
# leave this loop because system with this systemname does not exist
continue
# more than one system exists
elif system.count() > 1:
# autoincrement counter
systems_skipped_counter += 1
# add system name to list of skipped systems
skipped_systems.append(line)
# call logger
warning_logger(
str(request_user),
f' SYSTEM_MODIFICATOR_SYSTEM_NOT_DISTINCT system_id/system_name:{line}',
)
# leave this loop because system with this systemname is not distinct
continue
""" unique system (== 1) """
# get existing system
if system_char_field_used:
system = System.objects.get(system_name=line)
else:
system = System.objects.get(system_id=line)
# create form with request data
form = SystemModificatorForm(
request_post,
instance=system,
use_system_charfield=system_char_field_used)
# extract tags (list results from request object via multiple choice field)
tags = request_post.getlist('tag')
# extract companies (list results from request object via multiple choice field)
companies = request_post.getlist('company')
# TODO: [code] add condition for invalid form, not implemented yet because of multiple constraints
# modify system
if form.is_valid():
"""object modification"""
# don't save form yet
system = form.save(commit=False)
# set auto values
system.system_modified_by_user_id = request_user
""" status fields """
# replace, if 'change status' was selected (checking for status value independent of form field validation)
if (form['analysisstatus_choice'].value() == 'change_status'
and form['analysisstatus'].value()):
# replace status
analysisstatus_id = form['analysisstatus'].value()
analysisstatus = Analysisstatus.objects.get(
analysisstatus_id=analysisstatus_id)
system.analysisstatus = analysisstatus
# replace, if 'change status' was selected (checking for status value independent of form field validation)
if (form['systemstatus_choice'].value() == 'change_status'
and form['systemstatus'].value()):
# replace status
systemstatus_id = form['systemstatus'].value()
systemstatus = Systemstatus.objects.get(
systemstatus_id=systemstatus_id)
system.systemstatus = systemstatus
""" fk non-model fields """
# replace / delete, if 'switch_new / Switch to selected item or none' was selected
if form['contact_delete'].value() == 'switch_new':
# replace, if value was submitted via form
if form['contact'].value():
contact_id = form['contact'].value()
contact = Contact.objects.get(contact_id=contact_id)
system.contact = contact
# delete, if form field was empty
else:
system.contact = None
# replace / delete, if 'switch_new / Switch to selected item or none' was selected
if form['location_delete'].value() == 'switch_new':
# replace, if value was submitted via form
if form['location'].value():
location_id = form['location'].value()
location = Location.objects.get(location_id=location_id)
system.location = location
# delete, if form field was empty
else:
system.location = None
# replace / delete, if 'switch_new / Switch to selected item or none' was selected
if form['serviceprovider_delete'].value() == 'switch_new':
# replace, if value was submitted via form
if form['serviceprovider'].value():
serviceprovider_id = form['serviceprovider'].value()
serviceprovider = Serviceprovider.objects.get(
serviceprovider_id=serviceprovider_id)
system.serviceprovider = serviceprovider
# delete, if form field was empty
else:
system.serviceprovider = None
# save object
system.save()
""" object counter / log """
# autoincrement counter
systems_modified_counter += 1
# call logger
system.logger(str(request_user), ' SYSTEM_MODIFICATOR_EXECUTED')
""" many 2 many """
# remove existing relations if 'remove_and_add / Delete existing and add new items' was selected
if form['tag_delete'].value() == 'remove_and_add':
# remove all m2m
system.tag.clear()
# add new relations if not 'keep_not_add / Do not change and keep existing' was selected
if form['tag_delete'].value() != 'keep_not_add':
# add new tags (using save_m2m would replace existing tags it there were any)
for tag_id in tags:
# get object
tag = Tag.objects.get(tag_id=tag_id)
# add tag to system
system.tag.add(tag)
# remove existing relations if 'remove_and_add / Delete existing and add new items' was selected
if form['company_delete'].value() == 'remove_and_add':
# remove all m2m
system.company.clear()
# add new relations if not 'keep_not_add / Do not change and keep existing' was selected
if form['company_delete'].value() != 'keep_not_add':
# add new companies (using save_m2m would replace existing companies it there were any)
for company_id in companies:
# get object
company = Company.objects.get(company_id=company_id)
# add company to system
system.company.add(company)
# apply workflows
if workflows:
error_code = Workflow.apply(workflows, system, request_user)
if error_code:
system.logger(str(request_user),
' COULD_NOT_APPLY_WORKFLOW')
else:
workflows_applied += workflow_count
""" finish system modificator """
# call final messages
final_messages(
systems_modified_counter,
systems_skipped_counter,
lines_faulty_counter,
skipped_systems,
number_of_lines,
request_user,
workflow_count,
workflows_applied,
)
# call logger
info_logger(
str(request_user),
f' SYSTEM_MODIFICATOR_STATUS'
f' modified:{systems_modified_counter}'
f'|skipped:{systems_skipped_counter}'
f'|faulty_lines:{lines_faulty_counter}',
)
# call logger
debug_logger(str(request_user), ' SYSTEM_MODIFICATOR_END')
def add_many2many_attributes(system, system_created, model, row, row_counter, request=None):
""" add many2many relationships to system """
""" set username for logger and object """
# if function was called from 'system_instant' and 'system_upload'
if request:
# get user for object
csv_import_user = request.user
# get user for logger
logger_username = str(request.user)
# if function was called from 'system_cron'
else:
# get user for object
csv_import_user = model.csv_import_username
# get user for logger
logger_username = model.csv_import_username.username
""" IP addresses """
# add IPs for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_ip):
# remove IPs if not new system
if not system_created:
# remove all IPs
system.ip.clear()
# get IPs from CSV
if model.csv_choice_ip:
# check for index error
try:
# get IP string
ip_string = row[model.csv_column_ip - 1]
# check for empty string
if ip_string:
# get IP delimiter from config
if model.csv_ip_delimiter == 'ip_comma':
ip_delimiter = ','
elif model.csv_ip_delimiter == 'ip_semicolon':
ip_delimiter = ';'
elif model.csv_ip_delimiter == 'ip_space':
ip_delimiter = ' '
# split IP string to list depending on delimiter
ip_list = ip_string.split(ip_delimiter)
# iterate over list elements
for ip_ip in ip_list:
# if function was called from 'system_instant' and 'system_upload'
if request:
# check, get or create IP
ip = check_and_create_ip(ip_ip, model, row_counter, request)
# if function was called from 'system_cron'
else:
# check, get or create IP
ip = check_and_create_ip(ip_ip, model, row_counter)
# IP was returned from 'check_and_create_ip'
if ip:
# add ip to system
system.ip.add(ip)
# index out of range
except IndexError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Index for IP in row {row_counter} was out of range.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_IP_COLUMN row_{row_counter}:out_of_range')
""" case """
# set case for new system or change if remove old is set
if system_created or (not system_created and model.csv_remove_case):
# remove cases if not new system
if not system_created:
# remove all cases
system.case.clear()
# get case from CSV
if model.csv_choice_case:
# check for index error
try:
# get case from CSV column
case_name = row[model.csv_column_case - 1]
# check for empty string
if case_name:
# get existing case
try:
case = Case.objects.get(
case_name = case_name,
)
# create new case
except Case.DoesNotExist:
# value is valid
try:
case, created = Case.objects.get_or_create(
case_name = case_name,
case_is_incident = False,
case_created_by_user_id = csv_import_user,
)
# call logger if created
if created:
case.logger(logger_username, ' SYSTEM_IMPORTER_FILE_CSV_CASE_CREATED')
# value is not valid
except DataError:
# if function was called from 'system_instant' and 'system_upload'
if request:
# call message
messages.warning(request, f'Value for case in row {row_counter} was not a valid value.')
# call logger
warning_logger(logger_username, f' SYSTEM_IMPORTER_FILE_CSV_CASE_COLUMN row_{row_counter}:invalid_case')
# set empty value
case = None
def system_creator_async(request_post, request_user):
""" function to create many systems at once """
# call logger
debug_logger(str(request_user), ' SYSTEM_CREATOR_START')
# exctract lines from systemlist (list results from request object via large text area)
lines = request_post.get('systemlist').splitlines()
# count lines (needed for messages)
number_of_lines = len(lines)
# set systems_created_counter (needed for messages)
systems_created_counter = 0
# set systems_skipped_counter (needed for messages)
systems_skipped_counter = 0
# set lines_faulty_counter (needed for messages)
lines_faulty_counter = 0
# create empty list (needed for messages)
skipped_systems = []
# iterate over lines
for line in lines:
# skip emtpy lines
if line == '':
# autoincrement counter
lines_faulty_counter += 1
# call logger
warning_logger(str(request_user), ' SYSTEM_CREATOR_ROW_EMPTY')
continue
# check line for length of string
if len(line) > 50:
# autoincrement counter
lines_faulty_counter += 1
# call logger
warning_logger(str(request_user), ' SYSTEM_CREATOR_LONG_STRING')
continue
# check for existence of system
system = System.objects.filter(system_name = line)
""" already existing system """
# in case of existing system
if system.count() > 0:
# autoincrement counter
systems_skipped_counter += 1
# add system name to list of skipped systems
skipped_systems.append(line)
# call logger
warning_logger(str(request_user), f' SYSTEM_CREATOR_SYSTEM_EXISTS system_name:{line}')
# leave this loop because system with this systemname already exists
continue
""" new system """
# create form with request data
form = SystemCreatorForm(request_post)
# create system
if form.is_valid():
""" object creation """
# don't save form yet
system = form.save(commit=False)
# set system_name
system.system_name = line
# set auto values
system.system_created_by_user_id = request_user
system.system_modified_by_user_id = request_user
system.system_modify_time = timezone.now()
# save object
system.save()
# save manytomany
form.save_m2m()
""" object counter / log """
# autoincrement counter
systems_created_counter += 1
# call logger
system.logger(str(request_user), ' SYSTEM_CREATOR_EXECUTED')
""" finish system importer """
# call final messages
final_messages(systems_created_counter, systems_skipped_counter, lines_faulty_counter, skipped_systems, number_of_lines, request_user)
# call logger
info_logger(
str(request_user),
f' SYSTEM_CREATOR_STATUS'
f' created:{systems_created_counter}'
f'|skipped:{systems_skipped_counter}'
f'|faulty_lines:{lines_faulty_counter}'
)
# call logger
debug_logger(str(request_user), ' SYSTEM_CREATOR_END')
def system_modificator_async(request_post, request_user):
""" function to modify many systems at once """
# call logger
debug_logger(str(request_user), ' SYSTEM_MODIFICATOR_BEGIN')
# exctract lines from systemlist (list results either from request object via multiline selector or via large text area)
lines = request_post.getlist('systemlist')
system_char_field_used = False
# if large text area was used, the list contains only one entry with (one or more) line breaks
if len(lines) == 1 and ("\r\n" in lines[0] or not lines[0].isdigit()):
system_char_field_used = True
lines = lines[0].splitlines()
# count lines (needed for messages)
number_of_lines = len(lines)
# set systems_modified_counter (needed for messages)
systems_modified_counter = 0
# set systems_skipped_counter (needed for messages)
systems_skipped_counter = 0
# set lines_faulty_counter (needed for messages)
lines_faulty_counter = 0
# create empty list (needed for messages)
skipped_systems = []
# iterate over lines
for line in lines:
# skip emtpy lines
if line == '':
# autoincrement counter
lines_faulty_counter += 1
# call logger
warning_logger(str(request_user), ' SYSTEM_MODIFICATOR_ROW_EMPTY')
continue
# check line for string
if not isinstance(line, str): # coverage: ignore branch
# autoincrement counter
lines_faulty_counter += 1
# call logger
warning_logger(str(request_user), ' SYSTEM_MODIFICATOR_NO_STRING')
continue
# check line for length of string
if len(line) > 50:
# autoincrement counter
lines_faulty_counter += 1
# call logger
warning_logger(str(request_user),
' SYSTEM_MODIFICATOR_LONG_STRING')
continue
# check for existence of system
if system_char_field_used:
system = System.objects.filter(system_name=line)
else:
system = System.objects.filter(system_id=line)
""" handling non-existing or non-unique systems 2 """
# system does not exist
if system.count() == 0:
# autoincrement counter
systems_skipped_counter += 1
# add system name to list of skipped systems
skipped_systems.append(line)
# call logger
error_logger(
str(request_user),
' SYSTEM_MODIFICATOR_SYSTEM_DOES_NOT_EXISTS ' +
'system_id/system_name:' + line)
# leave this loop because system with this systemname does not exist
continue
# more than one system exists
elif system.count() > 1:
# autoincrement counter
systems_skipped_counter += 1
# add system name to list of skipped systems
skipped_systems.append(line)
# call logger
error_logger(
str(request_user), ' SYSTEM_MODIFICATOR_SYSTEM_NOT_DISTINCT ' +
'system_id/system_name:' + line)
# leave this loop because system with this systemname is not distinct
continue
# get existing system
if system_char_field_used:
system = System.objects.get(system_name=line)
else:
system = System.objects.get(system_id=line)
""" new system """
# create form with request data
form = SystemModificatorForm(
request_post,
instance=system,
use_system_charfield=system_char_field_used)
# extract tags (list results from request object via multiple choice field)
tags = request_post.getlist('tag')
# extract companies (list results from request object via multiple choice field)
companies = request_post.getlist('company')
# modify system
if form.is_valid():
""" object modification """
# don't save form yet
system = form.save(commit=False)
# set auto values
system.system_modified_by_user_id = request_user
system.system_modify_time = timezone.now()
# save object
system.save()
""" object counter / log """
# autoincrement counter
systems_modified_counter += 1
# call logger
system.logger(str(request_user), ' SYSTEM_MODIFICATOR_EXECUTED')
""" many 2 many """
# TODO: add check for empty list
# add tags (using save_m2m would replace existing tags)
for tag_id in tags:
# get object
tag = Tag.objects.get(tag_id=tag_id)
# add tag to system
system.tag.add(tag)
for company_id in companies:
# get object
company = Company.objects.get(company_id=company_id)
# add company to system
system.company.add(company)
""" call final messages """
# finish message
message_user(request_user, 'System modificator finished',
constants.SUCCESS)
# number messages
if systems_modified_counter > 0:
if systems_modified_counter == 1:
message_user(
request_user,
str(systems_modified_counter) + ' system was modified.',
constants.SUCCESS)
else:
message_user(
request_user,
str(systems_modified_counter) + ' systems were modified.',
constants.SUCCESS)
if systems_skipped_counter > 0:
if systems_skipped_counter == 1:
message_user(
request_user,
str(systems_skipped_counter) + ' system was skipped. ' +
str(skipped_systems), constants.ERROR)
else:
message_user(
request_user,
str(systems_skipped_counter) + ' systems were skipped. ' +
str(skipped_systems), constants.ERROR)
if lines_faulty_counter > 0:
if lines_faulty_counter == 1:
message_user(
request_user,
str(lines_faulty_counter) + ' line out of ' +
str(number_of_lines) +
' lines was faulty (see log file for details).',
constants.WARNING)
else:
message_user(
request_user,
str(lines_faulty_counter) + ' lines out of ' +
str(number_of_lines) +
' lines were faulty (see log file for details).',
constants.WARNING)
# call logger
info_logger(
str(request_user), ' SYSTEM_MODIFICATOR_STATUS ' + 'modified:' +
str(systems_modified_counter) + '|' + 'skipped:' +
str(systems_skipped_counter) + '|' + 'faulty_lines:' +
str(lines_faulty_counter))
# call logger
debug_logger(str(request_user), " SYSTEM_MODIFICATOR_END")
def systems_ips(request):
""" this function parses a csv file and tries to import systems and corresponding ips """
# form was valid to post
if request.method == "POST":
# call logger
debug_logger(str(request.user), " SYSTEM_IP_IMPORTER_BEGIN")
# get text out of file (variable results from request object via file upload field)
systemipcsv = TextIOWrapper(request.FILES['systemipcsv'].file, encoding=request.encoding)
# read rows out of csv
rows = csv.reader(systemipcsv, quotechar="'")
# set row counter (needed for logger)
i = 0
# check for wrong file type
try:
# iterate over rows
for row in rows:
# autoincrement row counter
i += 1
# check for empty rows
try:
# check system column for empty value
if row[0] == '':
warning_logger(str(request.user), " SYSTEM_IP_IMPORTER_SYSTEM_COLUMN " + "row_" + str(i) + ":empty_column")
continue
except IndexError:
warning_logger(str(request.user), " SYSTEM_IP_IMPORTER_ROW row_" + str(i) + ":empty_row")
continue
# check system column for string
if not isinstance(row[0], str):
warning_logger(str(request.user), " SYSTEM_IP_IMPORTER_SYSTEM_COLUMN " + "row_" + str(i) + ":no_string")
continue
# check system column for length of string
if len(row[0]) > 50:
warning_logger(str(request.user), " SYSTEM_IP_IMPORTER_SYSTEM_COLUMN " + "row_" + str(i) + ":long_string")
continue
# check ip column for ip
try:
ipaddress.ip_address(row[1])
except ValueError:
warning_logger(str(request.user), " SYSTEM_IP_IMPORTER_IP_COLUMN " + "row_" + str(i) + ":invalid_ip")
continue
# create ip
ip, created = Ip.objects.get_or_create(ip_ip=row[1])
if created == True:
ip.logger(str(request.user), " SYSTEMS_IP_IMPORTER_IP_CREATED")
# check for existence of system
system = System.objects.filter(system_name = row[0], ip = ip)
if system.count() > 0:
error_logger(str(request.user), " SYSTEM_IP_IMPORTER_SYSTEM_EXISTS " + "row_" + str(i) + ":system_exists|system_name:" + row[0] + "|ip:" + str(row[1]))
continue
# create form with request data
form = SystemIpFileImport(request.POST, request.FILES)
# create system
if form.is_valid():
# don't save form yet
system = form.save(commit=False)
# set system_name
system.system_name = row[0]
# set auto values
system.system_created_by_user_id = request.user
system.system_modified_by_user_id = request.user
system.system_modify_time = timezone.now()
# save object
system.save()
# save manytomany
form.save_m2m()
# save ip for system
system.ip.add(ip)
# call logger
system.logger(str(request.user), ' SYSTEM_IP_IMPORTER_EXECUTED')
# wrong file type
except UnicodeDecodeError:
critical_logger(str(request.user), " SYSTEM_IP_IMPORTER_WRONG_FILE_TYPE")
# call logger
debug_logger(str(request.user), " SYSTEM_IP_IMPORTER_END")
return redirect('/systems')
else:
# show empty form
form = SystemIpFileImport(initial={
'systemstatus': 2,
'analysisstatus': 1,
})
# call logger
debug_logger(str(request.user), " SYSTEM_IP_IMPORTER_ENTERED")
return render(request, 'dfirtrack_main/system/systems_ip_importer.html', {'form': form})
