def extract_features(rows):
users_list = list()
users = len(rows)
rows_amount = 0
logline(
'There are', users, 'users and', len(rows),
'rows matching your filter type',
'no computer users or anonymous users'
if io.get('users_only') else 'no anonymous users')
rows_max = get_dict_inner_length(rows)
logline('Setting timer for', rows_max, 'rows')
timer = Timer(rows_max)
try:
for name, group in rows.items():
completed_result, group_len = strip_group_length(
gen_features_for_user((name, group)))
timer.add_to_current(group_len)
rows_amount += group_len
if completed_result is not None:
users_list.append(completed_result)
if rows_amount > next_report == 0 or REPORT_EVERY_USER:
next_report = next_report + REPORT_SIZE
logline('At row ',
str(rows_amount),
'/~',
str(row_amount),
' - ETA is: ' + timer.get_eta(),
spaces_between=False)
logline('At user ',
len(users_list),
'/~',
max_users,
spaces_between=False)
if len(users_list) >= max_users:
break
except KeyboardInterrupt:
logline('User cancelled execution, wrapping up')
debug('Cancelled early at', len(users_list), 'instead of', users)
debug('You skipped a total of', users - len(users_list), 'users, or',
100 - ((len(users_list) / users) * 100), '%')
except Exception:
error('An error occurred during execution', traceback.format_exc())
debug('Salvaging all remaining users')
finally:
debug('Runtime is', timer.report_total_time())
logline("Did a total of", len(users_list), "users")
logline('Done gathering data')
logline('Closing file...')
output_data(users_list)
def do_detection(
session: Session,
users_list: List[Dict[str, Union[str, Dict[str, List[List[float]]]]]]
) -> Dict[str, List[Dict[str, int]]]:
logline('Calculating total dataset size')
total_samples = 0
for user in users_list:
total_samples += len(user["datasets"]["training"])
timer = Timer(total_samples)
logline("Starting anomaly detection")
all_anomalies = dict()
tested_users = 0
for user in users_list:
logline("")
percentage = round((tested_users * 100) / len(users_list))
logline("Checking user ",
tested_users,
"/",
len(users_list),
" (",
percentage,
"%)",
spaces_between=False)
logline("ETA is " + timer.get_eta())
current_user = Dataset(user)
try:
anomalies = find_anomalies(session, current_user)
if len(anomalies) > 0:
all_anomalies[current_user.user_name] = anomalies
tested_users += 1
timer.add_to_current(len(current_user.datasets.training))
except KeyboardInterrupt:
# Skip rest of users, report early
logline("\n\nSkipping rest of the users")
break
debug('Runtime is', timer.report_total_time())
return all_anomalies
def gen_features(f: pd.DataFrame, row_amount: int):
users_list = list()
logline('Calculating amount of groups...')
users = len(f)
logline(
'There are', users, 'users and', row_amount,
'rows matching your filter type',
'no computer users or anonymous users'
if io.get('users_only') else 'no anonymous users')
rows = 0
max_users = users
if not DO_ROWS_PERCENTAGE:
max_users = int(math.ceil(users * 0.01 * io.get('dataset_percentage')))
logline('Max amount of users is', max_users)
logline('Setting timer for',
int(math.ceil(row_amount * 0.01 * io.get('dataset_percentage'))),
'rows')
timer = Timer(
int(math.ceil(row_amount * 0.01 * io.get('dataset_percentage'))))
logline('Creating iterator')
dataset_iterator = DFIterator(f)
next_report = REPORT_SIZE
if not SKIP_MAIN:
try:
# Create groups of approx 1000 users big
if io.get('cpus') == 1:
logline('Only using a single CPU')
logline('Starting feature generation')
for name, group in f:
completed_result, group_len = strip_group_length(
gen_features_for_user((name, group)))
timer.add_to_current(group_len)
rows += group_len
if completed_result is not None:
users_list.append(completed_result)
if rows > next_report == 0 or REPORT_EVERY_USER:
next_report = next_report + REPORT_SIZE
logline('At row ',
str(rows),
'/~',
str(row_amount),
' - ETA is: ' + timer.get_eta(),
spaces_between=False)
logline('At user ',
len(users_list),
'/~',
max_users,
spaces_between=False)
if len(users_list) >= max_users:
break
else:
logline('Using', io.get('cpus'), 'cpus')
for i in range(
round(math.ceil(max_users / PROCESSING_GROUP_SIZE))):
dataset_iterator.set_max((i + 1) * PROCESSING_GROUP_SIZE)
if i == 0:
logline('Starting feature generation')
with multiprocessing.Pool(io.get('cpus')) as p:
for completed_result in p.imap_unordered(
gen_features_for_user,
dataset_iterator,
chunksize=100):
completed_result, group_len = strip_group_length(
completed_result)
timer.add_to_current(group_len)
rows += group_len
if completed_result is not None:
users_list.append(completed_result)
if rows > next_report or REPORT_EVERY_USER:
next_report = next_report + REPORT_SIZE
logline('At row ',
str(rows),
'/~',
str(row_amount),
' - ETA is: ' + timer.get_eta(),
spaces_between=False)
logline('At user',
len(users_list),
'/~',
max_users,
spaces_between=False)
except KeyboardInterrupt:
logline('User cancelled execution, wrapping up')
debug('Cancelled early at', len(users_list), 'instead of', users)
debug('You skipped a total of', users - len(users_list),
'users, or', 100 - ((len(users_list) / users) * 100), '%')
except Exception:
error('An error occurred during execution', traceback.format_exc())
debug('Salvaging all remaining users')
finally:
debug('Runtime is', timer.report_total_time())
logline("Did a total of", len(users_list), "users")
logline('Done gathering data')
logline('Closing file...')
output_data(users_list)
else:
debug('SKIPPING MAIN, DO NOT ENABLE IN PRODUCTION')
logline('Closing file')
output_data([])
def main():
if not io.run:
return
state_file = io.get('state_file')
input_file = io.get('input_file')
output_file = io.get('output_file')
dataset_file = io.get('dataset_file')
logline('Loading dataset file...')
f = pd.read_hdf(dataset_file,
get_dataset_name(),
start=0,
stop=calc_rows_amount())
logline('Filtering users')
f = filter_users(f)
logline('Grouping users')
f = group_df(f)
if state_file is not None:
initial_state = get_state(state_file)
logline('Waiting for state to reach different value, currently at ' +
str(initial_state) + '...')
while get_state(state_file) == initial_state:
time.sleep(60)
logline('State file has switched to ' + str(get_state(state_file)) +
', continuing execution')
logline('Loading anomalies')
anomalies = read_anomalies(input_file)
anomaly_rows_list = dict()
users = len(f)
max_users = users
if DO_ROWS_PERCENTAGE:
max_users = math.ceil(users * 0.01 * io.get('dataset_percentage'))
timer = Timer(math.ceil(len(f) * 0.01 * io.get('dataset_percentage')))
for name, group in f:
user_name = group.iloc[0].get('source_user').split('@')[0]
anomaly_collection = anomalies.get(user_name)
if anomaly_collection is not None:
# Print those rows
user_anomalies = list()
for anomaly in anomaly_collection:
anomaly_dict = {
"start":
anomaly["start"],
"end":
anomaly["end"],
"lines":
listify_df(group.iloc[anomaly["start"]:anomaly["end"]]),
"final_features":
translate_feature_arr(anomaly["final_row_features"]),
"predicted":
anomaly["predicted"],
"actual":
anomaly["actual"],
"loss":
anomaly["loss"]
}
user_anomalies.append(anomaly_dict)
anomaly_rows_list[user_name] = user_anomalies
timer.add_to_current(1)
if timer.current % REPORT_SIZE == 0:
logline('ETA is ' + timer.get_eta())
if timer.current >= max_users:
break
debug('Runtime is', timer.report_total_time())
logline('Generating concatenated results')
if output_file == 'stdout':
logline("Outputting results to stdout\n\n\n")
logline('Final value is', anomaly_rows_list)
logline(json.dumps(anomaly_rows_list))
else:
logline('Outputting results to', output_file)
with open(output_file, 'w') as out_file:
out_file.write(json.dumps(anomaly_rows_list))
logline('Output results to', output_file)
if REMOVE_INPUT_FILE:
os.remove(input_file)
logline('Removed encoded file')
else:
logline('Not Removing encoded file')
logline('Done, closing files and stuff')