def do_first_job(file_location: str):
experiments = get_experiments(file_location)
did_job = False
idx = 0
for cmd, state in experiments.items():
idx = idx + 1
if state == EXPERIMENT_NOT_DONE:
# Do this job
experiments[cmd] = EXPERIMENT_BUSY
upload_experiments(file_location, experiments)
did_job = True
exit_code = do_job(cmd +
(' -g 16' if io.get('use_gpus') else ' -g 0'))
if exit_code != 0:
error('An error occurred while executing command', cmd,
'giving exit code', exit_code)
try_notify('A command failed')
upload_experiments(file_location,
experiments,
is_error=True,
error_code=exit_code)
sys.exit(1)
else:
experiments[cmd] = EXPERIMENT_DONE
logline('Done with job', cmd)
try_notify('Done with job ' + str(idx))
upload_experiments(file_location, experiments)
break
return did_job
def extract_features(rows):
users_list = list()
users = len(rows)
rows_amount = 0
logline(
'There are', users, 'users and', len(rows),
'rows matching your filter type',
'no computer users or anonymous users'
if io.get('users_only') else 'no anonymous users')
rows_max = get_dict_inner_length(rows)
logline('Setting timer for', rows_max, 'rows')
timer = Timer(rows_max)
try:
for name, group in rows.items():
completed_result, group_len = strip_group_length(
gen_features_for_user((name, group)))
timer.add_to_current(group_len)
rows_amount += group_len
if completed_result is not None:
users_list.append(completed_result)
if rows_amount > next_report == 0 or REPORT_EVERY_USER:
next_report = next_report + REPORT_SIZE
logline('At row ',
str(rows_amount),
'/~',
str(row_amount),
' - ETA is: ' + timer.get_eta(),
spaces_between=False)
logline('At user ',
len(users_list),
'/~',
max_users,
spaces_between=False)
if len(users_list) >= max_users:
break
except KeyboardInterrupt:
logline('User cancelled execution, wrapping up')
debug('Cancelled early at', len(users_list), 'instead of', users)
debug('You skipped a total of', users - len(users_list), 'users, or',
100 - ((len(users_list) / users) * 100), '%')
except Exception:
error('An error occurred during execution', traceback.format_exc())
debug('Salvaging all remaining users')
finally:
debug('Runtime is', timer.report_total_time())
logline("Did a total of", len(users_list), "users")
logline('Done gathering data')
logline('Closing file...')
output_data(users_list)
def strip_group_length(data) -> Tuple[Union[Dict[str, Any], None], int]:
if 'error' in data and data['error']:
if data['group_len'] == -1:
error(
'Value too big for pickle returned, skipping it, ETA might be off now'
)
return None, 0
return None, data['group_len']
group_length = data['group_len']
return {
"user_name": data['user_name'],
"datasets": data['datasets']
}, group_length
def get_experiments(experiments_file_location: str) -> Dict[str, int]:
with open(experiments_file_location, 'r+') as experiments_file:
try:
obj = json.loads(experiments_file.read())
except Exception:
# No state set yet
set_state(experiments_file_location, 0)
return {}
if 'error' in obj:
error(
'An error occurred in another instance, exiting with error code',
obj['error_code'])
sys.exit(obj['error_code'])
return obj
def output_data(users_list: List[Dict[str, Union[str,
Dict[str,
List[List[float]]]]]]):
if io.get('output_file') == 'stdout':
logline('Outputting to stdout')
sys.stdout.write(json.dumps(users_list))
else:
logline('Outputting data to file', io.get('output_file'))
output = open(io.get('output_file'), 'wb')
try:
pickle.dump(users_list, output, protocol=4)
except:
try:
logline("Using JSON instead")
output.write(json.dumps(users_list))
except:
error('Outputting to console instead')
print(json.dumps(users_list))
raise
raise
logline('Done outputting data to file')
def get_state(state_file_location: str) -> int:
try:
with open(state_file_location, 'r+') as state_file:
try:
state_obj = json.loads(state_file.read())
except Exception:
error('State file does not exist, cancelling')
sys.exit(2)
if state_obj["error"]:
error(
'An error occurred in another instance, exiting with error code',
state_obj['error_code'])
sys.exit(state_obj['error_code'])
return state_obj['state']
except FileNotFoundError:
error('State file does not exist, cancelling')
sys.exit(2)
def gen_features(f: pd.DataFrame, row_amount: int):
users_list = list()
logline('Calculating amount of groups...')
users = len(f)
logline(
'There are', users, 'users and', row_amount,
'rows matching your filter type',
'no computer users or anonymous users'
if io.get('users_only') else 'no anonymous users')
rows = 0
max_users = users
if not DO_ROWS_PERCENTAGE:
max_users = int(math.ceil(users * 0.01 * io.get('dataset_percentage')))
logline('Max amount of users is', max_users)
logline('Setting timer for',
int(math.ceil(row_amount * 0.01 * io.get('dataset_percentage'))),
'rows')
timer = Timer(
int(math.ceil(row_amount * 0.01 * io.get('dataset_percentage'))))
logline('Creating iterator')
dataset_iterator = DFIterator(f)
next_report = REPORT_SIZE
if not SKIP_MAIN:
try:
# Create groups of approx 1000 users big
if io.get('cpus') == 1:
logline('Only using a single CPU')
logline('Starting feature generation')
for name, group in f:
completed_result, group_len = strip_group_length(
gen_features_for_user((name, group)))
timer.add_to_current(group_len)
rows += group_len
if completed_result is not None:
users_list.append(completed_result)
if rows > next_report == 0 or REPORT_EVERY_USER:
next_report = next_report + REPORT_SIZE
logline('At row ',
str(rows),
'/~',
str(row_amount),
' - ETA is: ' + timer.get_eta(),
spaces_between=False)
logline('At user ',
len(users_list),
'/~',
max_users,
spaces_between=False)
if len(users_list) >= max_users:
break
else:
logline('Using', io.get('cpus'), 'cpus')
for i in range(
round(math.ceil(max_users / PROCESSING_GROUP_SIZE))):
dataset_iterator.set_max((i + 1) * PROCESSING_GROUP_SIZE)
if i == 0:
logline('Starting feature generation')
with multiprocessing.Pool(io.get('cpus')) as p:
for completed_result in p.imap_unordered(
gen_features_for_user,
dataset_iterator,
chunksize=100):
completed_result, group_len = strip_group_length(
completed_result)
timer.add_to_current(group_len)
rows += group_len
if completed_result is not None:
users_list.append(completed_result)
if rows > next_report or REPORT_EVERY_USER:
next_report = next_report + REPORT_SIZE
logline('At row ',
str(rows),
'/~',
str(row_amount),
' - ETA is: ' + timer.get_eta(),
spaces_between=False)
logline('At user',
len(users_list),
'/~',
max_users,
spaces_between=False)
except KeyboardInterrupt:
logline('User cancelled execution, wrapping up')
debug('Cancelled early at', len(users_list), 'instead of', users)
debug('You skipped a total of', users - len(users_list),
'users, or', 100 - ((len(users_list) / users) * 100), '%')
except Exception:
error('An error occurred during execution', traceback.format_exc())
debug('Salvaging all remaining users')
finally:
debug('Runtime is', timer.report_total_time())
logline("Did a total of", len(users_list), "users")
logline('Done gathering data')
logline('Closing file...')
output_data(users_list)
else:
debug('SKIPPING MAIN, DO NOT ENABLE IN PRODUCTION')
logline('Closing file')
output_data([])
def try_notify(message: str):
try:
do_job(NOTIFY_TEMPLATE.substitute(message=message))
except Exception as e:
error('Got an error notifying that job is done, np', e)