예제 #1
0
def do_first_job(file_location: str):
    experiments = get_experiments(file_location)

    did_job = False
    idx = 0
    for cmd, state in experiments.items():
        idx = idx + 1
        if state == EXPERIMENT_NOT_DONE:
            # Do this job
            experiments[cmd] = EXPERIMENT_BUSY
            upload_experiments(file_location, experiments)
            did_job = True

            exit_code = do_job(cmd +
                               (' -g 16' if io.get('use_gpus') else ' -g 0'))
            if exit_code != 0:
                error('An error occurred while executing command', cmd,
                      'giving exit code', exit_code)
                try_notify('A command failed')
                upload_experiments(file_location,
                                   experiments,
                                   is_error=True,
                                   error_code=exit_code)
                sys.exit(1)
            else:
                experiments[cmd] = EXPERIMENT_DONE
                logline('Done with job', cmd)
                try_notify('Done with job ' + str(idx))
                upload_experiments(file_location, experiments)
            break

    return did_job
예제 #2
0
def extract_features(rows):
    users_list = list()
    users = len(rows)
    rows_amount = 0

    logline(
        'There are', users, 'users and', len(rows),
        'rows matching your filter type',
        'no computer users or anonymous users'
        if io.get('users_only') else 'no anonymous users')

    rows_max = get_dict_inner_length(rows)
    logline('Setting timer for', rows_max, 'rows')
    timer = Timer(rows_max)

    try:
        for name, group in rows.items():
            completed_result, group_len = strip_group_length(
                gen_features_for_user((name, group)))

            timer.add_to_current(group_len)
            rows_amount += group_len

            if completed_result is not None:
                users_list.append(completed_result)

                if rows_amount > next_report == 0 or REPORT_EVERY_USER:
                    next_report = next_report + REPORT_SIZE

                    logline('At row ',
                            str(rows_amount),
                            '/~',
                            str(row_amount),
                            ' - ETA is: ' + timer.get_eta(),
                            spaces_between=False)
                    logline('At user ',
                            len(users_list),
                            '/~',
                            max_users,
                            spaces_between=False)

            if len(users_list) >= max_users:
                break
    except KeyboardInterrupt:
        logline('User cancelled execution, wrapping up')
        debug('Cancelled early at', len(users_list), 'instead of', users)
        debug('You skipped a total of', users - len(users_list), 'users, or',
              100 - ((len(users_list) / users) * 100), '%')
    except Exception:
        error('An error occurred during execution', traceback.format_exc())
        debug('Salvaging all remaining users')
    finally:
        debug('Runtime is', timer.report_total_time())

        logline("Did a total of", len(users_list), "users")
        logline('Done gathering data')
        logline('Closing file...')
        output_data(users_list)
예제 #3
0
def strip_group_length(data) -> Tuple[Union[Dict[str, Any], None], int]:
    if 'error' in data and data['error']:
        if data['group_len'] == -1:
            error(
                'Value too big for pickle returned, skipping it, ETA might be off now'
            )
            return None, 0
        return None, data['group_len']

    group_length = data['group_len']
    return {
        "user_name": data['user_name'],
        "datasets": data['datasets']
    }, group_length
예제 #4
0
def get_experiments(experiments_file_location: str) -> Dict[str, int]:
    with open(experiments_file_location, 'r+') as experiments_file:
        try:
            obj = json.loads(experiments_file.read())
        except Exception:
            # No state set yet
            set_state(experiments_file_location, 0)
            return {}

        if 'error' in obj:
            error(
                'An error occurred in another instance, exiting with error code',
                obj['error_code'])
            sys.exit(obj['error_code'])
        return obj
예제 #5
0
def output_data(users_list: List[Dict[str, Union[str,
                                                 Dict[str,
                                                      List[List[float]]]]]]):
    if io.get('output_file') == 'stdout':
        logline('Outputting to stdout')
        sys.stdout.write(json.dumps(users_list))
    else:
        logline('Outputting data to file', io.get('output_file'))
        output = open(io.get('output_file'), 'wb')
        try:
            pickle.dump(users_list, output, protocol=4)
        except:
            try:
                logline("Using JSON instead")
                output.write(json.dumps(users_list))
            except:
                error('Outputting to console instead')
                print(json.dumps(users_list))
                raise
            raise
        logline('Done outputting data to file')
def get_state(state_file_location: str) -> int:
    try:
        with open(state_file_location, 'r+') as state_file:
            try:
                state_obj = json.loads(state_file.read())
            except Exception:
                error('State file does not exist, cancelling')
                sys.exit(2)

            if state_obj["error"]:
                error(
                    'An error occurred in another instance, exiting with error code',
                    state_obj['error_code'])
                sys.exit(state_obj['error_code'])
            return state_obj['state']
    except FileNotFoundError:
        error('State file does not exist, cancelling')
        sys.exit(2)
예제 #7
0
def gen_features(f: pd.DataFrame, row_amount: int):
    users_list = list()

    logline('Calculating amount of groups...')
    users = len(f)
    logline(
        'There are', users, 'users and', row_amount,
        'rows matching your filter type',
        'no computer users or anonymous users'
        if io.get('users_only') else 'no anonymous users')
    rows = 0

    max_users = users
    if not DO_ROWS_PERCENTAGE:
        max_users = int(math.ceil(users * 0.01 * io.get('dataset_percentage')))
    logline('Max amount of users is', max_users)

    logline('Setting timer for',
            int(math.ceil(row_amount * 0.01 * io.get('dataset_percentage'))),
            'rows')
    timer = Timer(
        int(math.ceil(row_amount * 0.01 * io.get('dataset_percentage'))))

    logline('Creating iterator')
    dataset_iterator = DFIterator(f)

    next_report = REPORT_SIZE

    if not SKIP_MAIN:
        try:
            # Create groups of approx 1000 users big
            if io.get('cpus') == 1:
                logline('Only using a single CPU')
                logline('Starting feature generation')
                for name, group in f:
                    completed_result, group_len = strip_group_length(
                        gen_features_for_user((name, group)))

                    timer.add_to_current(group_len)
                    rows += group_len

                    if completed_result is not None:
                        users_list.append(completed_result)

                        if rows > next_report == 0 or REPORT_EVERY_USER:
                            next_report = next_report + REPORT_SIZE

                            logline('At row ',
                                    str(rows),
                                    '/~',
                                    str(row_amount),
                                    ' - ETA is: ' + timer.get_eta(),
                                    spaces_between=False)
                            logline('At user ',
                                    len(users_list),
                                    '/~',
                                    max_users,
                                    spaces_between=False)

                    if len(users_list) >= max_users:
                        break

            else:
                logline('Using', io.get('cpus'), 'cpus')
                for i in range(
                        round(math.ceil(max_users / PROCESSING_GROUP_SIZE))):
                    dataset_iterator.set_max((i + 1) * PROCESSING_GROUP_SIZE)
                    if i == 0:
                        logline('Starting feature generation')

                    with multiprocessing.Pool(io.get('cpus')) as p:
                        for completed_result in p.imap_unordered(
                                gen_features_for_user,
                                dataset_iterator,
                                chunksize=100):

                            completed_result, group_len = strip_group_length(
                                completed_result)
                            timer.add_to_current(group_len)
                            rows += group_len

                            if completed_result is not None:
                                users_list.append(completed_result)

                                if rows > next_report or REPORT_EVERY_USER:
                                    next_report = next_report + REPORT_SIZE
                                    logline('At row ',
                                            str(rows),
                                            '/~',
                                            str(row_amount),
                                            ' - ETA is: ' + timer.get_eta(),
                                            spaces_between=False)
                                    logline('At user',
                                            len(users_list),
                                            '/~',
                                            max_users,
                                            spaces_between=False)
        except KeyboardInterrupt:
            logline('User cancelled execution, wrapping up')
            debug('Cancelled early at', len(users_list), 'instead of', users)
            debug('You skipped a total of', users - len(users_list),
                  'users, or', 100 - ((len(users_list) / users) * 100), '%')
        except Exception:
            error('An error occurred during execution', traceback.format_exc())
            debug('Salvaging all remaining users')
        finally:
            debug('Runtime is', timer.report_total_time())

            logline("Did a total of", len(users_list), "users")
            logline('Done gathering data')
            logline('Closing file...')
            output_data(users_list)
    else:
        debug('SKIPPING MAIN, DO NOT ENABLE IN PRODUCTION')
        logline('Closing file')
        output_data([])
예제 #8
0
def try_notify(message: str):
    try:
        do_job(NOTIFY_TEMPLATE.substitute(message=message))
    except Exception as e:
        error('Got an error notifying that job is done, np', e)