def _call_script(cool_off_days=37, batch_size=None, dry_run=None, start_date=None, end_date=None): """ Call the archive script with the given params and a generic config file. Returns the CliRunner.invoke results """ runner = CliRunner() with runner.isolated_filesystem(): with open('test_config.yml', 'w') as f: fake_config_file(f) base_args = [ '--config_file', 'test_config.yml', '--cool_off_days', cool_off_days, ] if batch_size: base_args += ['--batch_size', batch_size] if dry_run: base_args += ['--dry_run', dry_run] if start_date: base_args += ['--start_date', start_date] if end_date: base_args += ['--end_date', end_date] result = runner.invoke(archive_and_cleanup, args=base_args) print(result) print(result.output) return result
def _call_script(expected_user_files, cool_off_days=1, output_dir='test', user_count_error_threshold=200): """ Call the retired learner script with the given username and a generic, temporary config file. Returns the CliRunner.invoke results """ runner = CliRunner() with runner.isolated_filesystem(): with open('test_config.yml', 'w') as f: fake_config_file(f) result = runner.invoke( get_learners_to_retire, args=[ '--config_file', 'test_config.yml', '--cool_off_days', cool_off_days, '--output_dir', output_dir, '--user_count_error_threshold', user_count_error_threshold ] ) print(result) print(result.output) # This is the number of users in the mocked call, each should have a file if the number is # greater than 0, otherwise a failure is expected and the output dir should not exist if expected_user_files: assert len(os.listdir(output_dir)) == expected_user_files else: assert not os.path.exists(output_dir) return result
def _call_script(age_in_days=1, expect_success=True): """ Call the report deletion script with a generic, temporary config file. Returns the CliRunner.invoke results """ runner = CliRunner() with runner.isolated_filesystem(): with open(TEST_CONFIG_FILENAME, 'w') as config_f: fake_config_file(config_f) with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as secrets_f: fake_google_secrets_file(secrets_f) result = runner.invoke(delete_expired_reports, args=[ '--config_file', TEST_CONFIG_FILENAME, '--google_secrets_file', TEST_GOOGLE_SECRETS_FILENAME, '--age_in_days', age_in_days ]) print(result) print(result.output) if expect_success: assert result.exit_code == 0 return result
def _call_script(expect_success=True, config_orgs=None, file_ids=None): """ Call the retired learner script with generic, temporary config files and specified file IDs. Returns the CliRunner.invoke results. """ if config_orgs is None: config_orgs = FAKE_ORGS runner = CliRunner() with runner.isolated_filesystem(): with open(TEST_CONFIG_YML_NAME, 'w') as config_f: fake_config_file(config_f, config_orgs) with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as secrets_f: fake_google_secrets_file(secrets_f) cmd_args = [ '--config_file', TEST_CONFIG_YML_NAME, '--google_secrets_file', TEST_GOOGLE_SECRETS_FILENAME, ] if file_ids: for file_id in file_ids: cmd_args.extend(['--file_id', file_id]) result = runner.invoke(delete_files, args=cmd_args) print(result) print(result.output) if expect_success: assert result.exit_code == 0 return result
def _call_script(username, fetch_ecom_segment_id=False): """ Call the retired learner script with the given username and a generic, temporary config file. Returns the CliRunner.invoke results """ runner = CliRunner() with runner.isolated_filesystem(): with open('test_config.yml', 'w') as f: fake_config_file(f, fetch_ecom_segment_id=fetch_ecom_segment_id) result = runner.invoke(retire_learner, args=['--username', username, '--config_file', 'test_config.yml']) print(result) print(result.output) return result
def test_no_users_csv_file(): runner = CliRunner() with runner.isolated_filesystem(): with open(TEST_CONFIG_YML_NAME, 'w') as config_f: fake_config_file(config_f, []) result = runner.invoke(bulk_delete_segment_users, args=[ '--config_file', TEST_CONFIG_YML_NAME, ]) print(result.output) assert result.exit_code == ERR_NO_CSV_FILE assert 'No users CSV file passed in' in result.output
def _call_script(cool_off_days=37): """ Call the archive script with the given params and a generic config file. Returns the CliRunner.invoke results """ runner = CliRunner() with runner.isolated_filesystem(): with open('test_config.yml', 'w') as f: fake_config_file(f) result = runner.invoke(archive_and_cleanup, args=[ '--config_file', 'test_config.yml', '--cool_off_days', cool_off_days ]) print(result) print(result.output) return result
def test_bad_output_dir(): runner = CliRunner() with runner.isolated_filesystem(): with open(TEST_CONFIG_YML_NAME, 'w') as config_f: fake_config_file(config_f) with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: fake_google_secrets_file(config_f) result = runner.invoke(generate_report, args=[ '--config_file', TEST_CONFIG_YML_NAME, '--google_secrets_file', TEST_GOOGLE_SECRETS_FILENAME, '--output_dir', 'does_not_exist/at_all' ]) print(result.output) assert result.exit_code == ERR_NO_OUTPUT_DIR assert 'or path does not exist' in result.output
def test_bad_age_in_days(): runner = CliRunner() with runner.isolated_filesystem(): with open(TEST_CONFIG_FILENAME, 'w') as config_f: fake_config_file(config_f) with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: fake_google_secrets_file(config_f) result = runner.invoke(delete_expired_reports, args=[ '--config_file', TEST_CONFIG_FILENAME, '--google_secrets_file', TEST_GOOGLE_SECRETS_FILENAME, '--age_in_days', -1000 ]) print(result.output) assert result.exit_code == ERR_BAD_AGE assert 'must be a positive integer' in result.output
def test_bad_secrets(): runner = CliRunner() with runner.isolated_filesystem(): with open(TEST_CONFIG_YML_NAME, 'w') as config_f: fake_config_file(config_f) with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: config_f.write('{this is bad json') result = runner.invoke(delete_files, args=[ '--config_file', TEST_CONFIG_YML_NAME, '--google_secrets_file', TEST_GOOGLE_SECRETS_FILENAME, '--file_id', 'a_fake_file_id' ]) print(result.output) assert result.exit_code == ERR_BAD_SECRETS assert 'Failed to read' in result.output
def test_bad_secrets(): runner = CliRunner() with runner.isolated_filesystem(): with open(TEST_CONFIG_YML_NAME, 'w') as config_f: fake_config_file(config_f) with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f: config_f.write('{this is bad json') tmp_output_dir = 'test_output_dir' os.mkdir(tmp_output_dir) result = runner.invoke(generate_report, args=[ '--config_file', TEST_CONFIG_YML_NAME, '--google_secrets_file', TEST_GOOGLE_SECRETS_FILENAME, '--output_dir', tmp_output_dir ]) print(result.output) assert result.exit_code == ERR_BAD_SECRETS assert 'Failed to read' in result.output
def _call_script(initial_state='COMPLETE', new_state='PENDING', start_date='2018-01-01', end_date='2018-01-15'): """ Call the bulk update statuses script with the given params and a generic config file. Returns the CliRunner.invoke results """ runner = CliRunner() with runner.isolated_filesystem(): with open('test_config.yml', 'w') as f: fake_config_file(f) result = runner.invoke(update_statuses, args=[ '--config_file', 'test_config.yml', '--initial_state', initial_state, '--new_state', new_state, '--start_date', start_date, '--end_date', end_date ]) print(result) print(result.output) return result
def _call_script(expect_success=True, config_orgs=None, learners_to_delete=None): """ Call the retired learner script with generic, temporary config files and specified learners. Returns the CliRunner.invoke results. """ if config_orgs is None: config_orgs = FAKE_ORGS runner = CliRunner() with runner.isolated_filesystem(): with open(TEST_CONFIG_YML_NAME, 'w') as config_f: fake_config_file(config_f, config_orgs) if learners_to_delete: with open(TEST_RETIRED_USERS_CSV_NAME, 'w') as users_f: for learner in learners_to_delete: users_f.write(','.join(learner)) cmd_args = [ '--config_file', TEST_CONFIG_YML_NAME, '--retired_users_csv', TEST_RETIRED_USERS_CSV_NAME, ] result = runner.invoke( bulk_delete_segment_users, args=cmd_args ) print(result) print(result.output) if expect_success: assert result.exit_code == 0 return result
def _call_script(expect_success=True, expected_num_rows=10, config_orgs=None, expected_fields=None): """ Call the retired learner script with the given username and a generic, temporary config file. Returns the CliRunner.invoke results """ if expected_fields is None: expected_fields = DEFAULT_FIELD_VALUES if config_orgs is None: config_orgs = FAKE_ORGS runner = CliRunner() with runner.isolated_filesystem(): with open(TEST_CONFIG_YML_NAME, 'w') as config_f: fake_config_file(config_f, config_orgs) with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as secrets_f: fake_google_secrets_file(secrets_f) tmp_output_dir = 'test_output_dir' os.mkdir(tmp_output_dir) result = runner.invoke(generate_report, args=[ '--config_file', TEST_CONFIG_YML_NAME, '--google_secrets_file', TEST_GOOGLE_SECRETS_FILENAME, '--output_dir', tmp_output_dir ]) print(result) print(result.output) if expect_success: assert result.exit_code == 0 if config_orgs is None: # These are the orgs config_org_vals = FAKE_ORGS.values() else: config_org_vals = config_orgs.values() # Normalize the unicode as the script does if PY2: config_org_vals = [ org.decode('utf-8') for org in config_org_vals ] config_org_vals = [ unicodedata.normalize('NFKC', org) for org in config_org_vals ] for org in config_org_vals: outfile = os.path.join( tmp_output_dir, '{}_{}_{}_{}.csv'.format(REPORTING_FILENAME_PREFIX, TEST_PLATFORM_NAME, org, date.today().isoformat())) with open(outfile, 'r') as csvfile: reader = csv.DictReader(csvfile) rows = [] for row in reader: for field_key in expected_fields: field_value = expected_fields[field_key] assert field_value in row[field_key] rows.append(row) # Confirm the number of rows assert len(rows) == expected_num_rows return result
def _call_script(expect_success=True, config_orgs=None): """ Call the retired learner script with the given username and a generic, temporary config file. Returns the CliRunner.invoke results """ if config_orgs is None: config_orgs = FAKE_ORGS runner = CliRunner() with runner.isolated_filesystem(): with open(TEST_CONFIG_YML_NAME, 'w') as config_f: fake_config_file(config_f, config_orgs) with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as secrets_f: fake_google_secrets_file(secrets_f) tmp_output_dir = 'test_output_dir' os.mkdir(tmp_output_dir) result = runner.invoke(generate_report, args=[ '--config_file', TEST_CONFIG_YML_NAME, '--google_secrets_file', TEST_GOOGLE_SECRETS_FILENAME, '--output_dir', tmp_output_dir ]) print(result) print(result.output) if expect_success: assert result.exit_code == 0 if config_orgs is None: # These are the orgs config_org_vals = FAKE_ORGS.values() else: config_org_vals = config_orgs.values() # Normalize the unicode as the script does if PY2: config_org_vals = [ org.decode('utf-8') for org in config_org_vals ] config_org_vals = [ unicodedata.normalize('NFKC', org) for org in config_org_vals ] for org in config_org_vals: outfile = os.path.join( tmp_output_dir, '{}_{}_{}_{}.csv'.format(REPORTING_FILENAME_PREFIX, TEST_PLATFORM_NAME, org, date.today().isoformat())) with open(outfile, 'r') as csvfile: reader = csv.DictReader(csvfile) rows = [] for row in reader: # Check the user_id value is in the correct place assert USER_ID in row['user_id'] # Check username value is in the correct place assert 'username' in row['original_username'] # Check email value is in the correct place assert 'invalid' in row['original_email'] # Check name value is in the correct place assert UNICODE_NAME_CONSTANT in row['original_name'] # Check deletion_completed value is in the correct place assert DELETION_TIME in row['deletion_completed'] rows.append(row) # Confirm that there are rows at all assert rows return result