def _call_script(cool_off_days=37,
batch_size=None,
dry_run=None,
start_date=None,
end_date=None):
"""
Call the archive script with the given params and a generic config file.
Returns the CliRunner.invoke results
"""
runner = CliRunner()
with runner.isolated_filesystem():
with open('test_config.yml', 'w') as f:
fake_config_file(f)
base_args = [
'--config_file',
'test_config.yml',
'--cool_off_days',
cool_off_days,
]
if batch_size:
base_args += ['--batch_size', batch_size]
if dry_run:
base_args += ['--dry_run', dry_run]
if start_date:
base_args += ['--start_date', start_date]
if end_date:
base_args += ['--end_date', end_date]
result = runner.invoke(archive_and_cleanup, args=base_args)
print(result)
print(result.output)
return result
def _call_script(expected_user_files, cool_off_days=1, output_dir='test', user_count_error_threshold=200):
"""
Call the retired learner script with the given username and a generic, temporary config file.
Returns the CliRunner.invoke results
"""
runner = CliRunner()
with runner.isolated_filesystem():
with open('test_config.yml', 'w') as f:
fake_config_file(f)
result = runner.invoke(
get_learners_to_retire,
args=[
'--config_file', 'test_config.yml',
'--cool_off_days', cool_off_days,
'--output_dir', output_dir,
'--user_count_error_threshold', user_count_error_threshold
]
)
print(result)
print(result.output)
# This is the number of users in the mocked call, each should have a file if the number is
# greater than 0, otherwise a failure is expected and the output dir should not exist
if expected_user_files:
assert len(os.listdir(output_dir)) == expected_user_files
else:
assert not os.path.exists(output_dir)
return result
def _call_script(age_in_days=1, expect_success=True):
"""
Call the report deletion script with a generic, temporary config file.
Returns the CliRunner.invoke results
"""
runner = CliRunner()
with runner.isolated_filesystem():
with open(TEST_CONFIG_FILENAME, 'w') as config_f:
fake_config_file(config_f)
with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as secrets_f:
fake_google_secrets_file(secrets_f)
result = runner.invoke(delete_expired_reports,
args=[
'--config_file', TEST_CONFIG_FILENAME,
'--google_secrets_file',
TEST_GOOGLE_SECRETS_FILENAME,
'--age_in_days', age_in_days
])
print(result)
print(result.output)
if expect_success:
assert result.exit_code == 0
return result
def _call_script(expect_success=True, config_orgs=None, file_ids=None):
"""
Call the retired learner script with generic, temporary config files and specified file IDs.
Returns the CliRunner.invoke results.
"""
if config_orgs is None:
config_orgs = FAKE_ORGS
runner = CliRunner()
with runner.isolated_filesystem():
with open(TEST_CONFIG_YML_NAME, 'w') as config_f:
fake_config_file(config_f, config_orgs)
with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as secrets_f:
fake_google_secrets_file(secrets_f)
cmd_args = [
'--config_file',
TEST_CONFIG_YML_NAME,
'--google_secrets_file',
TEST_GOOGLE_SECRETS_FILENAME,
]
if file_ids:
for file_id in file_ids:
cmd_args.extend(['--file_id', file_id])
result = runner.invoke(delete_files, args=cmd_args)
print(result)
print(result.output)
if expect_success:
assert result.exit_code == 0
return result
def _call_script(username, fetch_ecom_segment_id=False):
"""
Call the retired learner script with the given username and a generic, temporary config file.
Returns the CliRunner.invoke results
"""
runner = CliRunner()
with runner.isolated_filesystem():
with open('test_config.yml', 'w') as f:
fake_config_file(f, fetch_ecom_segment_id=fetch_ecom_segment_id)
result = runner.invoke(retire_learner, args=['--username', username, '--config_file', 'test_config.yml'])
print(result)
print(result.output)
return result
def test_no_users_csv_file():
runner = CliRunner()
with runner.isolated_filesystem():
with open(TEST_CONFIG_YML_NAME, 'w') as config_f:
fake_config_file(config_f, [])
result = runner.invoke(bulk_delete_segment_users,
args=[
'--config_file',
TEST_CONFIG_YML_NAME,
])
print(result.output)
assert result.exit_code == ERR_NO_CSV_FILE
assert 'No users CSV file passed in' in result.output
def _call_script(cool_off_days=37):
"""
Call the archive script with the given params and a generic config file.
Returns the CliRunner.invoke results
"""
runner = CliRunner()
with runner.isolated_filesystem():
with open('test_config.yml', 'w') as f:
fake_config_file(f)
result = runner.invoke(archive_and_cleanup,
args=[
'--config_file', 'test_config.yml',
'--cool_off_days', cool_off_days
])
print(result)
print(result.output)
return result
def test_bad_output_dir():
runner = CliRunner()
with runner.isolated_filesystem():
with open(TEST_CONFIG_YML_NAME, 'w') as config_f:
fake_config_file(config_f)
with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f:
fake_google_secrets_file(config_f)
result = runner.invoke(generate_report,
args=[
'--config_file', TEST_CONFIG_YML_NAME,
'--google_secrets_file',
TEST_GOOGLE_SECRETS_FILENAME,
'--output_dir', 'does_not_exist/at_all'
])
print(result.output)
assert result.exit_code == ERR_NO_OUTPUT_DIR
assert 'or path does not exist' in result.output
def test_bad_age_in_days():
runner = CliRunner()
with runner.isolated_filesystem():
with open(TEST_CONFIG_FILENAME, 'w') as config_f:
fake_config_file(config_f)
with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f:
fake_google_secrets_file(config_f)
result = runner.invoke(delete_expired_reports,
args=[
'--config_file', TEST_CONFIG_FILENAME,
'--google_secrets_file',
TEST_GOOGLE_SECRETS_FILENAME,
'--age_in_days', -1000
])
print(result.output)
assert result.exit_code == ERR_BAD_AGE
assert 'must be a positive integer' in result.output
def test_bad_secrets():
runner = CliRunner()
with runner.isolated_filesystem():
with open(TEST_CONFIG_YML_NAME, 'w') as config_f:
fake_config_file(config_f)
with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f:
config_f.write('{this is bad json')
result = runner.invoke(delete_files,
args=[
'--config_file', TEST_CONFIG_YML_NAME,
'--google_secrets_file',
TEST_GOOGLE_SECRETS_FILENAME, '--file_id',
'a_fake_file_id'
])
print(result.output)
assert result.exit_code == ERR_BAD_SECRETS
assert 'Failed to read' in result.output
def test_bad_secrets():
runner = CliRunner()
with runner.isolated_filesystem():
with open(TEST_CONFIG_YML_NAME, 'w') as config_f:
fake_config_file(config_f)
with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as config_f:
config_f.write('{this is bad json')
tmp_output_dir = 'test_output_dir'
os.mkdir(tmp_output_dir)
result = runner.invoke(generate_report,
args=[
'--config_file', TEST_CONFIG_YML_NAME,
'--google_secrets_file',
TEST_GOOGLE_SECRETS_FILENAME,
'--output_dir', tmp_output_dir
])
print(result.output)
assert result.exit_code == ERR_BAD_SECRETS
assert 'Failed to read' in result.output
def _call_script(initial_state='COMPLETE',
new_state='PENDING',
start_date='2018-01-01',
end_date='2018-01-15'):
"""
Call the bulk update statuses script with the given params and a generic config file.
Returns the CliRunner.invoke results
"""
runner = CliRunner()
with runner.isolated_filesystem():
with open('test_config.yml', 'w') as f:
fake_config_file(f)
result = runner.invoke(update_statuses,
args=[
'--config_file', 'test_config.yml',
'--initial_state', initial_state,
'--new_state', new_state, '--start_date',
start_date, '--end_date', end_date
])
print(result)
print(result.output)
return result
def _call_script(expect_success=True, config_orgs=None, learners_to_delete=None):
"""
Call the retired learner script with generic, temporary config files and specified learners.
Returns the CliRunner.invoke results.
"""
if config_orgs is None:
config_orgs = FAKE_ORGS
runner = CliRunner()
with runner.isolated_filesystem():
with open(TEST_CONFIG_YML_NAME, 'w') as config_f:
fake_config_file(config_f, config_orgs)
if learners_to_delete:
with open(TEST_RETIRED_USERS_CSV_NAME, 'w') as users_f:
for learner in learners_to_delete:
users_f.write(','.join(learner))
cmd_args = [
'--config_file',
TEST_CONFIG_YML_NAME,
'--retired_users_csv',
TEST_RETIRED_USERS_CSV_NAME,
]
result = runner.invoke(
bulk_delete_segment_users,
args=cmd_args
)
print(result)
print(result.output)
if expect_success:
assert result.exit_code == 0
return result
def _call_script(expect_success=True,
expected_num_rows=10,
config_orgs=None,
expected_fields=None):
"""
Call the retired learner script with the given username and a generic, temporary config file.
Returns the CliRunner.invoke results
"""
if expected_fields is None:
expected_fields = DEFAULT_FIELD_VALUES
if config_orgs is None:
config_orgs = FAKE_ORGS
runner = CliRunner()
with runner.isolated_filesystem():
with open(TEST_CONFIG_YML_NAME, 'w') as config_f:
fake_config_file(config_f, config_orgs)
with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as secrets_f:
fake_google_secrets_file(secrets_f)
tmp_output_dir = 'test_output_dir'
os.mkdir(tmp_output_dir)
result = runner.invoke(generate_report,
args=[
'--config_file', TEST_CONFIG_YML_NAME,
'--google_secrets_file',
TEST_GOOGLE_SECRETS_FILENAME,
'--output_dir', tmp_output_dir
])
print(result)
print(result.output)
if expect_success:
assert result.exit_code == 0
if config_orgs is None:
# These are the orgs
config_org_vals = FAKE_ORGS.values()
else:
config_org_vals = config_orgs.values()
# Normalize the unicode as the script does
if PY2:
config_org_vals = [
org.decode('utf-8') for org in config_org_vals
]
config_org_vals = [
unicodedata.normalize('NFKC', org) for org in config_org_vals
]
for org in config_org_vals:
outfile = os.path.join(
tmp_output_dir,
'{}_{}_{}_{}.csv'.format(REPORTING_FILENAME_PREFIX,
TEST_PLATFORM_NAME, org,
date.today().isoformat()))
with open(outfile, 'r') as csvfile:
reader = csv.DictReader(csvfile)
rows = []
for row in reader:
for field_key in expected_fields:
field_value = expected_fields[field_key]
assert field_value in row[field_key]
rows.append(row)
# Confirm the number of rows
assert len(rows) == expected_num_rows
return result
def _call_script(expect_success=True, config_orgs=None):
"""
Call the retired learner script with the given username and a generic, temporary config file.
Returns the CliRunner.invoke results
"""
if config_orgs is None:
config_orgs = FAKE_ORGS
runner = CliRunner()
with runner.isolated_filesystem():
with open(TEST_CONFIG_YML_NAME, 'w') as config_f:
fake_config_file(config_f, config_orgs)
with open(TEST_GOOGLE_SECRETS_FILENAME, 'w') as secrets_f:
fake_google_secrets_file(secrets_f)
tmp_output_dir = 'test_output_dir'
os.mkdir(tmp_output_dir)
result = runner.invoke(generate_report,
args=[
'--config_file', TEST_CONFIG_YML_NAME,
'--google_secrets_file',
TEST_GOOGLE_SECRETS_FILENAME,
'--output_dir', tmp_output_dir
])
print(result)
print(result.output)
if expect_success:
assert result.exit_code == 0
if config_orgs is None:
# These are the orgs
config_org_vals = FAKE_ORGS.values()
else:
config_org_vals = config_orgs.values()
# Normalize the unicode as the script does
if PY2:
config_org_vals = [
org.decode('utf-8') for org in config_org_vals
]
config_org_vals = [
unicodedata.normalize('NFKC', org) for org in config_org_vals
]
for org in config_org_vals:
outfile = os.path.join(
tmp_output_dir,
'{}_{}_{}_{}.csv'.format(REPORTING_FILENAME_PREFIX,
TEST_PLATFORM_NAME, org,
date.today().isoformat()))
with open(outfile, 'r') as csvfile:
reader = csv.DictReader(csvfile)
rows = []
for row in reader:
# Check the user_id value is in the correct place
assert USER_ID in row['user_id']
# Check username value is in the correct place
assert 'username' in row['original_username']
# Check email value is in the correct place
assert 'invalid' in row['original_email']
# Check name value is in the correct place
assert UNICODE_NAME_CONSTANT in row['original_name']
# Check deletion_completed value is in the correct place
assert DELETION_TIME in row['deletion_completed']
rows.append(row)
# Confirm that there are rows at all
assert rows
return result
