def test_create_temp_dir_ok(self):
"""
Check if create_temp_dir behaves correctly when no exception is
raised from inside it.
By behave correctly we mean: creates and returns a tempdir, which
is deleted after we exit the context manager.
"""
with utils.create_temp_dir() as temp_dir:
self.assertTrue(os.path.exists(temp_dir))
self.assertTrue(os.path.isdir(temp_dir))
self.assertFalse(os.path.exists(temp_dir))
def open_repository(repo_url, ref='master'):
"""
Get a `Git` object for a repository URL and switch it to the reference `ref`.
Note that this clones the repository locally.
"""
with utils.create_temp_dir() as repo_dir_path:
logger.info('Cloning repository %s (ref=%s) in %s...', repo_url, ref,
repo_dir_path)
# We can technically clone into a branch directly, but that wouldn't work for arbitrary references.
repo = git.repo.base.Repo.clone_from(repo_url, repo_dir_path)
repo.git.checkout(ref)
repo.submodule_update()
yield repo.git
def run_playbook(requirements_path,
inventory_str,
vars_str,
playbook_path,
playbook_name,
username='******'):
"""
Runs ansible-playbook in a dedicated venv
Ansible only supports Python 2 - so we have to run it as a separate command, in its own venv
"""
with create_temp_dir() as ansible_tmp_dir:
vars_path = string_to_file_path(vars_str, root_dir=ansible_tmp_dir)
inventory_path = string_to_file_path(inventory_str,
root_dir=ansible_tmp_dir)
venv_path = os.path.join(ansible_tmp_dir, 'venv')
cmd = render_sandbox_creation_command(
requirements_path=requirements_path,
inventory_path=inventory_path,
vars_path=vars_path,
playbook_name=playbook_name,
remote_username=username,
venv_path=venv_path)
logger.info('Running: %s', cmd)
# Override TMPDIR environmental variable so any temp files created by ansible (and anything else)
# are created in a directory that we will safely delete after this command exits
env = dict(os.environ)
env['TMPDIR'] = ansible_tmp_dir
# Disable SSH host key checking by Ansible – since IP addresses are constantly reused,
# changing host keys are expected.
env['ANSIBLE_HOST_KEY_CHECKING'] = 'false'
yield subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1, # Buffer one line at a time
cwd=playbook_path,
shell=True,
env=env,
)
def test_create_temp_dir_exception(self):
"""
Check if create_temp_dir behaves correctly when an exception is
raised from inside it.
By behave correctly we mean: creates and returns a tempdir, which
is deleted after we exit the context manager.
"""
saved_temp_dir = None
with self.assertRaises(KeyboardInterrupt):
with utils.create_temp_dir() as temp_dir:
saved_temp_dir = temp_dir
self.assertTrue(os.path.exists(temp_dir))
self.assertTrue(os.path.isdir(temp_dir))
raise KeyboardInterrupt()
self.assertIsNotNone(saved_temp_dir)
self.assertFalse(os.path.exists(saved_temp_dir))
def activity_csv(self, out): # pylint: disable=too-many-locals
"""Generate the activity CSV."""
active_appservers = self.get_active_appservers()
if not active_appservers:
self.stderr.write(
self.style.SUCCESS(
'There are no active app servers! Nothing to do.'))
sys.exit(0)
self.stderr.write(self.style.SUCCESS('Running playbook...'))
with utils.create_temp_dir() as playbook_output_dir:
inventory = '[apps]\n{servers}'.format(
servers='\n'.join(active_appservers.keys()))
playbook_path = os.path.join(
settings.SITE_ROOT,
'playbooks/collect_activity/collect_activity.yml')
def log_line(line):
"""Helper to pass to capture_playbook_output()."""
self.stderr.write(self.style.SUCCESS(line))
log_line.info = log_line
log_line.error = log_line
# Launch the collect_activity playbook, which places a set of files into the `playbook_output_dir`
# on this host.
ansible.capture_playbook_output(
requirements_path=os.path.join(os.path.dirname(playbook_path),
'requirements.txt'),
inventory_str=inventory,
vars_str=(
'local_output_dir: {output_dir}\n'
'remote_output_filename: /tmp/activity_report').format(
output_dir=playbook_output_dir),
playbook_path=playbook_path,
username=settings.OPENSTACK_SANDBOX_SSH_USERNAME,
logger_=log_line,
)
csv_writer = csv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
csv_writer.writerow([
'Appserver IP', 'Internal LMS Domain', 'Name', 'Contact Email',
'Unique Hits', 'Total Users', 'Total Courses', 'Age (Days)'
])
filenames = [
os.path.join(playbook_output_dir, f)
for f in os.listdir(playbook_output_dir)
]
data = ConfigParser()
data.read(filenames)
for public_ip, instance in sorted(active_appservers.items(),
key=lambda tup: tup[1].id):
try:
section = data[public_ip]
except KeyError:
# Fill in stats for any instaces that failed with "N/A"
section = {
'unique_hits': 'N/A',
'users': 'N/A',
'courses': 'N/A'
}
instance_age = datetime.now(
instance.created.tzinfo) - instance.created
try:
email = instance.betatestapplication_set.get().user.email
except BetaTestApplication.DoesNotExist:
email = 'N/A'
csv_writer.writerow([
public_ip, instance.internal_lms_domain, instance.ref.name,
email, section['unique_hits'], section['users'],
section['courses'], instance_age.days
])
self.stderr.write(
self.style.SUCCESS('Done generating CSV output.'))