def run_playbook(self, ansible_vars):
"""
Run the playbook to perform the server reconfiguration.
This is factored out into a separate method so it can be mocked out in the tests.
"""
if settings.DISABLE_LOAD_BALANCER_CONFIGURATION:
self.logger.info(
'Direct load balancer reconfiguration disabled. Skipping %s configuration...',
self)
return
playbook_path = pathlib.Path(
settings.SITE_ROOT
) / "playbooks/load_balancer_conf/load_balancer_conf.yml"
returncode = ansible.capture_playbook_output(
requirements_path=str(playbook_path.parent / "requirements.txt"),
inventory_str=self.domain,
vars_str=ansible_vars,
playbook_path=str(playbook_path),
username=self.ssh_username,
logger_=self.logger,
)
if returncode != 0:
self.logger.error(
"Playbook to reconfigure load-balancing server %s failed.",
self)
raise ReconfigurationFailed
def run_cleanup(self):
"""Run the actual cleanup"""
logger.info("\n --- Starting Load balancer fragments cleanup ---")
if self.dry_run:
logger.info("Running in DRY_RUN mode, no actions will be taken")
playbook_path = pathlib.Path(
os.path.abspath(os.path.dirname(__file__))
) / "playbooks/load_balancer_cleanup.yml"
ansible_vars = (
"DAYS_OLDER_THAN: {age_limit}\n"
"FRAGMENT_PATTERN: {fragment_prefix}*\n"
"REMOVE_FRAGMENTS: {remove_fragments}\n"
).format(
age_limit=self.age_limit,
fragment_prefix=self.fragment_prefix,
remove_fragments='true' if self.dry_run else 'false'
)
return_code = ansible.capture_playbook_output(
requirements_path=str(playbook_path.parent / "requirements.txt"),
playbook_path=str(playbook_path),
inventory_str=self.load_balancer_address,
vars_str=ansible_vars,
username='******',
logger_=logger,
)
if return_code != 0:
raise Exception("Playbook to remove stale load balancer fragments failed")
def get_elasticsearch_hits_data_summary(self, playbook_output_dir, name_prefix, start_date, end_date):
""" Execute the collect_elasticsearch_data playbook to gather statistics """
if not settings.INSTANCE_LOGS_SERVER_HOST:
self.stderr.write(self.style.WARNING(
'Skipping Elasticsearch data collection because '
'INSTANCE_LOGS_SERVER_HOST is unset'
))
return
inventory = '[apps]\n{server}'.format(server=settings.INSTANCE_LOGS_SERVER_HOST)
playbook_path = os.path.join(
settings.SITE_ROOT,
'playbooks/collect_instance_statistics/collect_elasticsearch_data.yml'
)
def log_line(line):
"""Helper to pass to capture_playbook_output()."""
self.stderr.write(self.style.SUCCESS(line))
log_line.info = log_line
log_line.error = log_line
# Launch the collect_elasticsearch_data playbook, which places a file into the `playbook_output_dir`
# on this host.
ansible.capture_playbook_output(
requirements_path=os.path.join(
os.path.dirname(playbook_path),
'requirements.txt'
),
inventory_str=inventory,
vars_str=(
'local_output_dir: {output_dir}\n'
'remote_output_filename: /tmp/activity_report\n'
'server_name_prefix: {server_name_prefix}\n'
'start_date: {start_date}\n'
'end_date: {end_date}'
).format(
output_dir=playbook_output_dir,
server_name_prefix=name_prefix,
start_date=start_date,
end_date=end_date
),
playbook_path=playbook_path,
username=settings.INSTANCE_LOGS_SERVER_SSH_USERNAME,
logger_=log_line,
)
def _run_playbook(self, working_dir, playbook):
"""
Run a playbook against the AppServer's VM
"""
return ansible.capture_playbook_output(
requirements_path=os.path.join(working_dir, playbook.requirements_path),
inventory_str=self.inventory_str,
vars_str=playbook.variables,
playbook_path=os.path.join(working_dir, playbook.playbook_path),
username=settings.OPENSTACK_SANDBOX_SSH_USERNAME,
logger_=self.logger,
collect_logs=True,
)
def _run_playbook(self, working_dir, playbook):
"""
Run a playbook against the AppServer's VM
"""
return ansible.capture_playbook_output(
requirements_path=os.path.join(working_dir, playbook.requirements_path),
inventory_str=self.inventory_str,
vars_str=playbook.variables,
playbook_path=os.path.join(working_dir, playbook.playbook_path),
username=settings.OPENSTACK_SANDBOX_SSH_USERNAME,
logger_=self.logger,
collect_logs=True,
)
def get_instance_usage_data(self, playbook_output_dir, name_prefix, public_ip):
""" Execute the collect_activity playbook to gather statistics """
inventory = '[apps]\n{server}'.format(server=public_ip)
playbook_path = os.path.join(
settings.SITE_ROOT,
'playbooks/collect_activity/collect_activity.yml'
)
def log_line(line):
"""Helper to pass to capture_playbook_output()."""
self.stderr.write(self.style.SUCCESS(line))
log_line.info = log_line
log_line.error = log_line
playbook_extra_script_arguments = '--skip-hit-statistics' if settings.INSTANCE_LOGS_SERVER_HOST else ''
# Launch the collect_activity playbook, which places a file into the `playbook_output_dir`
# on this host.
ansible.capture_playbook_output(
requirements_path=os.path.join(os.path.dirname(playbook_path), 'requirements.txt'),
inventory_str=inventory,
vars_str=(
'local_output_dir: {output_dir}\n'
'local_output_filename: user_statistics\n'
'remote_output_filename: /tmp/activity_report\n'
'config_section: {config_section}\n'
'extra_script_arguments: {extra_script_arguments}'
).format(
output_dir=playbook_output_dir,
config_section=name_prefix,
extra_script_arguments=playbook_extra_script_arguments
),
playbook_path=playbook_path,
username=settings.INSTANCE_LOGS_SERVER_SSH_USERNAME,
logger_=log_line,
)
def run_playbook(self, ansible_vars):
"""
Run the playbook to perform the server reconfiguration.
This is factored out into a separate method so it can be mocked out in the tests.
"""
playbook_path = pathlib.Path(settings.SITE_ROOT) / "playbooks/load_balancer_conf/load_balancer_conf.yml"
with cache.lock("load_balancer_reconfigure:{}".format(self.domain), timeout=900):
returncode = ansible.capture_playbook_output(
requirements_path=str(playbook_path.parent / "requirements.txt"),
inventory_str=self.domain,
vars_str=ansible_vars,
playbook_path=str(playbook_path),
username=self.ssh_username,
logger_=self.logger,
)
if returncode != 0:
self.logger.error("Playbook to reconfigure load-balancing server %s failed.", self)
raise ReconfigurationFailed
def activity_csv(self, out):
"""Generate the activity CSV."""
active_appservers = self.get_active_appservers()
if not active_appservers:
self.stderr.write(
self.style.SUCCESS('There are no active app servers! Nothing to do.')
)
sys.exit(0)
self.stderr.write(self.style.SUCCESS('Running playbook...'))
with ansible.create_temp_dir() as playbook_output_dir:
inventory = '[apps]\n{servers}'.format(servers='\n'.join(active_appservers.keys()))
playbook_path = os.path.join(settings.SITE_ROOT, 'playbooks/collect_activity/collect_activity.yml')
def log_line(line):
"""Helper to pass to capture_playbook_output()."""
self.stderr.write(self.style.SUCCESS(line))
log_line.info = log_line
log_line.error = log_line
# Launch the collect_activity playbook, which places a set of files into the `playbook_output_dir`
# on this host.
ansible.capture_playbook_output(
requirements_path=os.path.join(os.path.dirname(playbook_path), 'requirements.txt'),
inventory_str=inventory,
vars_str=(
'local_output_dir: {output_dir}\n'
'remote_output_filename: /tmp/activity_report'
).format(output_dir=playbook_output_dir),
playbook_path=playbook_path,
username=settings.OPENSTACK_SANDBOX_SSH_USERNAME,
logger_=log_line,
)
csv_writer = csv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
csv_writer.writerow([
'Appserver IP', 'Internal LMS Domain', 'Name', 'Contact Email', 'Unique Hits', 'Total Users',
'Total Courses', 'Age (Days)'
])
filenames = [os.path.join(playbook_output_dir, f) for f in os.listdir(playbook_output_dir)]
data = ConfigParser()
data.read(filenames)
for public_ip, instance in sorted(active_appservers.items(), key=lambda tup: tup[1].id):
try:
section = data[public_ip]
except KeyError:
# Fill in stats for any instaces that failed with "N/A"
section = {'hits': 'N/A', 'users': 'N/A', 'courses': 'N/A'}
instance_age = datetime.now(instance.created.tzinfo) - instance.created
try:
email = instance.betatestapplication_set.get().user.email
except BetaTestApplication.DoesNotExist:
email = 'N/A'
csv_writer.writerow([
public_ip, instance.internal_lms_domain, instance.ref.name, email,
section['hits'], section['users'], section['courses'],
instance_age.days
])
self.stderr.write(self.style.SUCCESS('Done generating CSV output.'))
def activity_csv(self, out): # pylint: disable=missing-docstring
# Produce a mapping of public IPs (of active app servers) to parent instances.
active_appservers = {
instance.active_appserver.server.public_ip: instance for instance in OpenEdXInstance.objects.all()
if not (instance.active_appserver is None or instance.active_appserver.server.public_ip is None)
}
if not active_appservers:
self.stderr.write(
self.style.SUCCESS('There are no active app servers! Nothing to do.') # pylint: disable=no-member
)
sys.exit(0)
self.stderr.write(self.style.SUCCESS('Running playbook...')) # pylint: disable=no-member
with ansible.create_temp_dir() as playbook_output_dir:
inventory = '[apps]\n{servers}'.format(servers='\n'.join(active_appservers.keys()))
playbook_path = os.path.join(settings.SITE_ROOT, 'playbooks/collect_activity/collect_activity.yml')
def log_line(line):
"""Helper to pass to capture_playbook_output()."""
self.stderr.write(self.style.SUCCESS(line)) # pylint: disable=no-member
log_line.info = log_line
log_line.error = log_line
# Launch the collect_activity playbook, which places a set of files into the `playbook_output_dir`
# on this host.
ansible.capture_playbook_output(
requirements_path=os.path.join(os.path.dirname(playbook_path), 'requirements.txt'),
inventory_str=inventory,
vars_str=(
'local_output_dir: {output_dir}\n'
'remote_output_filename: /tmp/activity_report'
).format(output_dir=playbook_output_dir),
playbook_path=playbook_path,
username=settings.OPENSTACK_SANDBOX_SSH_USERNAME,
logger_=log_line,
)
csv_writer = csv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
csv_writer.writerow([
'Appserver IP', 'Internal LMS Domain', 'Name', 'Contact Email', 'Unique Hits', 'Total Users',
'Total Courses', 'Age (Days)'
])
filenames = [os.path.join(playbook_output_dir, f) for f in os.listdir(playbook_output_dir)]
data = ConfigParser()
data.read(filenames)
for public_ip in active_appservers.keys():
try:
section = data[public_ip]
except KeyError:
# Fill in stats for any instaces that failed with "N/A"
section = {'hits': 'N/A', 'users': 'N/A', 'courses': 'N/A'}
instance = active_appservers[public_ip]
instance_age = datetime.now(instance.created.tzinfo) - instance.created
try:
email = instance.betatestapplication_set.get().user.email
except BetaTestApplication.DoesNotExist:
email = 'N/A'
csv_writer.writerow([
public_ip, instance.internal_lms_domain, instance.ref.name, email,
section['hits'], section['users'], section['courses'],
instance_age.days
])
self.stderr.write(self.style.SUCCESS('Done generating CSV output.')) # pylint: disable=no-member
