def update_instance_from_pr(self, pr):
"""
Update/create the associated sandbox instance with settings from the given pull request.
This will not spawn a new AppServer.
This method will automatically save this WatchedPullRequest's 'instance' field.
"""
# The following fields should never change:
assert self.github_pr_url == pr.github_pr_url
assert self.fork_name == pr.fork_name
assert self.branch_name == pr.branch_name
# Create an instance if necessary:
instance = self.instance or OpenEdXInstance()
instance.internal_lms_domain = generate_internal_lms_domain('pr{number}.sandbox'.format(number=pr.number))
instance.edx_platform_repository_url = self.repository_url
instance.edx_platform_commit = self.get_branch_tip()
instance.name = (
'PR#{pr.number}: {pr.truncated_title} ({pr.username}) - {i.reference_name} ({commit_short_id})'
.format(pr=pr, i=self, commit_short_id=instance.edx_platform_commit[:7])
)
instance.configuration_extra_settings = yaml_merge(
self.watched_fork.configuration_extra_settings,
pr.extra_settings
)
if not instance.ref.creator or not instance.ref.owner:
user = UserProfile.objects.get(github_username=pr.username)
instance.ref.creator = user
instance.ref.owner = user.organization
# Configuration repo and version and edx release follow this precedence:
# 1) PR settings. 2) WatchedFork settings. 3) instance model defaults
instance.configuration_source_repo_url = pr.get_extra_setting(
'edx_ansible_source_repo',
default=(
self.watched_fork.configuration_source_repo_url or
instance.configuration_source_repo_url
)
)
instance.configuration_version = pr.get_extra_setting(
'configuration_version', default=(
self.watched_fork.configuration_version or
instance.configuration_version
)
)
instance.openedx_release = pr.get_extra_setting(
'openedx_release', default=(
self.watched_fork.openedx_release or
instance.openedx_release
)
)
# Save atomically. (because if the instance gets created but self.instance failed to
# update, then any subsequent call to update_instance_from_pr() would try to create
# another instance, which would fail due to unique domain name constraints.)
with transaction.atomic():
instance.save()
if not self.instance:
self.instance = instance
self.save(update_fields=["instance"])
def vars_str(self):
"""
The ansible vars (private configuration) as a string
"""
template = loader.get_template('instance/ansible/vars.yml')
vars_str = template.render({'instance': self})
for attr_name in self.ANSIBLE_SETTINGS:
additional_vars = getattr(self, attr_name)
vars_str = ansible.yaml_merge(vars_str, additional_vars)
self.log('debug', 'Vars.yml for instance {}:\n{}'.format(self, vars_str))
return vars_str
def production_instance_factory(**kwargs):
"""
Factory function for creating production instances.
Returns a newly created OpenEdXInstance.
Callers can use keyword arguments to pass in non-default values
for any field that is defined on the OpenEdXInstance model.
The only mandatory argument is `sub_domain`.
When called without any additional arguments, the instance that is returned
will have its fields set to default values that are appropriate
for *production* instances.
To create an instance with default settings that are suitable for sandboxes,
use `instance_factory`.
"""
# NOTE: The long-term goal is to eliminate differences between sandboxes
# and production instances, and for this function to disappear.
# Please do not add behavior that is specific to production instances here.
# Ensure caller provided required arguments
assert "sub_domain" in kwargs
# Check environment and report potential problems
environment_ready = _check_environment()
if not environment_ready:
logger.warning("Environment not ready. Please fix the problems above, then try again. Aborting.")
return
# Gather settings
production_settings = loader.get_template('instance/ansible/prod-vars.yml').render({})
configuration_extra_settings = kwargs.pop("configuration_extra_settings", "")
extra_settings = ansible.yaml_merge(production_settings, configuration_extra_settings)
instance_kwargs = dict(
use_ephemeral_databases=False,
edx_platform_repository_url=settings.STABLE_EDX_PLATFORM_REPO_URL,
edx_platform_commit=settings.STABLE_EDX_PLATFORM_COMMIT,
configuration_source_repo_url=settings.STABLE_CONFIGURATION_REPO_URL,
configuration_version=settings.STABLE_CONFIGURATION_VERSION,
openedx_release=settings.OPENEDX_RELEASE_STABLE_REF,
configuration_extra_settings=extra_settings,
)
instance_kwargs.update(kwargs)
# Create instance
production_instance = OpenEdXInstance.objects.create(**instance_kwargs)
return production_instance
def production_instance_factory(**kwargs):
"""
Factory function for creating production instances.
Returns a newly created OpenEdXInstance.
Callers can use keyword arguments to pass in non-default values
for any field that is defined on the OpenEdXInstance model.
The only mandatory argument is `sub_domain`.
When called without any additional arguments, the instance that is returned
will have its fields set to default values that are appropriate
for *production* instances.
To create an instance with default settings that are suitable for sandboxes,
use `instance_factory`.
"""
# NOTE: The long-term goal is to eliminate differences between sandboxes
# and production instances, and for this function to disappear.
# Please do not add behavior that is specific to production instances here.
# Ensure caller provided required arguments
assert "sub_domain" in kwargs
# Check environment and report potential problems
environment_ready = _check_environment()
if not environment_ready:
logger.warning("Environment not ready. Please fix the problems above, then try again. Aborting.")
return
# Gather settings
production_settings = loader.get_template('instance/ansible/prod-vars.yml').render({})
configuration_extra_settings = kwargs.pop("configuration_extra_settings", "")
extra_settings = ansible.yaml_merge(production_settings, configuration_extra_settings)
instance_kwargs = dict(
use_ephemeral_databases=False,
edx_platform_repository_url=settings.STABLE_EDX_PLATFORM_REPO_URL,
edx_platform_commit=settings.STABLE_EDX_PLATFORM_COMMIT,
configuration_source_repo_url=settings.STABLE_CONFIGURATION_REPO_URL,
configuration_version=settings.STABLE_CONFIGURATION_VERSION,
openedx_release=settings.OPENEDX_RELEASE_STABLE_REF,
configuration_extra_settings=extra_settings,
)
instance_kwargs.update(kwargs)
# Create instance
production_instance = OpenEdXInstance.objects.create(**instance_kwargs)
return production_instance
def create_configuration_settings(self):
"""
Generate the configuration settings.
This is a one-time thing, because configuration_settings, like all AppServer fields, is
immutable once this AppServer is saved.
"""
template = loader.get_template(self.CONFIGURATION_VARS_TEMPLATE)
vars_str = template.render({
'appserver': self,
'instance': self.instance,
'newrelic_license_key': settings.NEWRELIC_LICENSE_KEY,
})
for attr_name in self.CONFIGURATION_EXTRA_FIELDS:
additional_vars = getattr(self, attr_name)
vars_str = ansible.yaml_merge(vars_str, additional_vars)
self.logger.debug('Vars.yml:\n%s', vars_str)
return vars_str
def reset_ansible_settings(self, commit=True):
"""
Set the ansible_settings field from the Ansible vars template.
"""
template = loader.get_template('instance/ansible/vars.yml')
vars_str = template.render({
'instance': self,
# This proerty is needed twice in the template. To avoid evaluating it twice (and
# querying the Github API twice), we pass it as a context variable.
'github_admin_username_list': self.github_admin_username_list,
})
for attr_name in self.ANSIBLE_SETTINGS:
additional_vars = getattr(self, attr_name)
vars_str = ansible.yaml_merge(vars_str, additional_vars)
self.logger.debug('Vars.yml:\n%s', vars_str)
self.ansible_settings = vars_str
if commit:
self.save()
def test_yaml_merge(self):
"""
Merge of two yaml strings with overlapping variables
"""
yaml_result_str = ansible.yaml_merge(self.yaml_str1, self.yaml_str2)
self.assertEquals(yaml.load(yaml_result_str), {
'testa': 'firsta with unicode «ταБЬℓσ»',
'testb': 'secondb with unicode «ταБЬℓσ2»',
'testc': 'secondc',
'test_dict': {
'foo': 'secondfoo',
'bar': 'firstbar',
'other': 'secondother',
'recursive': {
'a': 1,
'b': 2,
},
}
})
def test_yaml_merge(self):
"""
Merge of two yaml strings with overlapping variables
"""
yaml_result_str = ansible.yaml_merge(self.yaml_str1, self.yaml_str2)
self.assertEqual(
yaml.load(yaml_result_str), {
'testa': 'firsta with unicode «ταБЬℓσ»',
'testb': 'secondb with unicode «ταБЬℓσ2»',
'testc': 'secondc',
'test_dict': {
'foo': 'secondfoo',
'bar': 'firstbar',
'other': 'secondother',
'recursive': {
'a': 1,
'b': 2,
},
}
})
def reset_ansible_settings(self, commit=True):
"""
Set the ansible_settings field from the Ansible vars template.
"""
template = loader.get_template('instance/ansible/vars.yml')
vars_str = template.render({
'instance':
self,
# This proerty is needed twice in the template. To avoid evaluating it twice (and
# querying the Github API twice), we pass it as a context variable.
'github_admin_username_list':
self.github_admin_username_list,
})
for attr_name in self.ANSIBLE_SETTINGS:
additional_vars = getattr(self, attr_name)
vars_str = ansible.yaml_merge(vars_str, additional_vars)
self.logger.debug('Vars.yml:\n%s', vars_str)
self.ansible_settings = vars_str
if commit:
self.save()
def create_configuration_settings(self):
"""
Generate the configuration settings.
This is a one-time thing, because configuration_settings, like all AppServer fields, is
immutable once this AppServer is saved.
"""
template = loader.get_template(self.CONFIGURATION_VARS_TEMPLATE)
vars_str = template.render({
'appserver':
self,
'instance':
self.instance,
'newrelic_license_key':
settings.NEWRELIC_LICENSE_KEY,
})
for attr_name in self.CONFIGURATION_EXTRA_FIELDS:
additional_vars = getattr(self, attr_name)
vars_str = ansible.yaml_merge(vars_str, additional_vars)
self.logger.debug('Vars.yml:\n%s', vars_str)
return vars_str
def test_yaml_merge_with_none(self):
"""
Merge of a yaml string with None
"""
self.assertEqual(ansible.yaml_merge(self.yaml_str1, None),
self.yaml_str1)
def test_yaml_merge_with_none(self):
"""
Merge of a yaml string with None
"""
self.assertEqual(ansible.yaml_merge(self.yaml_str1, None), self.yaml_str1)
