def main(args=None):
"""Updates the files corresponding to LOCAL_FECONF_PATH and
LOCAL_CONSTANTS_PATH after doing the prerequisite checks.
"""
options = _PARSER.parse_args(args=args)
# Do prerequisite checks.
feconf_config_path = os.path.join(options.deploy_data_path,
'feconf_updates.config')
constants_config_path = os.path.join(options.deploy_data_path,
'constants_updates.config')
release_feconf_path = os.path.join(options.release_dir_path,
common.FECONF_PATH)
release_constants_path = os.path.join(options.release_dir_path,
common.CONSTANTS_FILE_PATH)
if options.prompt_for_mailgun_and_terms_update:
try:
python_utils.url_open(TERMS_PAGE_FOLDER_URL)
except Exception:
raise Exception('Terms mainpage does not exist on Github.')
add_mailgun_api_key(release_feconf_path)
add_mailchimp_api_key(release_feconf_path)
check_updates_to_terms_of_service(release_feconf_path,
options.personal_access_token)
apply_changes_based_on_config(release_feconf_path, feconf_config_path,
FECONF_REGEX)
apply_changes_based_on_config(release_constants_path,
constants_config_path, CONSTANTS_REGEX)
verify_feconf(release_feconf_path,
options.prompt_for_mailgun_and_terms_update)
def download_and_unzip_files(
source_url, target_parent_dir, zip_root_name, target_root_name):
"""Downloads a zip file, unzips it, and saves the result in a given dir.
The download occurs only if the target directory that the zip file unzips
to does not exist.
NB: This function assumes that the root level of the zip file has exactly
one folder.
Args:
source_url: str. The URL from which to download the zip file.
target_parent_dir: str. The directory to save the contents of the zip
file to.
zip_root_name: str. The name of the top-level folder in the zip
directory.
target_root_name: str. The name that the top-level folder should be
renamed to in the local directory.
"""
if not os.path.exists(os.path.join(target_parent_dir, target_root_name)):
python_utils.PRINT('Downloading and unzipping file %s to %s ...' % (
zip_root_name, target_parent_dir))
common.ensure_directory_exists(target_parent_dir)
urlrequest.urlretrieve(source_url, filename=TMP_UNZIP_PATH)
try:
with zipfile.ZipFile(TMP_UNZIP_PATH, 'r') as zfile:
zfile.extractall(path=target_parent_dir)
os.remove(TMP_UNZIP_PATH)
except Exception:
if os.path.exists(TMP_UNZIP_PATH):
os.remove(TMP_UNZIP_PATH)
# Some downloads (like jqueryui-themes) may require a user-agent.
req = python_utils.url_request(source_url, None, {})
req.add_header('User-agent', 'python')
# This is needed to get a seekable filestream that can be used
# by zipfile.ZipFile.
file_stream = python_utils.string_io(
buffer_value=python_utils.url_open(req).read())
with zipfile.ZipFile(file_stream, 'r') as zfile:
zfile.extractall(path=target_parent_dir)
# Rename the target directory.
os.rename(
os.path.join(target_parent_dir, zip_root_name),
os.path.join(target_parent_dir, target_root_name))
python_utils.PRINT('Download of %s succeeded.' % zip_root_name)
def verify_target_version_compatible_with_latest_release(target_version):
"""Checks that the target version is consistent with the latest released
version on GitHub.
Args:
target_version: str. The release version.
Raises:
Exception. Failed to fetch latest release info from GitHub.
Exception. Could not parse version number of latest GitHub release.
AssertionError. The previous and the current major version are not the
same.
AssertionError. The current patch version is not equal to previous patch
version plus one.
AssertionError. The current patch version is greater or equal to 10.
AssertionError. The current minor version is not equal to previous
minor version plus one.
AssertionError. The current patch version is different than 0.
"""
response = python_utils.url_open(
'https://api.github.com/repos/oppia/oppia/releases/latest')
if response.getcode() != 200:
raise Exception(
'ERROR: Failed to fetch latest release info from GitHub.')
data = json.load(response)
latest_release_tag_name = data['tag_name']
match_result = re.match(r'v(\d)\.(\d)\.(\d)', latest_release_tag_name)
if match_result is None:
raise Exception(
'ERROR: Could not parse version number of latest GitHub release.')
prev_major, prev_minor, prev_patch = match_result.group(1, 2, 3)
match_result = re.match(r'(\d)\.(\d)\.(\d)', target_version)
curr_major, curr_minor, curr_patch = match_result.group(1, 2, 3)
# This will need to be overridden if the major version changes.
assert prev_major == curr_major, 'Unexpected major version change.'
if prev_minor == curr_minor:
assert int(curr_patch) == int(prev_patch) + 1, (
'The current patch version is not equal to previous patch '
'version plus one.')
else:
assert int(curr_minor) == int(prev_minor) + 1, (
'The current minor version is not equal to previous '
'minor version plus one.')
assert int(curr_patch) == 0, (
'The current patch version is different than 0.')
def lookup_pr(owner, repo, pull_number):
"""Lookup a PR using the GitHub API.
Args:
owner: str. Owner of the repository the PR is in.
repo: str. Repository the PR is in.
pull_number: str. PR number.
Returns:
dict. JSON object returned by the GitHub API v3. This is an
empty dictionary if the response code from the GitHub API is not
200.
"""
request = python_utils.url_request(
GITHUB_API_PR_ENDPOINT % (owner, repo, pull_number), None,
{'Accept': 'application/vnd.github.v3+json'})
response = python_utils.url_open(request)
if response.getcode() != 200:
return {}
pr = json.load(response)
response.close()
return pr
def send_email_to_recipients(
sender_email: str,
recipient_emails: List[str],
subject: str,
plaintext_body: str,
html_body: str,
bcc: Optional[List[str]] = None,
reply_to: Optional[str] = None,
recipient_variables: Optional[Dict[str, Dict[str, Union[str,
float]]]] = None
) -> bool:
"""Send POST HTTP request to mailgun api. This method is adopted from
the requests library's post method.
Args:
sender_email: str. The email address of the sender. This should be in
the form 'SENDER_NAME <SENDER_EMAIL_ADDRESS>' or
'SENDER_EMAIL_ADDRESS'. Must be utf-8.
recipient_emails: list(str). The email addresses of the recipients.
Must be utf-8.
subject: str. The subject line of the email, Must be utf-8.
plaintext_body: str. The plaintext body of the email. Must be utf-8.
html_body: str. The HTML body of the email. Must fit in a datastore
entity. Must be utf-8.
bcc: list(str)|None. Optional argument. List of bcc emails.
reply_to: str|None. Optional argument. Reply address formatted like
“reply+<reply_id>@<incoming_email_domain_name>
reply_id is the unique id of the sender.
recipient_variables: dict|None. Optional argument. If batch sending
requires differentiating each email based on the recipient, we
assign a unique id to each recipient, including info relevant to
that recipient so that we can reference it when composing the
email like so:
recipient_variables =
{"*****@*****.**": {"first":"Bob", "id":1},
"*****@*****.**": {"first":"Alice", "id":2}}
subject = 'Hey, %recipient.first%’
More info about this format at:
https://documentation.mailgun.com/en/
latest/user_manual.html#batch-sending.
Raises:
Exception. The mailgun api key is not stored in
feconf.MAILGUN_API_KEY.
Exception. The mailgun domain name is not stored in
feconf.MAILGUN_DOMAIN_NAME.
Returns:
bool. Whether the emails are sent successfully.
"""
if not feconf.MAILGUN_API_KEY:
raise Exception('Mailgun API key is not available.')
if not feconf.MAILGUN_DOMAIN_NAME:
raise Exception('Mailgun domain name is not set.')
# To send bulk emails we pass list of recipients in 'to' paarameter of
# post data. Maximum limit of recipients per request is 1000.
# For more detail check following link:
# https://documentation.mailgun.com/user_manual.html#batch-sending
recipient_email_lists = [
recipient_emails[i:i + 1000]
for i in python_utils.RANGE(0, len(recipient_emails), 1000)
]
for email_list in recipient_email_lists:
data = {
'from': sender_email,
'subject': subject.encode('utf-8'),
'text': plaintext_body.encode('utf-8'),
'html': html_body.encode('utf-8'),
'to': email_list[0] if len(email_list) == 1 else email_list
}
if bcc:
data['bcc'] = bcc[0] if len(bcc) == 1 else bcc
if reply_to:
data['h:Reply-To'] = reply_to
# 'recipient-variable' in post data forces mailgun to send individual
# email to each recipient (This is intended to be a workaround for
# sending individual emails).
data['recipient_variables'] = recipient_variables or {}
# The b64encode accepts and returns bytes, so we first need to encode
# the MAILGUN_API_KEY to bytes, then decode the returned bytes back
# to string.
base64_mailgun_api_key = base64.b64encode(
b'api:%b' %
feconf.MAILGUN_API_KEY.encode('utf-8')).strip().decode('utf-8')
auth_str = 'Basic %s' % base64_mailgun_api_key
header = {'Authorization': auth_str}
server = (('https://api.mailgun.net/v3/%s/messages') %
feconf.MAILGUN_DOMAIN_NAME)
encoded_url = python_utils.url_encode(data)
req = python_utils.url_request(server, encoded_url, header)
resp = python_utils.url_open(req)
# The function url_open returns a file_like object that can be queried
# for the status code of the url query. If it is not 200, the mail query
# failed so we return False (this function did not complete
# successfully).
if resp.getcode() != 200:
return False
return True
def test_url_open(self):
response = python_utils.url_open('http://www.google.com')
self.assertEqual(response.getcode(), 200)
self.assertEqual(response.url, 'http://www.google.com')
def managed_webdriver_server(chrome_version=None):
"""Returns context manager to start/stop the Webdriver server gracefully.
This context manager updates Google Chrome before starting the server.
Args:
chrome_version: str|None. The version of Google Chrome to run the tests
on. If None, then the currently-installed version of Google Chrome
is used instead.
Yields:
psutil.Process. The Webdriver process.
"""
if chrome_version is None:
# Although there are spaces between Google and Chrome in the path, we
# don't need to escape them for Popen (as opposed to on the terminal, in
# which case we would need to escape them for the command to run).
chrome_command = (
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
if common.is_mac_os() else 'google-chrome')
try:
output = subprocess.check_output([chrome_command, '--version'])
except OSError:
# For the error message on macOS, we need to add the backslashes in.
# This is because it is likely that a user will try to run the
# command on their terminal and, as mentioned above, the macOS
# chrome version command has spaces in the path which need to be
# escaped for successful terminal use.
raise Exception(
'Failed to execute "%s --version" command. This is used to '
'determine the chromedriver version to use. Please set the '
'chromedriver version manually using the '
'--chrome_driver_version flag. To determine the '
'chromedriver version to be used, please follow the '
'instructions mentioned in the following URL:\n'
'https://chromedriver.chromium.org/downloads/version-selection'
% chrome_command.replace(' ', r'\ '))
installed_version_parts = b''.join(re.findall(rb'[0-9.]', output))
installed_version = '.'.join(
installed_version_parts.decode('utf-8').split('.')[:-1])
response = python_utils.url_open(
'https://chromedriver.storage.googleapis.com/LATEST_RELEASE_%s' %
(installed_version))
chrome_version = response.read().decode('utf-8')
python_utils.PRINT('\n\nCHROME VERSION: %s' % chrome_version)
subprocess.check_call([
common.NODE_BIN_PATH,
common.WEBDRIVER_MANAGER_BIN_PATH,
'update',
'--versions.chrome',
chrome_version,
])
with contextlib.ExitStack() as exit_stack:
if common.is_windows_os():
# NOTE: webdriver-manager (version 13.0.0) uses `os.arch()` to
# determine the architecture of the operating system, however, this
# function can only be used to determine the architecture of the
# machine that compiled `node`. In the case of Windows, we are using
# the portable version, which was compiled on `ia32` machine so that
# is the value returned by this `os.arch` function. Unfortunately,
# webdriver-manager seems to assume that Windows wouldn't run on the
# ia32 architecture, so its help function used to determine download
# link returns null for this, which means that the application has
# no idea about where to download the correct version.
#
# https://github.com/angular/webdriver-manager/blob/b7539a5a3897a8a76abae7245f0de8175718b142/lib/provider/chromedriver.ts#L16
# https://github.com/angular/webdriver-manager/blob/b7539a5a3897a8a76abae7245f0de8175718b142/lib/provider/geckodriver.ts#L21
# https://github.com/angular/webdriver-manager/blob/b7539a5a3897a8a76abae7245f0de8175718b142/lib/provider/chromedriver.ts#L167
# https://github.com/nodejs/node/issues/17036
regex_pattern = re.escape('this.osArch = os.arch();')
arch = 'x64' if common.is_x64_architecture() else 'x86'
replacement_string = 'this.osArch = "%s";' % arch
exit_stack.enter_context(
common.inplace_replace_file_context(
common.CHROME_PROVIDER_FILE_PATH, regex_pattern,
replacement_string))
exit_stack.enter_context(
common.inplace_replace_file_context(
common.GECKO_PROVIDER_FILE_PATH, regex_pattern,
replacement_string))
# OK to use shell=True here because we are passing string literals and
# constants, so there is no risk of a shell-injection attack.
proc = exit_stack.enter_context(
managed_process([
common.NODE_BIN_PATH,
common.WEBDRIVER_MANAGER_BIN_PATH,
'start',
'--versions.chrome',
chrome_version,
'--quiet',
'--standalone',
],
human_readable_name='Webdriver manager',
shell=True))
common.wait_for_port_to_be_in_use(4444)
yield proc