def lint_py_files(self):
"""Prints a list of lint errors in the given list of Python files.
Returns:
TaskResult. A TaskResult object representing the result of the lint
check.
"""
pylintrc_path = os.path.join(os.getcwd(), '.pylintrc')
config_pylint = '--rcfile=%s' % pylintrc_path
config_pycodestyle = os.path.join(os.getcwd(), 'tox.ini')
files_to_lint = self.all_filepaths
errors_found = False
error_messages = []
full_error_messages = []
name = 'Pylint'
_batch_size = 50
current_batch_start_index = 0
stdout = python_utils.string_io()
while current_batch_start_index < len(files_to_lint):
# Note that this index is an exclusive upper bound -- i.e.,
# the current batch of files ranges from 'start_index' to
# 'end_index - 1'.
current_batch_end_index = min(
current_batch_start_index + _batch_size, len(files_to_lint))
current_files_to_lint = files_to_lint[
current_batch_start_index:current_batch_end_index]
pylint_report = python_utils.string_io()
pylinter = lint.Run(current_files_to_lint + [config_pylint],
reporter=text.TextReporter(pylint_report),
exit=False).linter
if pylinter.msg_status != 0:
lint_message = pylint_report.getvalue()
full_error_messages.append(lint_message)
pylint_error_messages = (
self.get_trimmed_error_output(lint_message))
error_messages.append(pylint_error_messages)
errors_found = True
with linter_utils.redirect_stdout(stdout):
# These lines invoke Pycodestyle and print its output
# to the target stdout.
style_guide = pycodestyle.StyleGuide(
config_file=config_pycodestyle)
pycodestyle_report = style_guide.check_files(
paths=current_files_to_lint)
if pycodestyle_report.get_count() != 0:
error_message = stdout.getvalue()
full_error_messages.append(error_message)
error_messages.append(error_message)
errors_found = True
current_batch_start_index = current_batch_end_index
return concurrent_task_utils.TaskResult(name, errors_found,
error_messages,
full_error_messages)
def test_string_io(self):
stdout = python_utils.string_io()
self.assertIsInstance(stdout, StringIO.StringIO)
os.path.join(
_PARENT_DIR, 'oppia_tools', 'webtest-%s' % common.WEBTEST_VERSION),
os.path.join(
_PARENT_DIR, 'oppia_tools', 'PyGithub-%s' % common.PYGITHUB_VERSION),
os.path.join(
_PARENT_DIR, 'oppia_tools', 'Pillow-%s' % common.PILLOW_VERSION),
os.path.join(
_PARENT_DIR, 'oppia_tools', 'psutil-%s' % common.PSUTIL_VERSION),
os.path.join(
_PARENT_DIR, 'oppia_tools', 'pip-tools-%s' % common.PIP_TOOLS_VERSION),
common.THIRD_PARTY_PYTHON_LIBS_DIR
]
for path in _PATHS_TO_INSERT:
sys.path.insert(0, path)
_TARGET_STDOUT = python_utils.string_io()
_STDOUT_LIST = multiprocessing.Manager().list()
_FILES = multiprocessing.Manager().dict()
class FileCache(python_utils.OBJECT):
"""Provides thread-safe access to cached file content."""
def __init__(self):
self._CACHE_DATA_DICT = {}
def read(self, filepath, mode='r'):
"""Returns the data read from the file in unicode form.
Args:
filepath: str. The file path from which data is to be read.
def _lint_py_files_for_python3_compatibility(self):
"""Prints a list of Python 3 compatibility errors in the given list of
Python files.
Returns:
summary_messages: list(str). Summary of lint check.
"""
files_to_lint = self.all_filepaths
start_time = time.time()
any_errors = False
stdout = python_utils.string_io()
summary_messages = []
files_to_lint_for_python3_compatibility = [
file_name for file_name in files_to_lint
if not re.match(r'^.*python_utils.*\.py$', file_name)
]
num_py_files = len(files_to_lint_for_python3_compatibility)
if not files_to_lint_for_python3_compatibility:
python_utils.PRINT('')
python_utils.PRINT(
'There are no Python files to lint for Python 3 compatibility.'
)
return []
python_utils.PRINT(
'Linting %s Python files for Python 3 compatibility.' %
(num_py_files))
_batch_size = 50
current_batch_start_index = 0
while current_batch_start_index < len(
files_to_lint_for_python3_compatibility):
# Note that this index is an exclusive upper bound -- i.e.,
# the current batch of files ranges from 'start_index' to
# 'end_index - 1'.
current_batch_end_index = min(
current_batch_start_index + _batch_size,
len(files_to_lint_for_python3_compatibility))
current_files_to_lint = files_to_lint_for_python3_compatibility[
current_batch_start_index:current_batch_end_index]
if self.verbose_mode_enabled:
python_utils.PRINT(
'Linting Python files for Python 3 compatibility %s to %s..'
% (current_batch_start_index + 1, current_batch_end_index))
with linter_utils.redirect_stdout(stdout):
# This line invokes Pylint and prints its output
# to the target stdout.
python_utils.PRINT('Messages for Python 3 support:')
pylinter_for_python3 = lint.Run(current_files_to_lint +
['--py3k'],
exit=False).linter
if pylinter_for_python3.msg_status != 0:
summary_message = stdout.getvalue()
python_utils.PRINT(summary_message)
summary_messages.append(summary_message)
any_errors = True
current_batch_start_index = current_batch_end_index
if any_errors:
summary_message = (
'%s Python linting for Python 3 compatibility failed' %
linter_utils.FAILED_MESSAGE_PREFIX)
else:
summary_message = (
'%s %s Python files linted for Python 3 compatibility '
'(%.1f secs)' %
(linter_utils.SUCCESS_MESSAGE_PREFIX, num_py_files,
(time.time() - start_time)))
python_utils.PRINT(summary_message)
summary_messages.append(summary_message)
python_utils.PRINT(
'Python linting for Python 3 compatibility finished.')
return summary_messages
def _lint_py_files(self, config_pylint, config_pycodestyle):
"""Prints a list of lint errors in the given list of Python files.
Args:
config_pylint: str. Path to the .pylintrc file.
config_pycodestyle: str. Path to the tox.ini file.
Return:
summary_messages: list(str). Summary messages of lint check.
"""
files_to_lint = self.all_filepaths
start_time = time.time()
are_there_errors = False
summary_messages = []
num_py_files = len(files_to_lint)
python_utils.PRINT('Linting %s Python files' % num_py_files)
_batch_size = 50
current_batch_start_index = 0
stdout = python_utils.string_io()
while current_batch_start_index < len(files_to_lint):
# Note that this index is an exclusive upper bound -- i.e.,
# the current batch of files ranges from 'start_index' to
# 'end_index - 1'.
current_batch_end_index = min(
current_batch_start_index + _batch_size, len(files_to_lint))
current_files_to_lint = files_to_lint[
current_batch_start_index:current_batch_end_index]
if self.verbose_mode_enabled:
python_utils.PRINT(
'Linting Python files %s to %s...' %
(current_batch_start_index + 1, current_batch_end_index))
with linter_utils.redirect_stdout(stdout):
# This line invokes Pylint and prints its output
# to the target stdout.
pylinter = lint.Run(current_files_to_lint + [config_pylint],
exit=False).linter
# These lines invoke Pycodestyle and print its output
# to the target stdout.
style_guide = pycodestyle.StyleGuide(
config_file=config_pycodestyle)
pycodestyle_report = style_guide.check_files(
paths=current_files_to_lint)
if pylinter.msg_status != 0 or pycodestyle_report.get_count() != 0:
summary_message = stdout.getvalue()
python_utils.PRINT(summary_message)
summary_messages.append(summary_message)
are_there_errors = True
current_batch_start_index = current_batch_end_index
if are_there_errors:
summary_message = ('%s Python linting failed' %
(linter_utils.FAILED_MESSAGE_PREFIX))
else:
summary_message = ('%s %s Python files linted (%.1f secs)' %
(linter_utils.SUCCESS_MESSAGE_PREFIX,
num_py_files, time.time() - start_time))
python_utils.PRINT(summary_message)
summary_messages.append(summary_message)
python_utils.PRINT('Python linting finished.')
return summary_messages
def test_export_account_handler(self):
# Update user settings to constants.
user_id = self.get_user_id_from_email(self.EDITOR_EMAIL)
user_settings = user_services.get_user_settings(user_id)
user_settings.last_agreed_to_terms = self.GENERIC_DATE
user_settings.last_logged_in = self.GENERIC_DATE
user_settings.validate()
user_models.UserSettingsModel(
id=user_settings.user_id,
gae_id=user_settings.gae_id,
email=user_settings.email,
role=user_settings.role,
username=user_settings.username,
normalized_username=user_settings.normalized_username,
last_agreed_to_terms=user_settings.last_agreed_to_terms,
last_started_state_editor_tutorial=(
user_settings.last_started_state_editor_tutorial),
last_started_state_translation_tutorial=(
user_settings.last_started_state_translation_tutorial),
last_logged_in=user_settings.last_logged_in,
last_edited_an_exploration=user_settings.
last_edited_an_exploration,
last_created_an_exploration=(
user_settings.last_created_an_exploration),
profile_picture_data_url=user_settings.profile_picture_data_url,
default_dashboard=user_settings.default_dashboard,
creator_dashboard_display_pref=(
user_settings.creator_dashboard_display_pref),
user_bio=user_settings.user_bio,
subject_interests=user_settings.subject_interests,
first_contribution_msec=user_settings.first_contribution_msec,
preferred_language_codes=user_settings.preferred_language_codes,
preferred_site_language_code=(
user_settings.preferred_site_language_code),
preferred_audio_language_code=(
user_settings.preferred_audio_language_code),
deleted=user_settings.deleted).put()
constants_swap = self.swap(constants, 'ENABLE_ACCOUNT_EXPORT', True)
time_swap = self.swap(user_services, 'record_user_logged_in',
lambda *args: None)
with constants_swap, time_swap:
data = self.get_custom_response('/export-account-handler',
'text/plain')
# Check downloaded zip file.
filename = 'oppia_takeout_data.zip'
self.assertEqual(data.headers['Content-Disposition'],
'attachment; filename=%s' % filename)
zf_saved = zipfile.ZipFile(
python_utils.string_io(buffer_value=data.body))
self.assertEqual(zf_saved.namelist(), [
'oppia_takeout_data.json',
'images/user_settings_profile_picture.png'
])
# Load golden zip file.
golden_zip_filepath = os.path.join(feconf.TESTS_DATA_DIR,
'oppia_takeout_data.zip')
with python_utils.open_file(golden_zip_filepath,
'rb',
encoding=None) as f:
golden_zipfile = f.read()
zf_gold = zipfile.ZipFile(
python_utils.string_io(buffer_value=golden_zipfile))
self.assertEqual(
zf_saved.open('oppia_takeout_data.json').read(),
zf_gold.open('oppia_takeout_data.json').read())
self.assertEqual(
zf_saved.open(
'images/user_settings_profile_picture.png').read(),
zf_gold.open(
'images/user_settings_profile_picture.png').read())
def post(self, exploration_id):
"""Saves an audio file uploaded by a content creator."""
raw_audio_file = self.request.get('raw_audio_file')
filename = self.payload.get('filename')
allowed_formats = list(feconf.ACCEPTED_AUDIO_EXTENSIONS.keys())
if not raw_audio_file:
raise self.InvalidInputException('No audio supplied')
dot_index = filename.rfind('.')
extension = filename[dot_index + 1:].lower()
if dot_index == -1 or dot_index == 0:
raise self.InvalidInputException(
'No filename extension: it should have '
'one of the following extensions: %s' % allowed_formats)
if extension not in feconf.ACCEPTED_AUDIO_EXTENSIONS:
raise self.InvalidInputException(
'Invalid filename extension: it should have '
'one of the following extensions: %s' % allowed_formats)
tempbuffer = python_utils.string_io()
tempbuffer.write(raw_audio_file)
tempbuffer.seek(0)
try:
# For every accepted extension, use the mutagen-specific
# constructor for that type. This will catch mismatched audio
# types e.g. uploading a flac file with an MP3 extension.
if extension == 'mp3':
audio = mp3.MP3(tempbuffer)
else:
audio = mutagen.File(tempbuffer)
except mutagen.MutagenError:
# The calls to mp3.MP3() versus mutagen.File() seem to behave
# differently upon not being able to interpret the audio.
# mp3.MP3() raises a MutagenError whereas mutagen.File()
# seems to return None. It's not clear if this is always
# the case. Occasionally, mutagen.File() also seems to
# raise a MutagenError.
raise self.InvalidInputException('Audio not recognized '
'as a %s file' % extension)
tempbuffer.close()
if audio is None:
raise self.InvalidInputException('Audio not recognized '
'as a %s file' % extension)
if audio.info.length > feconf.MAX_AUDIO_FILE_LENGTH_SEC:
raise self.InvalidInputException(
'Audio files must be under %s seconds in length. The uploaded '
'file is %.2f seconds long.' %
(feconf.MAX_AUDIO_FILE_LENGTH_SEC, audio.info.length))
if len(
set(audio.mime).intersection(
set(feconf.ACCEPTED_AUDIO_EXTENSIONS[extension]))) == 0:
raise self.InvalidInputException(
'Although the filename extension indicates the file '
'is a %s file, it was not recognized as one. '
'Found mime types: %s' % (extension, audio.mime))
mimetype = audio.mime[0]
# For a strange, unknown reason, the audio variable must be
# deleted before opening cloud storage. If not, cloud storage
# throws a very mysterious error that entails a mutagen
# object being recursively passed around in app engine.
del audio
# Audio files are stored to the datastore in the dev env, and to GCS
# in production.
file_system_class = fs_services.get_entity_file_system_class()
fs = fs_domain.AbstractFileSystem(
file_system_class(feconf.ENTITY_TYPE_EXPLORATION, exploration_id))
fs.commit(self.user_id,
'%s/%s' % (self._FILENAME_PREFIX, filename),
raw_audio_file,
mimetype=mimetype)
self.render_json({'filename': filename})
