def lint_py_files(self): """Prints a list of lint errors in the given list of Python files. Returns: TaskResult. A TaskResult object representing the result of the lint check. """ pylintrc_path = os.path.join(os.getcwd(), '.pylintrc') config_pylint = '--rcfile=%s' % pylintrc_path config_pycodestyle = os.path.join(os.getcwd(), 'tox.ini') files_to_lint = self.all_filepaths errors_found = False error_messages = [] full_error_messages = [] name = 'Pylint' _batch_size = 50 current_batch_start_index = 0 stdout = python_utils.string_io() while current_batch_start_index < len(files_to_lint): # Note that this index is an exclusive upper bound -- i.e., # the current batch of files ranges from 'start_index' to # 'end_index - 1'. current_batch_end_index = min( current_batch_start_index + _batch_size, len(files_to_lint)) current_files_to_lint = files_to_lint[ current_batch_start_index:current_batch_end_index] pylint_report = python_utils.string_io() pylinter = lint.Run(current_files_to_lint + [config_pylint], reporter=text.TextReporter(pylint_report), exit=False).linter if pylinter.msg_status != 0: lint_message = pylint_report.getvalue() full_error_messages.append(lint_message) pylint_error_messages = ( self.get_trimmed_error_output(lint_message)) error_messages.append(pylint_error_messages) errors_found = True with linter_utils.redirect_stdout(stdout): # These lines invoke Pycodestyle and print its output # to the target stdout. style_guide = pycodestyle.StyleGuide( config_file=config_pycodestyle) pycodestyle_report = style_guide.check_files( paths=current_files_to_lint) if pycodestyle_report.get_count() != 0: error_message = stdout.getvalue() full_error_messages.append(error_message) error_messages.append(error_message) errors_found = True current_batch_start_index = current_batch_end_index return concurrent_task_utils.TaskResult(name, errors_found, error_messages, full_error_messages)
def test_string_io(self): stdout = python_utils.string_io() self.assertIsInstance(stdout, StringIO.StringIO)
os.path.join( _PARENT_DIR, 'oppia_tools', 'webtest-%s' % common.WEBTEST_VERSION), os.path.join( _PARENT_DIR, 'oppia_tools', 'PyGithub-%s' % common.PYGITHUB_VERSION), os.path.join( _PARENT_DIR, 'oppia_tools', 'Pillow-%s' % common.PILLOW_VERSION), os.path.join( _PARENT_DIR, 'oppia_tools', 'psutil-%s' % common.PSUTIL_VERSION), os.path.join( _PARENT_DIR, 'oppia_tools', 'pip-tools-%s' % common.PIP_TOOLS_VERSION), common.THIRD_PARTY_PYTHON_LIBS_DIR ] for path in _PATHS_TO_INSERT: sys.path.insert(0, path) _TARGET_STDOUT = python_utils.string_io() _STDOUT_LIST = multiprocessing.Manager().list() _FILES = multiprocessing.Manager().dict() class FileCache(python_utils.OBJECT): """Provides thread-safe access to cached file content.""" def __init__(self): self._CACHE_DATA_DICT = {} def read(self, filepath, mode='r'): """Returns the data read from the file in unicode form. Args: filepath: str. The file path from which data is to be read.
def _lint_py_files_for_python3_compatibility(self): """Prints a list of Python 3 compatibility errors in the given list of Python files. Returns: summary_messages: list(str). Summary of lint check. """ files_to_lint = self.all_filepaths start_time = time.time() any_errors = False stdout = python_utils.string_io() summary_messages = [] files_to_lint_for_python3_compatibility = [ file_name for file_name in files_to_lint if not re.match(r'^.*python_utils.*\.py$', file_name) ] num_py_files = len(files_to_lint_for_python3_compatibility) if not files_to_lint_for_python3_compatibility: python_utils.PRINT('') python_utils.PRINT( 'There are no Python files to lint for Python 3 compatibility.' ) return [] python_utils.PRINT( 'Linting %s Python files for Python 3 compatibility.' % (num_py_files)) _batch_size = 50 current_batch_start_index = 0 while current_batch_start_index < len( files_to_lint_for_python3_compatibility): # Note that this index is an exclusive upper bound -- i.e., # the current batch of files ranges from 'start_index' to # 'end_index - 1'. current_batch_end_index = min( current_batch_start_index + _batch_size, len(files_to_lint_for_python3_compatibility)) current_files_to_lint = files_to_lint_for_python3_compatibility[ current_batch_start_index:current_batch_end_index] if self.verbose_mode_enabled: python_utils.PRINT( 'Linting Python files for Python 3 compatibility %s to %s..' % (current_batch_start_index + 1, current_batch_end_index)) with linter_utils.redirect_stdout(stdout): # This line invokes Pylint and prints its output # to the target stdout. python_utils.PRINT('Messages for Python 3 support:') pylinter_for_python3 = lint.Run(current_files_to_lint + ['--py3k'], exit=False).linter if pylinter_for_python3.msg_status != 0: summary_message = stdout.getvalue() python_utils.PRINT(summary_message) summary_messages.append(summary_message) any_errors = True current_batch_start_index = current_batch_end_index if any_errors: summary_message = ( '%s Python linting for Python 3 compatibility failed' % linter_utils.FAILED_MESSAGE_PREFIX) else: summary_message = ( '%s %s Python files linted for Python 3 compatibility ' '(%.1f secs)' % (linter_utils.SUCCESS_MESSAGE_PREFIX, num_py_files, (time.time() - start_time))) python_utils.PRINT(summary_message) summary_messages.append(summary_message) python_utils.PRINT( 'Python linting for Python 3 compatibility finished.') return summary_messages
def _lint_py_files(self, config_pylint, config_pycodestyle): """Prints a list of lint errors in the given list of Python files. Args: config_pylint: str. Path to the .pylintrc file. config_pycodestyle: str. Path to the tox.ini file. Return: summary_messages: list(str). Summary messages of lint check. """ files_to_lint = self.all_filepaths start_time = time.time() are_there_errors = False summary_messages = [] num_py_files = len(files_to_lint) python_utils.PRINT('Linting %s Python files' % num_py_files) _batch_size = 50 current_batch_start_index = 0 stdout = python_utils.string_io() while current_batch_start_index < len(files_to_lint): # Note that this index is an exclusive upper bound -- i.e., # the current batch of files ranges from 'start_index' to # 'end_index - 1'. current_batch_end_index = min( current_batch_start_index + _batch_size, len(files_to_lint)) current_files_to_lint = files_to_lint[ current_batch_start_index:current_batch_end_index] if self.verbose_mode_enabled: python_utils.PRINT( 'Linting Python files %s to %s...' % (current_batch_start_index + 1, current_batch_end_index)) with linter_utils.redirect_stdout(stdout): # This line invokes Pylint and prints its output # to the target stdout. pylinter = lint.Run(current_files_to_lint + [config_pylint], exit=False).linter # These lines invoke Pycodestyle and print its output # to the target stdout. style_guide = pycodestyle.StyleGuide( config_file=config_pycodestyle) pycodestyle_report = style_guide.check_files( paths=current_files_to_lint) if pylinter.msg_status != 0 or pycodestyle_report.get_count() != 0: summary_message = stdout.getvalue() python_utils.PRINT(summary_message) summary_messages.append(summary_message) are_there_errors = True current_batch_start_index = current_batch_end_index if are_there_errors: summary_message = ('%s Python linting failed' % (linter_utils.FAILED_MESSAGE_PREFIX)) else: summary_message = ('%s %s Python files linted (%.1f secs)' % (linter_utils.SUCCESS_MESSAGE_PREFIX, num_py_files, time.time() - start_time)) python_utils.PRINT(summary_message) summary_messages.append(summary_message) python_utils.PRINT('Python linting finished.') return summary_messages
def test_export_account_handler(self): # Update user settings to constants. user_id = self.get_user_id_from_email(self.EDITOR_EMAIL) user_settings = user_services.get_user_settings(user_id) user_settings.last_agreed_to_terms = self.GENERIC_DATE user_settings.last_logged_in = self.GENERIC_DATE user_settings.validate() user_models.UserSettingsModel( id=user_settings.user_id, gae_id=user_settings.gae_id, email=user_settings.email, role=user_settings.role, username=user_settings.username, normalized_username=user_settings.normalized_username, last_agreed_to_terms=user_settings.last_agreed_to_terms, last_started_state_editor_tutorial=( user_settings.last_started_state_editor_tutorial), last_started_state_translation_tutorial=( user_settings.last_started_state_translation_tutorial), last_logged_in=user_settings.last_logged_in, last_edited_an_exploration=user_settings. last_edited_an_exploration, last_created_an_exploration=( user_settings.last_created_an_exploration), profile_picture_data_url=user_settings.profile_picture_data_url, default_dashboard=user_settings.default_dashboard, creator_dashboard_display_pref=( user_settings.creator_dashboard_display_pref), user_bio=user_settings.user_bio, subject_interests=user_settings.subject_interests, first_contribution_msec=user_settings.first_contribution_msec, preferred_language_codes=user_settings.preferred_language_codes, preferred_site_language_code=( user_settings.preferred_site_language_code), preferred_audio_language_code=( user_settings.preferred_audio_language_code), deleted=user_settings.deleted).put() constants_swap = self.swap(constants, 'ENABLE_ACCOUNT_EXPORT', True) time_swap = self.swap(user_services, 'record_user_logged_in', lambda *args: None) with constants_swap, time_swap: data = self.get_custom_response('/export-account-handler', 'text/plain') # Check downloaded zip file. filename = 'oppia_takeout_data.zip' self.assertEqual(data.headers['Content-Disposition'], 'attachment; filename=%s' % filename) zf_saved = zipfile.ZipFile( python_utils.string_io(buffer_value=data.body)) self.assertEqual(zf_saved.namelist(), [ 'oppia_takeout_data.json', 'images/user_settings_profile_picture.png' ]) # Load golden zip file. golden_zip_filepath = os.path.join(feconf.TESTS_DATA_DIR, 'oppia_takeout_data.zip') with python_utils.open_file(golden_zip_filepath, 'rb', encoding=None) as f: golden_zipfile = f.read() zf_gold = zipfile.ZipFile( python_utils.string_io(buffer_value=golden_zipfile)) self.assertEqual( zf_saved.open('oppia_takeout_data.json').read(), zf_gold.open('oppia_takeout_data.json').read()) self.assertEqual( zf_saved.open( 'images/user_settings_profile_picture.png').read(), zf_gold.open( 'images/user_settings_profile_picture.png').read())
def post(self, exploration_id): """Saves an audio file uploaded by a content creator.""" raw_audio_file = self.request.get('raw_audio_file') filename = self.payload.get('filename') allowed_formats = list(feconf.ACCEPTED_AUDIO_EXTENSIONS.keys()) if not raw_audio_file: raise self.InvalidInputException('No audio supplied') dot_index = filename.rfind('.') extension = filename[dot_index + 1:].lower() if dot_index == -1 or dot_index == 0: raise self.InvalidInputException( 'No filename extension: it should have ' 'one of the following extensions: %s' % allowed_formats) if extension not in feconf.ACCEPTED_AUDIO_EXTENSIONS: raise self.InvalidInputException( 'Invalid filename extension: it should have ' 'one of the following extensions: %s' % allowed_formats) tempbuffer = python_utils.string_io() tempbuffer.write(raw_audio_file) tempbuffer.seek(0) try: # For every accepted extension, use the mutagen-specific # constructor for that type. This will catch mismatched audio # types e.g. uploading a flac file with an MP3 extension. if extension == 'mp3': audio = mp3.MP3(tempbuffer) else: audio = mutagen.File(tempbuffer) except mutagen.MutagenError: # The calls to mp3.MP3() versus mutagen.File() seem to behave # differently upon not being able to interpret the audio. # mp3.MP3() raises a MutagenError whereas mutagen.File() # seems to return None. It's not clear if this is always # the case. Occasionally, mutagen.File() also seems to # raise a MutagenError. raise self.InvalidInputException('Audio not recognized ' 'as a %s file' % extension) tempbuffer.close() if audio is None: raise self.InvalidInputException('Audio not recognized ' 'as a %s file' % extension) if audio.info.length > feconf.MAX_AUDIO_FILE_LENGTH_SEC: raise self.InvalidInputException( 'Audio files must be under %s seconds in length. The uploaded ' 'file is %.2f seconds long.' % (feconf.MAX_AUDIO_FILE_LENGTH_SEC, audio.info.length)) if len( set(audio.mime).intersection( set(feconf.ACCEPTED_AUDIO_EXTENSIONS[extension]))) == 0: raise self.InvalidInputException( 'Although the filename extension indicates the file ' 'is a %s file, it was not recognized as one. ' 'Found mime types: %s' % (extension, audio.mime)) mimetype = audio.mime[0] # For a strange, unknown reason, the audio variable must be # deleted before opening cloud storage. If not, cloud storage # throws a very mysterious error that entails a mutagen # object being recursively passed around in app engine. del audio # Audio files are stored to the datastore in the dev env, and to GCS # in production. file_system_class = fs_services.get_entity_file_system_class() fs = fs_domain.AbstractFileSystem( file_system_class(feconf.ENTITY_TYPE_EXPLORATION, exploration_id)) fs.commit(self.user_id, '%s/%s' % (self._FILENAME_PREFIX, filename), raw_audio_file, mimetype=mimetype) self.render_json({'filename': filename})