def map(item): if item.deleted: return err_dict = {} try: exploration = exp_fetchers.get_exploration_from_model(item) except Exception as e: yield ('Error %s when loading exploration' % python_utils.UNICODE(e), [item.id]) return html_list = exploration.get_all_html_content_strings() err_dict = html_validation_service.validate_customization_args( html_list) for key in err_dict: err_value_with_exp_id = err_dict[key] err_value_with_exp_id.append('Exp ID: %s' % item.id) yield (key, err_value_with_exp_id)
def save_hashes_to_file(file_hashes): """Return JS code that loads hashes needed for frontend into variable. Args: file_hashes: dict(str, str). Dictionary with filepaths as keys and hashes of file content as values. Returns: str. JS code loading hashes as JSON into variable. """ # Only some of the hashes are needed in the frontend. filtered_hashes = filter_hashes(file_hashes) ensure_directory_exists(HASHES_JSON_FILEPATH) with python_utils.open_file(HASHES_JSON_FILEPATH, 'w+') as hashes_json_file: hashes_json_file.write( python_utils.UNICODE( json.dumps(filtered_hashes, ensure_ascii=False))) hashes_json_file.write(u'\n')
def run_job_sync( job_name: str, job_args: List[str], namespace: Optional[str] = None) -> beam_job_domain.BeamJobRun: """Runs the specified job synchronously. In other words, the function will wait for the job to finish running before returning a value. Args: job_name: str. The name of the job to run. job_args: list(str). The arguments to the job's run() method. namespace: str. The namespace in which models should be created. Returns: BeamJobRun. Contains metadata related to the execution status of the job. """ job_pipeline = beam.Pipeline( runner=runners.DirectRunner(), options=job_options.JobOptions(namespace=namespace)) job_class = registry.get_job_class_by_name(job_name) job = job_class(job_pipeline) run_model = beam_job_services.create_beam_job_run_model(job_name, job_args) try: with job_pipeline: unused_pdone = job.run(*job_args) | job_io.PutResults(run_model.id) except Exception as exception: run_model.latest_job_state = beam_job_models.BeamJobState.FAILED.value # If the pipeline fails to put the results into storage, then we'll # explicitly write them to storage by using the caught exception. result_model = beam_job_services.create_beam_job_run_result_model( run_model.id, '', python_utils.UNICODE(exception)) result_model.put() else: run_model.latest_job_state = beam_job_models.BeamJobState.DONE.value finally: run_model.put() return beam_job_services.get_beam_job_run_from_model(run_model)
def add_svg_filenames_for_latex_strings_in_html_string( raw_latex_to_image_data_dict, html_string): """Adds the filenames for math rich-text components with empty svg_filename field based on the given images data. TODO(#10045): Remove this function once all the math-rich text components in explorations have a valid math SVG stored in the datastore. Args: raw_latex_to_image_data_dict: dict(str, LatexStringSvgImageData). The dictionary having the key as a LaTeX string and the corresponding value as the SVG image data for that LaTeX string. html_string: str. HTML string to modify. Returns: str. Updated HTML string with all Math component tags having a filename. """ soup = bs4.BeautifulSoup(html_string.encode(encoding='utf-8'), 'html.parser') for math_tag in soup.findAll(name='oppia-noninteractive-math'): math_content_dict = (json.loads( unescape_html(math_tag['math_content-with-value']))) raw_latex = (objects.UnicodeString.normalize( math_content_dict['raw_latex'])) svg_filename = (objects.UnicodeString.normalize( math_content_dict['svg_filename'])) if svg_filename == '' and (raw_latex in raw_latex_to_image_data_dict.keys()): dimensions = (raw_latex_to_image_data_dict[raw_latex]. latex_string_svg_image_dimensions) filename = (generate_math_svgs_filename(dimensions)) math_content_dict = { 'raw_latex': raw_latex, 'svg_filename': objects.UnicodeString.normalize(filename) } normalized_math_content_dict = ( objects.MathExpressionContent.normalize(math_content_dict)) math_tag['math_content-with-value'] = (escape_html( json.dumps(normalized_math_content_dict, sort_keys=True))) return python_utils.UNICODE(soup)
def test_recursively_convert_to_str_with_nested_structure(self): test_var_1_in_unicode = python_utils.UNICODE('test_var_1') test_list_1 = [ test_var_1_in_unicode, test_var_1_in_unicode.encode('utf-8'), 'test_var_2', b'test_var_3', {'test_var_4': b'test_var_5'}] test_dict = {test_var_1_in_unicode: test_list_1} self.assertEqual( test_dict, { 'test_var_1': [ 'test_var_1', b'test_var_1', 'test_var_2', b'test_var_3', {'test_var_4': b'test_var_5'}] } ) dict_in_str = python_utils._recursively_convert_to_str(test_dict) # pylint: disable=protected-access self.assertEqual( dict_in_str, { 'test_var_1': [ 'test_var_1', b'test_var_1', 'test_var_2', 'test_var_3', {'test_var_4': 'test_var_5'}] } ) for key, value in dict_in_str.items(): self.assertNotEqual(type(key), future.types.newstr) self.assertNotEqual(type(key), future.types.newbytes) self.assertTrue(isinstance(key, unicode)) for item in value: self.assertNotEqual(type(item), future.types.newstr) self.assertNotEqual(type(item), future.types.newbytes) self.assertTrue(isinstance(item, (unicode, bytes, dict))) for k, v in value[-1].items(): self.assertNotEqual(type(k), future.types.newstr) self.assertNotEqual(type(k), future.types.newbytes) self.assertNotEqual(type(v), future.types.newstr) self.assertNotEqual(type(v), future.types.newbytes) self.assertEqual(type(k), unicode) self.assertEqual(type(v), bytes)
def test_all_fields_have_export_policy(self): """Ensure every field in every model has an export policy defined.""" all_models = [ clazz for clazz in test_utils.get_storage_model_classes() if (not clazz.__name__ in test_utils.BASE_MODEL_CLASSES_WITHOUT_DATA_POLICIES) ] for model in all_models: export_policy = model.get_export_policy() self.assertEqual( sorted([ python_utils.UNICODE(prop) for prop in model._properties ]), # pylint: disable=protected-access sorted(export_policy.keys())) self.assertTrue( set(export_policy.values()).issubset({ base_models.EXPORT_POLICY.EXPORTED, (base_models.EXPORT_POLICY.EXPORTED_AS_KEY_FOR_TAKEOUT_DICT ), base_models.EXPORT_POLICY.NOT_APPLICABLE }))
def add_caption_attr_to_image(html_string): """Adds caption attribute to all oppia-noninteractive-image tags. Args: html_string: str. HTML string in which the caption attribute is to be added. Returns: str. Updated HTML string with the caption attribute for all oppia-noninteractive-image tags. """ soup = bs4.BeautifulSoup(html_string.encode(encoding='utf-8'), 'html.parser') for image in soup.findAll(name='oppia-noninteractive-image'): attrs = image.attrs if 'caption-with-value' not in attrs: image['caption-with-value'] = escape_html(json.dumps('')) return python_utils.UNICODE(soup)
def gather_logs(start, stop='HEAD'): """Gathers the logs between the start and endpoint. Args: start: str. Tag, Branch or SHA1 of start point stop: str. Tag, Branch or SHA1 of end point, defaults to HEAD Returns: list(Log): List of Logs. """ get_logs_cmd = GIT_CMD_GET_LOGS_FORMAT_STRING.format( GROUP_SEP, start, stop) # The unicode conversion is required because there can be non-ascii # characters in the logs and it can result in breaking the flow # of release summary generation. out = python_utils.UNICODE(common.run_cmd(get_logs_cmd.split(' ')), 'utf-8').split('\x00') if len(out) == 1 and out[0] == '': return [] else: return [Log(*line.strip().split(GROUP_SEP)) for line in out]
def generate_new_blog_post_id(cls): """Generates a new blog post ID which is unique and is in the form of random hash of 12 chars. Returns: str. A blog post ID that is different from the IDs of all the existing blog posts. Raises: Exception. There were too many collisions with existing blog post IDs when attempting to generate a new blog post ID. """ for _ in python_utils.RANGE(base_models.MAX_RETRIES): blog_post_id = utils.convert_to_hash( python_utils.UNICODE( utils.get_random_int(base_models.RAND_RANGE)), base_models.ID_LENGTH) if not cls.get_by_id(blog_post_id): return blog_post_id raise Exception( 'New blog post id generator is producing too many collisions.')
def convert_tag_contents_to_rte_format(html_data, rte_conversion_fn): """This function converts the rich text content within tabs and collapsible components to given RTE format. If the html_data does not contain tab or collapsible components it will do nothing. Args: html_data: str. The HTML string whose content is to be converted. rte_conversion_fn: function. The RTE conversion function for html strings. Returns: str. The HTML string with converted content within tag. """ soup = bs4.BeautifulSoup(html_data.encode(encoding='utf-8'), 'html.parser') for collapsible in soup.findAll(name='oppia-noninteractive-collapsible'): # To ensure that collapsible tags have content-with-value attribute. if 'content-with-value' not in collapsible.attrs or ( collapsible['content-with-value'] == ''): collapsible['content-with-value'] = escape_html(json.dumps('')) content_html = json.loads( unescape_html(collapsible['content-with-value'])) collapsible['content-with-value'] = escape_html( json.dumps(rte_conversion_fn(content_html))) # To ensure that collapsible tags have heading-with-value attribute. if 'heading-with-value' not in collapsible.attrs: collapsible['heading-with-value'] = escape_html(json.dumps('')) for tabs in soup.findAll(name='oppia-noninteractive-tabs'): tab_content_json = unescape_html(tabs['tab_contents-with-value']) tab_content_list = json.loads(tab_content_json) for index, tab_content in enumerate(tab_content_list): tab_content_list[index]['content'] = rte_conversion_fn( tab_content['content']) tabs['tab_contents-with-value'] = escape_html( json.dumps(tab_content_list)) return python_utils.UNICODE(soup)
def _sort_and_slice_similarities( similarities: Iterable[Dict[str, Union[str, float]]]) -> List[str]: """Sorts similarities of explorations and slices them to a maximum length. Args: similarities:iterable(). Iterable of dictionaries. The structure of the dictionaries is: exp_id: str. The ID of the similar exploration. similarity_score: float. The similarity score for the exploration. Returns: list(str). List of exploration IDs, sorted by the similarity. """ sorted_similarities = sorted(similarities, reverse=True, key=lambda x: x['similarity_score']) return [ python_utils.UNICODE(item['exp_id']) for item in sorted_similarities ][:MAX_RECOMMENDATIONS]
def test_reply_to_id_flag(self): """Verifies that the reply_to_id flag is working properly.""" mailgun_api = self.swap(feconf, 'MAILGUN_API_KEY', 'api') mailgun_domain = self.swap(feconf, 'MAILGUN_DOMAIN_NAME', 'domain') allow_emailing = self.swap(feconf, 'CAN_SEND_EMAILS', True) reply_id = 123 # Lambda function, will replace post_to_mailgun(). req_post_lambda = (lambda data=None: self.assertEqual( data['h:Reply-To'], 'reply+' + python_utils.UNICODE(reply_id) + '@' + feconf.INCOMING_EMAILS_DOMAIN_NAME)) post_request = self.swap(mailgun_email_services, 'post_to_mailgun', req_post_lambda) with mailgun_api, mailgun_domain, post_request, allow_emailing: mailgun_email_services.send_mail(feconf.SYSTEM_EMAIL_ADDRESS, feconf.ADMIN_EMAIL_ADDRESS, 'subject', 'body', 'html', bcc_admin=False, reply_to_id=reply_id)
def establish_firebase_connection(): """Establishes the connection to Firebase needed by the rest of the SDK. All Firebase operations require an "app", the abstraction used for a Firebase server connection. The initialize_app() function raises an error when it's called more than once, however, so we make this function idempotent by trying to "get" the app first. Returns: firebase_admin.App. The App being by the Firebase SDK. Raises: Exception. The Firebase app has a genuine problem. """ try: firebase_admin.get_app() except ValueError as error: if 'initialize_app' in python_utils.UNICODE(error): firebase_admin.initialize_app( options={'projectId': feconf.OPPIA_PROJECT_ID}) else: raise
def permanently_delete_user_from_list(user_email: str) -> None: """Permanently deletes the user with the given email from the Mailchimp list. NOTE TO DEVELOPERS: This should only be called from the wipeout service since once a user is permanently deleted from mailchimp, they cannot be programmatically added back via their API (the user would have to manually resubscribe back). Args: user_email: str. Email ID of the user. Email is used to uniquely identify the user in the mailchimp DB. Raises: Exception. Any error raised by the mailchimp API. """ client = _get_mailchimp_class() if not client: return None subscriber_hash = _get_subscriber_hash(user_email) try: client.lists.members.get( feconf.MAILCHIMP_AUDIENCE_ID, subscriber_hash) client.lists.members.delete_permanent( feconf.MAILCHIMP_AUDIENCE_ID, subscriber_hash) except mailchimpclient.MailChimpError as error: # This has to be done since the message can only be accessed from # MailChimpError by error.message in Python2, but this is deprecated in # Python3. # In Python3, the message can be accessed directly by KeyError # (https://github.com/VingtCinq/python-mailchimp/pull/65), so as a # workaround for Python2, the 'message' attribute is obtained by # str() and then it is converted to dict. This works in Python3 as well. error_message = ast.literal_eval(python_utils.UNICODE(error)) # Ignore if the error corresponds to "User does not exist". if error_message['status'] != 404: raise Exception(error_message['detail'])
def _fetch_field_name_to_external_model_references(cls, item): """Fetch external models based on _get_external_id_relationships. This should be called before we call other _validate methods. Args: item: datastore_services.Model. Entity to validate. """ multiple_models_ids_to_fetch = {} try: for external_model_fetcher_details in ( cls._get_external_id_relationships(item)): multiple_models_ids_to_fetch[ external_model_fetcher_details.field_name] = ( external_model_fetcher_details.model_class, external_model_fetcher_details.model_ids) except utils.ValidationError as err: cls._add_error( ERROR_CATEGORY_INVALID_USER_SETTING_IDS, 'Entity id %s: %s' % (item.id, python_utils.UNICODE(err))) else: fetched_model_instances_for_all_ids = ( datastore_services.fetch_multiple_entities_by_ids_and_models( list(multiple_models_ids_to_fetch.values()))) for index, field_name in enumerate(multiple_models_ids_to_fetch): (model_class, model_ids) = ( multiple_models_ids_to_fetch[field_name]) fetched_model_instances = fetched_model_instances_for_all_ids[ index] for (model_id, model_instance) in python_utils.ZIP( model_ids, fetched_model_instances): cls.field_name_to_external_model_references[ field_name].append( ExternalModelReference( model_class, model_id, model_instance))
def _create_user_in_mailchimp_db(user_email: str) -> bool: """Creates a new user in the mailchimp database and handles the case where the user was permanently deleted from the database. Args: user_email: str. Email ID of the user. Email is used to uniquely identify the user in the mailchimp DB. Returns: bool. Whether the user was successfully added to the db. (This will be False if the user was permanently deleted earlier and therefore cannot be added back.) Raises: Exception. Any error (other than the one mentioned below) raised by the mailchimp API. """ post_data = { 'email_address': user_email, 'status': 'subscribed' } client = _get_mailchimp_class() try: client.lists.members.create(feconf.MAILCHIMP_AUDIENCE_ID, post_data) except mailchimpclient.MailChimpError as error: error_message = ast.literal_eval(python_utils.UNICODE(error)) # This is the specific error message returned for the case where the # user was permanently deleted from the Mailchimp database earlier. # This was found by experimenting with the MailChimp API. Note that the # error reference # (https://mailchimp.com/developer/marketing/docs/errors/) is not # comprehensive, since, under status 400, they only list a subset of the # common error titles. if error_message['title'] == 'Forgotten Email Not Subscribed': return False raise Exception(error_message['detail']) return True
def _job_bookkeeping_context( job_name: str ) -> Iterator[beam_job_models.BeamJobRunModel]: """Returns a context manager which commits failure details if an exception occurs. Args: job_name: str. The name of the job. Yields: BeamJobRunModel. The bookkeeping model used to record execution details. """ run_model = beam_job_services.create_beam_job_run_model(job_name) try: yield run_model except Exception as exception: run_model.latest_job_state = beam_job_models.BeamJobState.FAILED.value _put_job_stderr(run_model.id, python_utils.UNICODE(exception)) finally: run_model.put()
def _get_new_id(cls): """Generates a unique ID for the question in the form of random hash of 12 chars. Returns: new_id: int. ID of the new QuestionModel instance. Raises: Exception. The ID generator for QuestionModel is producing too many collisions. """ for _ in python_utils.RANGE(base_models.MAX_RETRIES): new_id = utils.convert_to_hash( python_utils.UNICODE( utils.get_random_int(base_models.RAND_RANGE)), base_models.ID_LENGTH) if not cls.get_by_id(new_id): return new_id raise Exception( 'The id generator for QuestionModel is producing too many ' 'collisions.')
def run(self, job_id, job_class_str, output): """Extracts the results of a MR job and registers its completion. Args: job_id: str. The ID of the job to run. job_class_str: str. Should uniquely identify each type of job. output: str. The output produced by the job. """ job_class = mapreduce_util.for_name(job_class_str) try: iterator = input_readers.GoogleCloudStorageInputReader(output, 0) results_list = [] for item_reader in iterator: for item in item_reader: results_list.append(json.loads(item)) job_class.register_completion(job_id, results_list) except Exception as e: logging.exception('Job %s failed at %s' % (job_id, utils.get_current_time_in_millisecs())) job_class.register_failure( job_id, '%s\n%s' % (python_utils.UNICODE(e), traceback.format_exc()))
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) error_messages = [] for _, state in exploration.states.items(): if state.interaction.id is None: continue try: ca_specs = ( interaction_registry.Registry.get_interaction_by_id( state.interaction.id).customization_arg_specs ) customization_args_dict = {} for ca_name in state.interaction.customization_args: customization_args_dict[ca_name] = ( state.interaction.customization_args[ ca_name].to_customization_arg_dict() ) customization_args_util.validate_customization_args_and_values( 'interaction', state.interaction.id, customization_args_dict, ca_specs, fail_on_validation_errors=True ) except Exception as e: error_messages.append( '%s: %s' % (state.interaction.id, python_utils.UNICODE(e))) if error_messages: yield ( 'Failed customization args validation for exp ' 'id %s' % item.id, ', '.join(error_messages))
def get_extra_commits_in_new_release(base_commit, repo): """Gets extra commits in the new release. Args: base_commit: str. The base commit common between current branch and the latest release. repo: github.Repository.Repository. The PyGithub object for the repo. Returns: list(github.Commit.Commit). List of commits from the base commit up to the current commit, which haven't been cherrypicked already. """ get_commits_cmd = GIT_CMD_TEMPLATE_GET_NEW_COMMITS % base_commit out = python_utils.UNICODE(common.run_cmd(get_commits_cmd.split(' ')), 'utf-8').split('\n') commits = [] for line in out: # Lines that start with a - are already cherrypicked. The commits of # interest are on lines that start with +. if line[0] == '+': line = line[2:] commit = repo.get_commit(line[:line.find(' ')]) commits.append(commit) return commits
def mock_print(*args): """Mock for python_utils.PRINT.""" self.log = ' '.join(python_utils.UNICODE(arg) for arg in args)
def delete_multi(cls, entity_ids, committer_id, commit_message, force_deletion=False): """Deletes the given cls instancies with the given entity_ids. Args: entity_ids: list(str). Ids of entities to delete. committer_id: str. The user_id of the user who committed the change. commit_message: str. The commit description message. force_deletion: bool. If True these models are deleted completely from storage, otherwise there are only marked as deleted. Default is False. Raises: Exception: This model instance has been already deleted. """ versioned_models = cls.get_multi(entity_ids) if force_deletion: all_models_metadata_keys = [] all_models_content_keys = [] for model in versioned_models: model_version_numbers = [ python_utils.UNICODE(num + 1) for num in python_utils.RANGE(model.version) ] model_snapshot_ids = [ model.get_snapshot_id(model.id, version_number) for version_number in model_version_numbers ] all_models_metadata_keys.extend([ ndb.Key(model.SNAPSHOT_METADATA_CLASS, snapshot_id) for snapshot_id in model_snapshot_ids ]) all_models_content_keys.extend([ ndb.Key(model.SNAPSHOT_CONTENT_CLASS, snapshot_id) for snapshot_id in model_snapshot_ids ]) versioned_models_keys = [model.key for model in versioned_models] transaction_services.run_in_transaction( ndb.delete_multi, all_models_metadata_keys + all_models_content_keys + versioned_models_keys) else: for model in versioned_models: model._require_not_marked_deleted() # pylint: disable=protected-access model.deleted = True commit_cmds = [{'cmd': cls.CMD_DELETE_COMMIT}] snapshot_metadata_models = [] snapshot_content_models = [] for model in versioned_models: model.version += 1 snapshot = model.compute_snapshot() snapshot_id = model.get_snapshot_id(model.id, model.version) snapshot_metadata_models.append( model.SNAPSHOT_METADATA_CLASS.create( snapshot_id, committer_id, cls._COMMIT_TYPE_DELETE, commit_message, commit_cmds)) snapshot_content_models.append( model.SNAPSHOT_CONTENT_CLASS.create(snapshot_id, snapshot)) transaction_services.run_in_transaction( BaseModel.put_multi, snapshot_metadata_models + snapshot_content_models + versioned_models)
def map(snapshot_model): job_class = AddMissingCommitLogsOneOffJob model_class_name = snapshot_model.__class__.__name__ model_id, version_str = snapshot_model.id.rsplit('-', 1) model_properties = job_class.MODEL_NAMES_TO_PROPERTIES[ model_class_name] version = int(version_str) commit_log_id = (model_properties['id_string_format'] % (model_id, version)) commit_log_model = (model_properties['commit_log_model_class']. get_by_id(commit_log_id)) commit_logs_should_exist = True parent_model = ( model_properties['parent_model_class'].get_by_id(model_id)) if model_class_name == 'ExplorationRightsSnapshotMetadataModel': if snapshot_model.commit_type in ['create', 'delete']: commit_logs_should_exist = False if commit_log_model is not None or not commit_logs_should_exist: yield ('Found commit log model-%s' % model_class_name, 1) return if parent_model is None: yield ('Missing Parent Model-No changes-%s' % model_class_name, 1) return if parent_model.deleted: model_properties['delete_method'](feconf.SYSTEM_COMMITTER_ID, parent_model.id, force_deletion=True) yield ('SUCCESS-Parent model marked deleted-' + 'Deleted all related models-%s' % (model_class_name), snapshot_model.id) return commit_log_model = model_properties['commit_log_model_class']( id=python_utils.UNICODE(commit_log_id), user_id=snapshot_model.committer_id, commit_type=snapshot_model.commit_type, commit_message=snapshot_model.commit_message, commit_cmds=snapshot_model.commit_cmds, created_on=snapshot_model.created_on, last_updated=snapshot_model.last_updated, version=version) setattr(commit_log_model, model_properties['id_field'], model_id) if model_class_name in ( job_class.MODEL_NAMES_WITH_DEFAULT_COMMIT_STATUS): commit_log_model.post_commit_status = ( constants.ACTIVITY_STATUS_PUBLIC) elif model_class_name in ( job_class.MODEL_NAMES_WITH_COMMIT_STATUS_IN_RIGHTS): rights_model = exp_models.ExplorationRightsModel.get_version( model_id, version) commit_log_model.post_commit_status = rights_model.status commit_log_model.post_commit_is_private = ( commit_log_model.post_commit_status == ( constants.ACTIVITY_STATUS_PRIVATE)) commit_log_model.update_timestamps(update_last_updated_time=False) commit_log_model.put() yield ('SUCCESS-Added missing commit log model-%s' % model_class_name, snapshot_model.id)
def normalize_against_schema(obj, schema, apply_custom_validators=True): """Validate the given object using the schema, normalizing if necessary. Args: obj: *. The object to validate and normalize. schema: dict(str, *). The schema to validate and normalize the value against. apply_custom_validators: bool. Whether to validate the normalized object using the validators defined in the schema. Returns: *. The normalized object. Raises: AssertionError: The object fails to validate against the schema. """ normalized_obj = None if schema[SCHEMA_KEY_TYPE] == SCHEMA_TYPE_BOOL: assert isinstance(obj, bool), ('Expected bool, received %s' % obj) normalized_obj = obj elif schema[SCHEMA_KEY_TYPE] == SCHEMA_TYPE_CUSTOM: # Importing this at the top of the file causes a circular dependency. # TODO(sll): Either get rid of custom objects or find a way to merge # them into the schema framework -- probably the latter. from core.domain import obj_services obj_class = obj_services.Registry.get_object_class_by_type( schema[SCHEMA_KEY_OBJ_TYPE]) if not apply_custom_validators: normalized_obj = normalize_against_schema( obj, obj_class.SCHEMA, apply_custom_validators=False) else: normalized_obj = obj_class.normalize(obj) elif schema[SCHEMA_KEY_TYPE] == SCHEMA_TYPE_DICT: assert isinstance(obj, dict), ('Expected dict, received %s' % obj) expected_dict_keys = [ p[SCHEMA_KEY_NAME] for p in schema[SCHEMA_KEY_PROPERTIES] ] assert set(obj.keys()) == set(expected_dict_keys), ( 'Missing keys: %s, Extra keys: %s' % (list(set(expected_dict_keys) - set(obj.keys())), list(set(obj.keys()) - set(expected_dict_keys)))) normalized_obj = {} for prop in schema[SCHEMA_KEY_PROPERTIES]: key = prop[SCHEMA_KEY_NAME] normalized_obj[key] = normalize_against_schema( obj[key], prop[SCHEMA_KEY_SCHEMA]) elif schema[SCHEMA_KEY_TYPE] == SCHEMA_TYPE_FLOAT: obj = float(obj) assert isinstance(obj, numbers.Real), ('Expected float, received %s' % obj) normalized_obj = obj elif schema[SCHEMA_KEY_TYPE] == SCHEMA_TYPE_INT: obj = int(obj) assert isinstance( obj, numbers.Integral), ('Expected int, received %s' % obj) assert isinstance(obj, int), ('Expected int, received %s' % obj) normalized_obj = obj elif schema[SCHEMA_KEY_TYPE] == SCHEMA_TYPE_HTML: assert isinstance(obj, python_utils.BASESTRING), ( 'Expected unicode HTML string, received %s' % obj) if isinstance(obj, bytes): obj = obj.decode('utf-8') else: obj = python_utils.UNICODE(obj) assert isinstance( obj, python_utils.UNICODE), ('Expected unicode, received %s' % obj) normalized_obj = html_cleaner.clean(obj) elif schema[SCHEMA_KEY_TYPE] == SCHEMA_TYPE_LIST: assert isinstance(obj, list), ('Expected list, received %s' % obj) item_schema = schema[SCHEMA_KEY_ITEMS] if SCHEMA_KEY_LEN in schema: assert len(obj) == schema[SCHEMA_KEY_LEN] normalized_obj = [ normalize_against_schema(item, item_schema) for item in obj ] elif schema[SCHEMA_KEY_TYPE] == SCHEMA_TYPE_UNICODE: assert isinstance( obj, python_utils.BASESTRING), ('Expected unicode string, received %s' % obj) if isinstance(obj, bytes): obj = obj.decode('utf-8') else: obj = python_utils.UNICODE(obj) assert isinstance( obj, python_utils.UNICODE), ('Expected unicode, received %s' % obj) normalized_obj = obj else: raise Exception('Invalid schema type: %s' % schema[SCHEMA_KEY_TYPE]) if SCHEMA_KEY_CHOICES in schema: assert normalized_obj in schema[SCHEMA_KEY_CHOICES], ( 'Received %s which is not in the allowed range of choices: %s' % (normalized_obj, schema[SCHEMA_KEY_CHOICES])) # When type normalization is finished, apply the post-normalizers in the # given order. if SCHEMA_KEY_POST_NORMALIZERS in schema: for normalizer in schema[SCHEMA_KEY_POST_NORMALIZERS]: kwargs = dict(normalizer) del kwargs['id'] normalized_obj = Normalizers.get(normalizer['id'])(normalized_obj, **kwargs) # Validate the normalized object. if apply_custom_validators: if SCHEMA_KEY_VALIDATORS in schema: for validator in schema[SCHEMA_KEY_VALIDATORS]: kwargs = dict(validator) del kwargs['id'] assert get_validator(validator['id'])( normalized_obj, **kwargs), ('Validation failed: %s (%s) for object %s' % (validator['id'], kwargs, normalized_obj)) return normalized_obj
def handle_exception(self, exception, unused_debug_mode): """Overwrites the default exception handler. Args: exception: Exception. The exception that was thrown. unused_debug_mode: bool. True if the web application is running in debug mode. """ if isinstance(exception, self.NotLoggedInException): # This checks if the response should be JSON or HTML. # For GET requests, there is no payload, so we check against # GET_HANDLER_ERROR_RETURN_TYPE. # Otherwise, we check whether self.payload exists. if (self.payload is not None or self.GET_HANDLER_ERROR_RETURN_TYPE == feconf.HANDLER_TYPE_JSON): self.error(401) self._render_exception( 401, { 'error': ( 'You must be logged in to access this resource.')}) else: self.redirect(user_services.create_login_url(self.request.uri)) return logging.exception('Exception raised: %s', exception) if isinstance(exception, self.PageNotFoundException): logging.warning('Invalid URL requested: %s', self.request.uri) self.error(404) self._render_exception( 404, { 'error': 'Could not find the page %s.' % self.request.uri}) return logging.exception('Exception raised: %s', exception) if isinstance(exception, self.UnauthorizedUserException): self.error(401) self._render_exception( 401, {'error': python_utils.UNICODE(exception)}) return if isinstance(exception, self.InvalidInputException): self.error(400) self._render_exception( 400, {'error': python_utils.UNICODE(exception)}) return if isinstance(exception, self.InternalErrorException): self.error(500) self._render_exception( 500, {'error': python_utils.UNICODE(exception)}) return if isinstance(exception, self.TemporaryMaintenanceException): self.error(503) self._render_exception( 503, {'error': python_utils.UNICODE(exception)}) return self.error(500) self._render_exception( 500, {'error': python_utils.UNICODE(exception)})
def post(self): """Handles POST requests.""" try: result = {} if self.payload.get('action') == 'reload_exploration': exploration_id = self.payload.get('exploration_id') self._reload_exploration(exploration_id) elif self.payload.get('action') == 'reload_collection': collection_id = self.payload.get('collection_id') self._reload_collection(collection_id) elif self.payload.get('action') == 'generate_dummy_explorations': num_dummy_exps_to_generate = self.payload.get( 'num_dummy_exps_to_generate') num_dummy_exps_to_publish = self.payload.get( 'num_dummy_exps_to_publish') if not isinstance(num_dummy_exps_to_generate, int): raise self.InvalidInputException( '%s is not a number' % num_dummy_exps_to_generate) elif not isinstance(num_dummy_exps_to_publish, int): raise self.InvalidInputException('%s is not a number' % num_dummy_exps_to_publish) elif num_dummy_exps_to_generate < num_dummy_exps_to_publish: raise self.InvalidInputException( 'Generate count cannot be less than publish count') else: self._generate_dummy_explorations( num_dummy_exps_to_generate, num_dummy_exps_to_publish) elif self.payload.get('action') == 'clear_search_index': search_services.clear_collection_search_index() search_services.clear_exploration_search_index() elif (self.payload.get('action') == 'generate_dummy_new_structures_data'): self._load_dummy_new_structures_data() elif self.payload.get('action') == ( 'flush_migration_bot_contribution_data'): user_services.flush_migration_bot_contributions_model() elif self.payload.get('action') == 'save_config_properties': new_config_property_values = self.payload.get( 'new_config_property_values') logging.info('[ADMIN] %s saved config property values: %s' % (self.user_id, new_config_property_values)) for (name, value) in new_config_property_values.items(): config_services.set_property(self.user_id, name, value) elif self.payload.get('action') == 'revert_config_property': config_property_id = self.payload.get('config_property_id') logging.info('[ADMIN] %s reverted config property: %s' % (self.user_id, config_property_id)) config_services.revert_property(self.user_id, config_property_id) elif self.payload.get('action') == 'start_new_job': for klass in (jobs_registry.ONE_OFF_JOB_MANAGERS + (jobs_registry.AUDIT_JOB_MANAGERS)): if klass.__name__ == self.payload.get('job_type'): klass.enqueue(klass.create_new()) break elif self.payload.get('action') == 'cancel_job': job_id = self.payload.get('job_id') job_type = self.payload.get('job_type') for klass in (jobs_registry.ONE_OFF_JOB_MANAGERS + (jobs_registry.AUDIT_JOB_MANAGERS)): if klass.__name__ == job_type: klass.cancel(job_id, self.user_id) break elif self.payload.get('action') == 'start_computation': computation_type = self.payload.get('computation_type') for klass in jobs_registry.ALL_CONTINUOUS_COMPUTATION_MANAGERS: if klass.__name__ == computation_type: klass.start_computation() break elif self.payload.get('action') == 'stop_computation': computation_type = self.payload.get('computation_type') for klass in jobs_registry.ALL_CONTINUOUS_COMPUTATION_MANAGERS: if klass.__name__ == computation_type: klass.stop_computation(self.user_id) break elif self.payload.get('action') == 'upload_topic_similarities': data = self.payload.get('data') recommendations_services.update_topic_similarities(data) elif self.payload.get('action') == ( 'regenerate_topic_related_opportunities'): topic_id = self.payload.get('topic_id') opportunities_count = ( opportunity_services. regenerate_opportunities_related_to_topic( topic_id, delete_existing_opportunities=True)) result = {'opportunities_count': opportunities_count} self.render_json(result) except Exception as e: self.render_json({'error': python_utils.UNICODE(e)}) raise
def add_math_content_to_math_rte_components(html_string): """Replaces the attribute raw_latex-with-value in all Math component tags with a new attribute math_content-with-value. The new attribute has an additional field for storing SVG filenames. The field for SVG filename will be an empty string. Args: html_string: str. HTML string to modify. Returns: str. Updated HTML string with all Math component tags having the new attribute. """ soup = bs4.BeautifulSoup( html_string.encode(encoding='utf-8'), 'html.parser') for math_tag in soup.findAll(name='oppia-noninteractive-math'): if math_tag.has_attr('raw_latex-with-value'): # There was a case in prod where the attr value was empty. This was # dealt with manually in an earlier migration (states schema v34), # but we are not sure how it arose. We can't migrate those snapshots # manually, hence the addition of the logic here. After all # snapshots are migrated to states schema v42 (or above), this # 'if' branch will no longer be needed. if not math_tag['raw_latex-with-value']: math_tag.decompose() continue try: # The raw_latex attribute value should be enclosed in # double quotes(&quot;) and should be a valid unicode # string. raw_latex = ( json.loads(unescape_html(math_tag['raw_latex-with-value']))) normalized_raw_latex = ( objects.UnicodeString.normalize(raw_latex)) except Exception as e: logging.exception( 'Invalid raw_latex string found in the math tag : %s' % ( python_utils.UNICODE(e) ) ) python_utils.reraise_exception() math_content_dict = { 'raw_latex': normalized_raw_latex, 'svg_filename': '' } # Normalize and validate the value before adding to the math # tag. normalized_math_content_dict = ( objects.MathExpressionContent.normalize(math_content_dict)) # Add the new attribute math_expression_contents-with-value. math_tag['math_content-with-value'] = ( escape_html( json.dumps(normalized_math_content_dict, sort_keys=True))) # Delete the attribute raw_latex-with-value. del math_tag['raw_latex-with-value'] elif math_tag.has_attr('math_content-with-value'): pass else: # Invalid math tag with no proper attribute found. math_tag.decompose() # We need to replace the <br/> tags (if any) with <br> because for passing # the textangular migration tests we need to have only <br> tags. return python_utils.UNICODE(soup).replace('<br/>', '<br>')
def add_or_update_user_status(user_email, can_receive_email_updates): """Subscribes/unsubscribes an existing user or creates a new user with correct status in the mailchimp DB. NOTE: Callers should ensure that the user's corresponding UserEmailPreferencesModel.site_updates field is kept in sync. Args: user_email: str. Email ID of the user. Email is used to uniquely identify the user in the mailchimp DB. can_receive_email_updates: bool. Whether they want to be subscribed to the bulk email list or not. Returns: bool. Whether the user was successfully added to the db. (This will be False if the user was permanently deleted earlier and therefore cannot be added back.) Raises: Exception. Any error (other than the case where the user was permanently deleted earlier) raised by the mailchimp API. """ client = _get_mailchimp_class() subscriber_hash = _get_subscriber_hash(user_email) subscribed_mailchimp_data = { 'email_address': user_email, 'status': 'subscribed' } unsubscribed_mailchimp_data = { 'email_address': user_email, 'status': 'unsubscribed' } try: member_details = client.lists.members.get(feconf.MAILCHIMP_AUDIENCE_ID, subscriber_hash) # If member is already added to mailchimp list, we cannot permanently # delete a list member, since they cannot be programmatically added # back, so we change their status based on preference. if (can_receive_email_updates and member_details['status'] != 'subscribed'): client.lists.members.update(feconf.MAILCHIMP_AUDIENCE_ID, subscriber_hash, subscribed_mailchimp_data) elif (not can_receive_email_updates and member_details['status'] == 'subscribed'): client.lists.members.update(feconf.MAILCHIMP_AUDIENCE_ID, subscriber_hash, unsubscribed_mailchimp_data) except mailchimpclient.MailChimpError as error: # This has to be done since the message can only be accessed from # MailChimpError by error.message in Python2, but this is deprecated in # Python3. # In Python3, the message can be accessed directly by KeyError # (https://github.com/VingtCinq/python-mailchimp/pull/65), so as a # workaround for Python2, the 'message' attribute is obtained by # str() and then it is converted to dict. This works in Python3 as well. error_message = ast.literal_eval(python_utils.UNICODE(error)) # Error 404 corresponds to "User does not exist". if error_message['status'] == 404: if can_receive_email_updates: user_creation_successful = _create_user_in_mailchimp_db( user_email) if not user_creation_successful: return False else: raise Exception(error_message['detail']) return True
def post(self): """Handles POST requests.""" action = self.normalized_payload.get('action') try: result = {} if action == 'reload_exploration': exploration_id = self.normalized_payload.get('exploration_id') self._reload_exploration(exploration_id) elif action == 'reload_collection': collection_id = self.normalized_payload.get('collection_id') self._reload_collection(collection_id) elif action == 'generate_dummy_explorations': num_dummy_exps_to_generate = self.normalized_payload.get( 'num_dummy_exps_to_generate') num_dummy_exps_to_publish = self.normalized_payload.get( 'num_dummy_exps_to_publish') if num_dummy_exps_to_generate < num_dummy_exps_to_publish: raise self.InvalidInputException( 'Generate count cannot be less than publish count') else: self._generate_dummy_explorations( num_dummy_exps_to_generate, num_dummy_exps_to_publish) elif action == 'clear_search_index': search_services.clear_collection_search_index() search_services.clear_exploration_search_index() elif action == 'generate_dummy_new_structures_data': self._load_dummy_new_structures_data() elif action == 'generate_dummy_new_skill_data': self._generate_dummy_skill_and_questions() elif action == 'save_config_properties': new_config_property_values = self.normalized_payload.get( 'new_config_property_values') logging.info('[ADMIN] %s saved config property values: %s' % (self.user_id, new_config_property_values)) for (name, value) in new_config_property_values.items(): config_services.set_property(self.user_id, name, value) elif action == 'revert_config_property': config_property_id = self.normalized_payload.get( 'config_property_id') logging.info('[ADMIN] %s reverted config property: %s' % (self.user_id, config_property_id)) config_services.revert_property(self.user_id, config_property_id) elif action == 'upload_topic_similarities': data = self.normalized_payload.get('data') recommendations_services.update_topic_similarities(data) elif action == 'regenerate_topic_related_opportunities': topic_id = self.normalized_payload.get('topic_id') opportunities_count = ( opportunity_services. regenerate_opportunities_related_to_topic( topic_id, delete_existing_opportunities=True)) result = {'opportunities_count': opportunities_count} elif action == 'update_feature_flag_rules': feature_name = self.normalized_payload.get('feature_name') new_rule_dicts = self.normalized_payload.get('new_rules') commit_message = self.normalized_payload.get('commit_message') try: feature_services.update_feature_flag_rules( feature_name, self.user_id, commit_message, new_rule_dicts) except (utils.ValidationError, feature_services.FeatureFlagNotFoundException) as e: raise self.InvalidInputException(e) logging.info('[ADMIN] %s updated feature %s with new rules: ' '%s.' % (self.user_id, feature_name, new_rule_dicts)) self.render_json(result) except Exception as e: logging.exception('[ADMIN] %s', e) self.render_json({'error': python_utils.UNICODE(e)}) python_utils.reraise_exception()