def app_xml(app_id):
    try:
        app = db.session.query(App).filter_by(unique_id=app_id, composer = 'adapt').first()
        if app is None:
            return "App not found", 404

        data = json.loads(app.data)
        if data['adaptor_type'] != 'jsconfig':
            return "App deprecated", 404

        url = data['url'].strip()
        contents = get_cached_session().get(url).text

        # If the user hasn't clicked on "Save" yet, do not replace configuration script
        if data.get('configuration_name'):
            contents = replace_default_configuration_script(contents,
                                                            url_for('.configuration', app_id=app_id, _external=True))

        contents = inject_absolute_urls(contents, url)
        xmldoc = minidom.parseString(contents)
        inject_original_url_in_xmldoc(xmldoc, url)
        inject_absolute_locales_in_xmldoc(xmldoc, url)
        contents = xmldoc.toprettyxml()
    except Exception as e:
        traceback.print_exc()
        # TODO: some bootstrap magic
        return "Could not convert the application. %s" % str(e)
    else:
        return Response(contents, mimetype='text/xml')
Exemple #2
0
def get_contents(lang, trials=2):
    if lang == 'en':
        resource_id = '560410f1f0e1b09f6c8117ec'
        requests = get_cached_session()
        request_url = "http://graasp.eu/resources/{0}/raw".format(resource_id)
        try:
            r = requests.get(request_url, timeout=(10,10))
            r.raise_for_status()
        except Exception:
            raise TimeoutError("Timeout")

        try:
            return json.JSONDecoder(object_pairs_hook=OrderedDict).decode(r.text)
        except ValueError as ve:
            if len(r.text) == 0:
                if trials == 0:
                    raise ValueError("{}: {} returned empty result!".format(ve, request_url))
                return get_contents(lang, trials-1)
                
            if len(r.text) >= 20:
                response = '{!r}...'.format(r.text[:20])
            else:
                response = r.text
            raise ValueError("{}: {}: {!r}".format(ve, request_url, response))
    else:
        return None
Exemple #3
0
def app_xml(app_id):
    try:
        app = db.session.query(App).filter_by(unique_id=app_id,
                                              composer='adapt').first()
        if app is None:
            return "App not found", 404

        data = json.loads(app.data)
        if data['adaptor_type'] != 'jsconfig':
            return "App deprecated", 404

        url = data['url'].strip()
        contents = get_cached_session().get(url).text

        # If the user hasn't clicked on "Save" yet, do not replace configuration script
        if data.get('configuration_name'):
            contents = replace_default_configuration_script(
                contents,
                url_for('.configuration', app_id=app_id, _external=True))

        contents = inject_absolute_urls(contents, url)
        xmldoc = minidom.parseString(contents)
        inject_original_url_in_xmldoc(xmldoc, url)
        inject_absolute_locales_in_xmldoc(xmldoc, url)
        contents = xmldoc.toprettyxml()
    except Exception as e:
        traceback.print_exc()
        # TODO: some bootstrap magic
        return "Could not convert the application. %s" % str(e)
    else:
        return Response(contents, mimetype='text/xml')
Exemple #4
0
def get_contents(lang, trials=2):
    if lang == 'en':
        resource_id = '5a2ee559256f60731402f4da'
        requests = get_cached_session()
        request_url = "http://graasp.eu/resources/{0}/raw".format(resource_id)
        try:
            r = requests.get(request_url, timeout=(10,10))
            r.raise_for_status()
        except Exception:
            raise TimeoutError("Timeout")

        try:
            return json.JSONDecoder(object_pairs_hook=OrderedDict).decode(r.text)
        except ValueError as ve:
            if len(r.text) == 0:
                if trials == 0:
                    raise ValueError("{}: {} returned empty result!".format(ve, request_url))
                return get_contents(lang, trials-1)

            if len(r.text) >= 20:
                response = '{!r}...'.format(r.text[:20])
            else:
                response = r.text
            raise ValueError("{}: {}: {!r}".format(ve, request_url, response))
    else:
        return None
 def __init__(self, app_url, force_reload):
     threading.Thread.__init__(self)
     self.cached_requests = get_cached_session()
     self.app_url = app_url
     self.force_reload = force_reload
     self.finished = False
     self.failing = False
     self.metadata_information = None
Exemple #6
0
def get_languages():
    return ['en']
    requests = get_cached_session()
    languages = []
    for line in requests.get(
            "http://go-lab.gw.utwente.nl/production/common/languages/list.txt"
    ).text.splitlines():
        languages.append(line.split("_")[1])
    return languages
def synchronize_apps_no_cache():
    """Force obtaining the results and checking everything again to avoid inconsistences. This should be run once a day."""
    sync_id = start_synchronization()
    try:
        cached_requests = get_cached_session()
        synced_apps = _sync_golab_translations(cached_requests, force_reload = True)
        _sync_regular_apps(cached_requests, synced_apps, force_reload = True)
    finally:
        end_synchronization(sync_id)
Exemple #8
0
 def __init__(self, app_url, force_reload):
     threading.Thread.__init__(self)
     self.cached_requests = trutils.get_cached_session(
         caching=not force_reload)
     self.app_url = app_url
     self.force_reload = force_reload
     self.finished = False
     self.failing = False
     self.metadata_information = None
def synchronize_apps_cache():
    """Force obtaining the results and checking everything again to avoid inconsistences. 
    This can safely be run every few minutes, since most applications will be in the cache."""
    sync_id = start_synchronization()
    try:
        cached_requests = get_cached_session()
        synced_apps = _sync_golab_translations(cached_requests, force_reload = False)
        _sync_regular_apps(cached_requests, synced_apps, force_reload = False)
    finally:
        end_synchronization(sync_id)
def locale(language):
    requests = get_cached_session()
    if language not in get_languages():
        return "Language not found", 404

    # xml_response = requests.get('http://go-lab.gw.utwente.nl/production/commons/commons_en_ALL.xml')
    xml_response = requests.get('http://go-lab.gw.utwente.nl/production/commons/languages/common_{0}_ALL.xml'.format(language))
    response = make_response(get_text_from_response(xml_response))
    response.content_type = 'application/xml'
    return response
Exemple #11
0
def synchronize_apps_no_cache(source, single_app_url = None):
    """Force obtaining the results and checking everything again to avoid inconsistences. This should be run once a day."""
    sync_id = start_synchronization(source = source, cached = False, single_app_url = single_app_url)
    number = 0
    try:
        cached_requests = trutils.get_cached_session(caching = False)
        synced_apps = []
        all_golab_apps = _get_golab_translations(cached_requests)
        all_golab_apps.extend(get_other_apps())
        number += _sync_translations(cached_requests, "Go-Lab apps", synced_apps, all_golab_apps, force_reload = True, single_app_url = single_app_url)
        number += _sync_regular_apps(cached_requests, synced_apps, force_reload = True, single_app_url = single_app_url)
    finally:
        end_synchronization(sync_id, number)
Exemple #12
0
 def run(self):
     self.failing = False
     cached_requests = trutils.get_cached_session(caching = not self.force_reload)
     try:
         self.metadata_information = extract_metadata_information(self.app_url, self.preview_link, cached_requests, self.force_reload, app_format=self.app_format)
     except Exception:
         logger.warning("Error extracting information from %s" % self.app_url, exc_info = True)
         if DEBUG_VERBOSE:
             print("Error extracting information from %s" % self.app_url)
             traceback.print_exc()
         self.metadata_information = {}
         self.failing = True
     else:
         self.failing = self.metadata_information.get('failing', False)
     self.finished = True
Exemple #13
0
def locale(language):
    requests = get_cached_session()
    if language not in get_languages():
        return "Language not found", 404

    # xml_response = requests.get('http://go-lab.gw.utwente.nl/production/commons/commons_en_ALL.xml')
    try:
        xml_response = requests.get(
            'http://go-lab.gw.utwente.nl/production/common/languages/common_{0}_ALL.xml'
            .format(language),
            timeout=(10, 10))
    except Exception:
        return "Error requesting external resource", 502
    else:
        response = make_response(get_text_from_response(xml_response))
        response.content_type = 'application/xml'
        return response
Exemple #14
0
def synchronize_apps_no_cache(source, single_app_url=None):
    """Force obtaining the results and checking everything again to avoid inconsistences. This should be run once a day."""
    sync_id = start_synchronization(source=source,
                                    cached=False,
                                    single_app_url=single_app_url)
    number = 0
    try:
        cached_requests = trutils.get_cached_session(caching=False)
        synced_apps = []
        all_golab_apps = _get_golab_translations(cached_requests)
        all_golab_apps.extend(get_other_apps())
        number += _sync_translations(cached_requests,
                                     "Go-Lab apps",
                                     synced_apps,
                                     all_golab_apps,
                                     force_reload=True,
                                     single_app_url=single_app_url)
        number += _sync_regular_apps(cached_requests,
                                     synced_apps,
                                     force_reload=True,
                                     single_app_url=single_app_url)
    finally:
        end_synchronization(sync_id, number)
Exemple #15
0
 def run(self):
     self.failing = False
     cached_requests = trutils.get_cached_session(
         caching=not self.force_reload)
     try:
         self.metadata_information = extract_metadata_information(
             self.app_url,
             self.preview_link,
             cached_requests,
             self.force_reload,
             app_format=self.app_format)
     except Exception:
         logger.warning("Error extracting information from %s" %
                        self.app_url,
                        exc_info=True)
         if DEBUG_VERBOSE:
             print("Error extracting information from %s" % self.app_url)
             traceback.print_exc()
         self.metadata_information = {}
         self.failing = True
     else:
         self.failing = self.metadata_information.get('failing', False)
     self.finished = True
Exemple #16
0
def extract_metadata_information(app_url, preview_link, cached_requests = None, force_reload = False, app_format=None):
    if cached_requests is None:
        cached_requests = get_cached_session()

    if not app_format or app_format == 'opensocial':
        app_information = _extract_information_opensocial(app_url, cached_requests)
    elif app_format == 'html':
        app_information = _extract_information_html(app_url, cached_requests)
    else:
        raise TranslatorError('Invalid app format: {}'.format(app_format))

    original_translations = {}
    original_translation_urls = {}
    default_translations = {}
    default_translation_url = None
    default_metadata = {}

    if len(app_information.locales) == 0:
        translatable = False
    else:
        translatable = True
        default_locale = None
        for locale in app_information.locales:
            lang = locale['lang']
            country = locale['country']
            messages_url = locale['messages']
            if lang and messages_url and lang.lower() != 'all':
                if country:
                    lang = u'{}_{}'.format(lang, country)
                elif len(lang) == 2:
                    lang = u'%s_ALL' % lang

                if lang.lower() == 'zh_all':
                    lang = 'zh_CN'
                if '_' in lang and lang.split('_')[0].lower() == lang.split('_')[1].lower():
                    # es_ES is es_ALL, fr_FR is fr_ALL
                    lang = '{}_ALL'.format(lang.split('_'))

                try:
                    absolute_url, messages, metadata, locale_contents = _retrieve_messages_from_relative_url(app_url, messages_url, cached_requests)
                except TranslatorError as e:
                    logging.warning(u"Could not load %s translation for app URL: %s Reason: %s" % (lang, app_url, e), exc_info = True)
                    continue
                else:
                    new_messages = {}
                    if messages:
                        for key, value in messages.iteritems():
                            new_messages[key] = value['text']
                    original_translations[lang] = new_messages
                    original_translation_urls[lang] = absolute_url

            if (lang is None or lang.lower() == 'all') and messages_url:
                # Process this later. This way we can force we get the results for the default translation
                default_locale = locale

        if default_locale is not None:
            messages_url = default_locale['messages']
            absolute_url, messages, metadata, locale_contents = _retrieve_messages_from_relative_url(app_url, messages_url, cached_requests)
            default_translations = messages
            default_translation_url = absolute_url
            default_metadata = metadata

            # No English? Default is always English!
            if 'en_ALL' not in original_translations:
                lang = 'en_ALL'
                new_messages = {}
                if messages:
                    for key, value in messages.iteritems():
                        new_messages[key] = value['text']

                original_translations[lang] = new_messages
                original_translation_urls[lang] = absolute_url

    check_urls = app_information.check_urls
    if preview_link:
        check_urls.append(preview_link)

    metadata = {
        'translatable' : translatable,
        'check_urls' : check_urls,
        'uses_proxy': app_information.uses_proxy,
        'adaptable' : False, # Not supported anymore (all 0 in the database right now)
        'original_translations' : original_translations,
        'original_translation_urls' : original_translation_urls,
        'default_translations' : default_translations,
        'default_translation_url' : default_translation_url,
        'default_metadata' : default_metadata,
    }

    translation_hash, serialized = _calculate_translations_hash(original_translations, default_translations)
    metadata['translation_hash'] = translation_hash
    metadata['check_urls_hash'] = unicode(zlib.crc32(json.dumps(sorted(check_urls))))
    return metadata
Exemple #17
0
def extract_metadata_information(app_url,
                                 preview_link,
                                 cached_requests=None,
                                 force_reload=False,
                                 app_format=None):
    if cached_requests is None:
        cached_requests = get_cached_session()

    if not app_format or app_format == 'opensocial':
        app_information = _extract_information_opensocial(
            app_url, cached_requests)
    elif app_format == 'html':
        app_information = _extract_information_html(app_url, cached_requests)
    else:
        raise TranslatorError('Invalid app format: {}'.format(app_format))

    original_translations = {}
    original_translation_urls = {}
    default_translations = {}
    default_translation_url = None
    default_metadata = {}

    if len(app_information.locales) == 0:
        translatable = False
    else:
        translatable = True
        default_locale = None
        for locale in app_information.locales:
            lang = locale['lang']
            country = locale['country']
            messages_url = locale['messages']
            if lang and messages_url and lang.lower() != 'all':
                if country:
                    lang = u'{}_{}'.format(lang, country)
                elif len(lang) == 2:
                    lang = u'%s_ALL' % lang

                if lang.lower() == 'zh_all':
                    lang = 'zh_CN'
                if '_' in lang and lang.split('_')[0].lower() == lang.split(
                        '_')[1].lower():
                    # es_ES is es_ALL, fr_FR is fr_ALL
                    lang = '{}_ALL'.format(lang.split('_'))

                try:
                    absolute_url, messages, metadata, locale_contents = _retrieve_messages_from_relative_url(
                        app_url, messages_url, cached_requests)
                except TranslatorError as e:
                    logging.warning(
                        u"Could not load %s translation for app URL: %s Reason: %s"
                        % (lang, app_url, e),
                        exc_info=True)
                    continue
                else:
                    new_messages = {}
                    if messages:
                        for key, value in messages.iteritems():
                            new_messages[key] = value['text']
                    original_translations[lang] = new_messages
                    original_translation_urls[lang] = absolute_url

            if (lang is None or lang.lower() == 'all') and messages_url:
                # Process this later. This way we can force we get the results for the default translation
                default_locale = locale

        if default_locale is not None:
            messages_url = default_locale['messages']
            absolute_url, messages, metadata, locale_contents = _retrieve_messages_from_relative_url(
                app_url, messages_url, cached_requests)
            default_translations = messages
            default_translation_url = absolute_url
            default_metadata = metadata

            # No English? Default is always English!
            if 'en_ALL' not in original_translations:
                lang = 'en_ALL'
                new_messages = {}
                if messages:
                    for key, value in messages.iteritems():
                        new_messages[key] = value['text']

                original_translations[lang] = new_messages
                original_translation_urls[lang] = absolute_url

    check_urls = app_information.check_urls
    if preview_link:
        check_urls.append(preview_link)

    metadata = {
        'translatable': translatable,
        'check_urls': check_urls,
        'uses_proxy': app_information.uses_proxy,
        'offline': app_information.offline,
        'adaptable':
        False,  # Not supported anymore (all 0 in the database right now)
        'original_translations': original_translations,
        'original_translation_urls': original_translation_urls,
        'default_translations': default_translations,
        'default_translation_url': default_translation_url,
        'default_metadata': default_metadata,
    }

    translation_hash, serialized = _calculate_translations_hash(
        original_translations, default_translations)
    metadata['translation_hash'] = translation_hash
    metadata['check_urls_hash'] = unicode(
        zlib.crc32(json.dumps(sorted(check_urls))))
    return metadata
Exemple #18
0
def extract_local_translations_url(app_url, force_local_cache=False):
    redis_key = 'appcomposer:fast-cache:{}'.format(app_url)

    if force_local_cache:
        # Under some situations (e.g., updating a single message), it is better to have a cache
        # than contacting the foreign server. Only if requested, this method will try to check
        # in a local cache in Redis.
        cached = redis_store.get(redis_key)
        if cached:
            translation_url, original_messages, metadata = json.loads(cached)
            if metadata is not None:
                original_messages_loaded = json.loads(original_messages)
                metadata_loaded = json.loads(metadata)
                return translation_url, original_messages_loaded, metadata_loaded

    cached_requests = get_cached_session()

    repository_app = db.session.query(RepositoryApp).filter_by(
        url=app_url).first()
    if repository_app is not None:
        guessing = False
        app_format = repository_app.app_format
    else:
        guessing = True
        app_url_path = urlparse.urlparse(app_url).path
        if app_url_path.endswith('.xml'):
            app_format = 'opensocial'
        elif app_url_path.endswith('.html'):
            app_format = 'html'
        else:
            r = cached_requests.get(app_url)
            if 'xml' in (r.headers.get('Content-Type') or ''):
                app_format = 'opensocial'
            else:
                app_format = 'html'

    if app_format == 'opensocial' or not app_format:  # Default to opensocial
        try:
            app_information = _extract_information_opensocial(
                app_url, cached_requests)
        except TranslatorError as err:
            if not guessing:
                raise

            try:
                app_information = _extract_information_html(
                    app_url, cached_requests)
            except TranslatorError as err2:
                raise TranslatorError(
                    "Error trying both opensocial and html: {} / {}".format(
                        err, err2))
    else:
        try:
            app_information = _extract_information_html(
                app_url, cached_requests)
        except TranslatorError as err:
            try:
                app_information = _extract_information_opensocial(
                    app_url, cached_requests)
            except TranslatorError as err2:
                raise TranslatorError(
                    "Error trying both html and opensocial: {} / ".format(
                        err, err2))

    locales_without_lang = [
        locale for locale in app_information.locales
        if not locale['lang'] or locale['lang'].lower() == 'all'
    ]
    if not locales_without_lang:
        raise TranslatorError(
            "That application does not provide any default locale. The application has probably not been adopted to be translated."
        )

    relative_translation_url = locales_without_lang[0]['messages']
    if not relative_translation_url:
        raise TranslatorError("Default Locale not provided message attribute")

    absolute_translation_url, messages, metadata, contents = _retrieve_messages_from_relative_url(
        app_url, relative_translation_url, cached_requests)

    redis_value = json.dumps(
        [absolute_translation_url,
         json.dumps(messages),
         json.dumps(metadata)])
    redis_store.setex(name=redis_key, time=10 * 60,
                      value=redis_value)  # For 10 minutes
    return absolute_translation_url, messages, metadata
Exemple #19
0
def get_languages():
    requests = get_cached_session()
    languages = []
    for line in requests.get("http://go-lab.gw.utwente.nl/production/commons/languages/list.txt").text.splitlines():
        languages.append(line.split("_")[1])
    return languages
def index():
    requests = get_cached_session()
    languages = get_languages()
    response = make_response(render_template('graasp_i18n.xml', languages = languages, title = "Twente commons"))
    response.content_type = 'application/xml'
    return response
Exemple #21
0
def extract_local_translations_url(app_url, force_local_cache = False):
    redis_key = 'appcomposer:fast-cache:{}'.format(app_url)

    if force_local_cache:
        # Under some situations (e.g., updating a single message), it is better to have a cache
        # than contacting the foreign server. Only if requested, this method will try to check
        # in a local cache in Redis.
        cached = redis_store.get(redis_key)
        if cached:
            translation_url, original_messages, metadata = json.loads(cached)
            if metadata is not None:
                original_messages_loaded = json.loads(original_messages)
                metadata_loaded = json.loads(metadata)
                return translation_url, original_messages_loaded, metadata_loaded

    cached_requests = get_cached_session()

    repository_app = db.session.query(RepositoryApp).filter_by(url=app_url).first()
    if repository_app is not None:
        guessing = False
        app_format = repository_app.app_format
    else:
        guessing = True
        app_url_path = urlparse.urlparse(app_url).path
        if app_url_path.endswith('.xml'):
            app_format = 'opensocial'
        elif app_url_path.endswith('.html'):
            app_format = 'html'
        else:
            r = cached_requests.get(app_url)
            if 'xml' in (r.headers.get('Content-Type') or ''):
                app_format = 'opensocial'
            else:
                app_format = 'html'

    if app_format == 'opensocial' or not app_format: # Default to opensocial
        try:
            app_information = _extract_information_opensocial(app_url, cached_requests)
        except TranslatorError as err:
            if not guessing:
                raise

            try:
                app_information = _extract_information_html(app_url, cached_requests)
            except TranslatorError as err2:
                raise TranslatorError("Error trying both opensocial and html: {} / {}".format(err, err2))
    else:
        try:
            app_information = _extract_information_html(app_url, cached_requests)
        except TranslatorError as err:
            try:
                app_information = _extract_information_opensocial(app_url, cached_requests)
            except TranslatorError as err2:
                raise TranslatorError("Error trying both html and opensocial: {} / ".format(err, err2))


    locales_without_lang = [ locale for locale in app_information.locales if not locale['lang'] or locale['lang'].lower() == 'all' ]
    if not locales_without_lang:
        raise TranslatorError("That application does not provide any default locale. The application has probably not been adopted to be translated.")

    relative_translation_url = locales_without_lang[0]['messages']
    if not relative_translation_url:
        raise TranslatorError("Default Locale not provided message attribute")

    absolute_translation_url, messages, metadata, contents = _retrieve_messages_from_relative_url(app_url, relative_translation_url, cached_requests)

    redis_value = json.dumps([
        absolute_translation_url,
        json.dumps(messages),
        json.dumps(metadata)
    ])
    redis_store.setex(name=redis_key, time=10 * 60, value=redis_value) # For 10 minutes
    return absolute_translation_url, messages, metadata