def _analyze_diagnostic_info(diagnostic_info, reply_info):
    '''
    Determines what response should be sent based on `diagnostic_info` content.
    Returns a list of response IDs.
    Returns None if no response should be sent.
    '''

    logger.debug_log('_analyze_diagnostic_info: enter')

    responses = None

    AddressParts = collections.namedtuple('AddressParts',
                                          ['name', 'plus', 'domain'])

    # Get our local email address parts
    match = _address_splitter.match(config['emailUsername'])
    local_email_parts = AddressParts(*match.groups())

    # We'll break apart the "to" address (if applicable)
    to_parts = AddressParts(None, None, None)
    if reply_info.to:
        match = _address_splitter.match(reply_info.to)
        if not match:
            # Someone is messing with us
            raise ValueError('invalid email address: to-address match failed')
        to_parts = AddressParts(*match.groups())

    # TODO: more flexible rules, not so hard-coded
    if utils.coalesce(
            diagnostic_info,
        ['DiagnosticInfo', 'SystemInformation', 'isPlayStoreBuild']):
        # No download links in Play Store email
        responses = ['generic_info']
    elif to_parts.name == local_email_parts.name and \
         to_parts.domain == local_email_parts.domain:
        # E.g., [email protected], [email protected]
        responses = [
            'download_new_version_links',
            # Disabling attachment responses for now. Not sure if
            # it's a good idea. Note that it needs to be tested.
            # 'download_new_version_attachments',
        ]
    elif to_parts.domain == local_email_parts.domain:
        # E.g., *@psiphon.ca
        responses = ['generic_info']
    elif not reply_info.to and \
         utils.coalesce(diagnostic_info, ('Metadata', 'platform'), utils.string_types):
        # Windows S3 feedback
        responses = ['download_new_version_links']

    logger.debug_log('_analyze_diagnostic_info: exit: %s' % responses)

    return responses
def _get_email_reply_info(autoresponder_info):
    """Returns None if no reply info found, otherwise an instance of _ReplyInfo.
    Note that this function also validates the email address.
    """

    logger.debug_log('_get_email_reply_info: enter')

    email_info = autoresponder_info.get('email_info')
    diagnostic_info = autoresponder_info.get('diagnostic_info')
    reply_info = None

    if email_info:
        reply_info = _ReplyInfo(email_info.get('address'),
                                email_info.get('message_id'),
                                email_info.get('subject'),
                                email_info.get('to'))
    elif utils.coalesce(diagnostic_info, 'EmailInfo', required_types=dict):
        reply_info = _ReplyInfo(diagnostic_info['EmailInfo'].get('address'),
                                diagnostic_info['EmailInfo'].get('message_id'),
                                diagnostic_info['EmailInfo'].get('subject'),
                                diagnostic_info['EmailInfo'].get('to'))
    elif utils.coalesce(diagnostic_info, 'Feedback', required_types=dict):
        reply_info = _ReplyInfo(diagnostic_info['Feedback'].get('email'), None,
                                None, None)

    if not reply_info:
        logger.debug_log('_get_email_reply_info: no/bad reply_info, exiting')
        return None

    # Sometimes the recorded address looks like "<*****@*****.**>"
    if reply_info.address:
        reply_info.address = reply_info.address.strip('<>')

    if not reply_info.address:
        logger.debug_log(
            '_get_email_reply_info: no/bad reply_info.address, exiting')
        return None

    if reply_info.to:
        reply_info.to = reply_info.to.strip('<>')

    validator = email_validator.EmailValidator(fix=True, lookup_dns='mx')
    try:
        fixed_address = validator.validate_or_raise(reply_info.address)
        reply_info.address = fixed_address
    except:
        logger.debug_log(
            '_get_email_reply_info: address validator raised, exiting')
        return None

    logger.debug_log('_get_email_reply_info: exit')
    return reply_info
def _process_diagnostic_info(diagnostic_info):
    # We are not supporting old Android diagnostic data packages
    if coalesce(diagnostic_info, ('Metadata', 'platform')) == 'android' \
       and coalesce(diagnostic_info, ('Metadata', 'version'), 0, int) < 2:
        return

    session = None

    try:
        session = _new_session()

        diagnostic_data = DiagnosticData.create(diagnostic_info)
        session.add(diagnostic_data)

        # Get the ID for the new DiagnosticData object, so we can FK it.
        session.flush()

        for table_class in _table_class_registry:
            objs = table_class.create(diagnostic_info)

            if not objs:
                continue

            if not isinstance(objs, (tuple, list)):
                objs = (objs,)

            for obj in objs:
                setattr(obj, 'diagnostic_data_id', diagnostic_data.id)
                session.add(obj)

        _sanitize_session(session)
        session.commit()
    except sqlalchemy.exc.IntegrityError as e:
        if session:
            session.rollback()

        # Note that MySQL discards milliseconds ("fractional seconds": http://dev.mysql.com/doc/refman/5.0/en/datetime.html).
        # This means that the "last timestamp" will actually be a smaller value
        # than the timestamp of the last-inserted record. This means that there
        # will always(?) be an attempt to re-insert the last-inserted record,
        # which will result in an exception. We need to ignore it and carry on.
        expected = e.orig.args[1].startswith('Duplicate entry') and e.orig.args[1].endswith("for key 'obj_id'")

        if not expected:
          raise
    except:
        if session:
            session.rollback()
        raise
    finally:
        if session:
            session.close()
Beispiel #4
0
def _analyze_diagnostic_info(diagnostic_info, reply_info):
    '''
    Determines what response should be sent based on `diagnostic_info` content.
    Returns a list of response IDs.
    Returns None if no response should be sent.
    '''

    logger.debug_log('_analyze_diagnostic_info: enter')

    responses = None

    AddressParts = collections.namedtuple('AddressParts', ['name', 'plus', 'domain'])

    # Get our local email address parts
    match = _address_splitter.match(config['emailUsername'])
    local_email_parts = AddressParts(*match.groups())

    # We'll break apart the "to" address (if applicable)
    to_parts = AddressParts(None, None, None)
    if reply_info.to:
        match = _address_splitter.match(reply_info.to)
        if not match:
            # Someone is messing with us
            raise ValueError('invalid email address: to-address match failed')
        to_parts = AddressParts(*match.groups())

    # TODO: more flexible rules, not so hard-coded
    if utils.coalesce(diagnostic_info,
                      ['DiagnosticInfo', 'SystemInformation', 'isPlayStoreBuild']):
        # No download links in Play Store email
        responses = ['generic_info']
    elif to_parts.name == local_email_parts.name and \
         to_parts.domain == local_email_parts.domain:
        # E.g., [email protected], [email protected]
        responses = ['download_new_version_links',
                     # Disabling attachment responses for now. Not sure if
                     # it's a good idea. Note that it needs to be tested.
                     # 'download_new_version_attachments',
                    ]
    elif to_parts.domain == local_email_parts.domain:
        # E.g., *@psiphon.ca
        responses = ['generic_info']
    elif not reply_info.to and \
         utils.coalesce(diagnostic_info, ('Metadata', 'platform'), utils.string_types):
        # Windows S3 feedback
        responses = ['download_new_version_links']

    logger.debug_log('_analyze_diagnostic_info: exit: %s' % responses)

    return responses
Beispiel #5
0
def _get_email_reply_info(autoresponder_info):
    """Returns None if no reply info found, otherwise an instance of _ReplyInfo.
    Note that this function also validates the email address.
    """

    logger.debug_log('_get_email_reply_info: enter')

    email_info = autoresponder_info.get('email_info')
    diagnostic_info = autoresponder_info.get('diagnostic_info')
    reply_info = None

    if email_info:
        reply_info = _ReplyInfo(email_info.get('address'),
                                email_info.get('message_id'),
                                email_info.get('subject'),
                                email_info.get('to'))
    elif utils.coalesce(diagnostic_info, 'EmailInfo', required_types=dict):
        reply_info = _ReplyInfo(diagnostic_info['EmailInfo'].get('address'),
                                diagnostic_info['EmailInfo'].get('message_id'),
                                diagnostic_info['EmailInfo'].get('subject'),
                                diagnostic_info['EmailInfo'].get('to'))
    elif utils.coalesce(diagnostic_info, 'Feedback', required_types=dict):
        reply_info = _ReplyInfo(diagnostic_info['Feedback'].get('email'),
                                None, None, None)

    if not reply_info:
        logger.debug_log('_get_email_reply_info: no/bad reply_info, exiting')
        return None

    # Sometimes the recorded address looks like "<*****@*****.**>"
    if reply_info.address:
        reply_info.address = reply_info.address.strip('<>')

    if not reply_info.address:
        logger.debug_log('_get_email_reply_info: no/bad reply_info.address, exiting')
        return None

    if reply_info.to:
        reply_info.to = reply_info.to.strip('<>')

    validator = email_validator.EmailValidator(fix=True, lookup_dns='mx')
    try:
        fixed_address = validator.validate_or_raise(reply_info.address)
        reply_info.address = fixed_address
    except:
        logger.debug_log('_get_email_reply_info: address validator raised, exiting')
        return None

    logger.debug_log('_get_email_reply_info: exit')
    return reply_info
def _get_lang_id_from_diagnostic_info(diagnostic_info):
    '''
    Derive the lanague from `diagnostic_info` and return its ID/code.
    Returns `None` if the language can't be determined.
    '''

    logger.debug_log('_get_lang_id_from_diagnostic_info: enter')

    lang_id = None

    # There can be different -- and better or worse -- ways of determining the
    # user's language depending on platform, the type of feedback, and so on.

    # Windows, with feedback message
    lang_id = lang_id or utils.coalesce(
        diagnostic_info, ['Feedback', 'Message', 'text_lang_code'],
        required_types=utils.string_types)
    if lang_id and lang_id.find('INDETERMINATE') >= 0:
        lang_id = None

    # All Windows feedback
    lang_id = lang_id or utils.coalesce(diagnostic_info, [
        'DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LocaleInfo',
        'language_code'
    ],
                                        required_types=utils.string_types)

    # All Windows feedback
    lang_id = lang_id or utils.coalesce(diagnostic_info, [
        'DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LanguageInfo',
        'language_code'
    ],
                                        required_types=utils.string_types)
    # Android, from email
    lang_id = lang_id or utils.coalesce(
        diagnostic_info, ['EmailInfo', 'body', 'text_lang_code'],
        required_types=utils.string_types)
    if lang_id and lang_id.find('INDETERMINATE') >= 0:
        lang_id = None

    # Android, from system language
    lang_id = lang_id or utils.coalesce(
        diagnostic_info, ['DiagnosticInfo', 'SystemInformation', 'language'],
        required_types=utils.string_types)

    logger.debug_log(
        '_get_lang_id_from_diagnostic_info: exiting with lang_id=%s' % lang_id)

    return lang_id
    def create(cls, diagnostic_info):
        obj = cls()

        obj_id = coalesce(diagnostic_info, '_id')
        if not obj_id:
            # This shouldn't happen, since this field is added by MongoDB
            raise ValueError('diagnostic_info missing _id')

        obj.obj_id = str(obj_id)

        obj.datetime = coalesce(diagnostic_info, 'datetime')
        obj.platform = coalesce(diagnostic_info, ('Metadata', 'platform'))
        obj.version = coalesce(diagnostic_info, ('Metadata', 'version'))

        return obj
Beispiel #8
0
    def forward(self, x: np.ndarray, edge_index: np.ndarray):
        """

        Parameters
        ----------
        x
        edge_index

        Returns
        -------

        """
        x_len, x_dim = x.shape
        z = sps.csr_matrix(x)
        z = z @ self.w

        # make sparse adjacency matrix
        adj = coalesce(edge_index, None, x_len, x_len).tocsr()

        # add self loops
        idx = range(x_len)
        adj[idx, idx] = 2

        # compute message weights
        deg_inv_sqrt = sps.diags(
            np.array(adj.sum(axis=-1)).clip(1).reshape(-1)**-0.5)
        A = deg_inv_sqrt @ adj @ deg_inv_sqrt

        # do message passing
        z = A @ z

        if type(x) is np.ndarray:
            z = z.toarray()
        return z
    def create(cls, diagnostic_info):
        # This is for both platforms

        objs = []
        for entry in coalesce(diagnostic_info, ('DiagnosticInfo', 'DiagnosticHistory'), []):
            # DiagnosticHistory is a pretty free form set of data. The only type
            # of data that goes into it that we care about here are the
            # SelectedRegion entries.
            if coalesce(entry, 'msg') != 'SelectedRegion':
                continue

            obj = cls()
            obj.timestamp = coalesce(entry, 'timestamp')
            obj.regionCode = coalesce(entry, ('data', 'regionCode'))
            objs.append(obj)

        return objs
    def create(cls, diagnostic_info):
        if coalesce(diagnostic_info, ('Metadata', 'platform')) != 'windows':
            return None

        objs = []
        for sec_name, sec_type in (('AntiSpywareInfo', 'antispyware'), ('AntiVirusInfo', 'antivirus'), ('FirewallInfo', 'firewall')):
            for item in coalesce(diagnostic_info, ('DiagnosticInfo', 'SystemInformation', 'SecurityInfo', sec_name), []):
                obj = cls()
                obj.sec_type = sec_type
                obj.displayName = coalesce(item, 'displayName')

                data_version = coalesce(item, 'version')

                [setattr(obj, key, val) for key, val in coalesce(item, data_version, {}).iteritems()]

                objs.append(obj)

        return objs
Beispiel #11
0
def _get_lang_id_from_diagnostic_info(diagnostic_info):
    '''
    Derive the lanague from `diagnostic_info` and return its ID/code.
    Returns `None` if the language can't be determined.
    '''

    logger.debug_log('_get_lang_id_from_diagnostic_info: enter')

    lang_id = None

    # There can be different -- and better or worse -- ways of determining the
    # user's language depending on platform, the type of feedback, and so on.

    # Windows, with feedback message
    lang_id = lang_id or utils.coalesce(diagnostic_info,
                                        ['Feedback', 'Message', 'text_lang_code'],
                                        required_types=utils.string_types)
    if lang_id and lang_id.find('INDETERMINATE') >= 0:
        lang_id = None

    # All Windows feedback
    lang_id = lang_id or utils.coalesce(diagnostic_info,
                                        ['DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LocaleInfo', 'language_code'],
                                        required_types=utils.string_types)

    # All Windows feedback
    lang_id = lang_id or utils.coalesce(diagnostic_info,
                                        ['DiagnosticInfo', 'SystemInformation', 'OSInfo', 'LanguageInfo', 'language_code'],
                                        required_types=utils.string_types)
    # Android, from email
    lang_id = lang_id or utils.coalesce(diagnostic_info,
                                        ['EmailInfo', 'body', 'text_lang_code'],
                                        required_types=utils.string_types)
    if lang_id and lang_id.find('INDETERMINATE') >= 0:
        lang_id = None

    # Android, from system language
    lang_id = lang_id or utils.coalesce(diagnostic_info,
                                        ['DiagnosticInfo', 'SystemInformation', 'language'],
                                        required_types=utils.string_types)

    logger.debug_log('_get_lang_id_from_diagnostic_info: exiting with lang_id=%s' % lang_id)

    return lang_id
Beispiel #12
0
    def sort_by_weekdays_and_seasons(command: commands.Command) -> str:
        """Special sort that takes into accounts days and seasons"""

        name = command.name

        return utils.coalesce(
            lambda: str(utils.DAYS.index(name)),
            lambda: str(tuple(utils.SEASONS.values()).index(name)),
            ignored_exc=ValueError,
            default=name)
Beispiel #13
0
def report_training():
    data = read_yaml('../data/training')

    print("\n\n## {}\n".format(data['about']))

    training = data['entries']
    training.sort(key=lambda i: i['date'], reverse=True)

    for training in training:
        row = {
            'type': training['title'],
            'date': training['date'].strftime("%Y"),
            'summary': coalesce(training, 'subtitle', 'category'),
        }
        row['indent'] = tab_indent(len(row['type']))
        print("{type}{indent}{date}\t\n    {summary}".format(**row))
Beispiel #14
0
    def __merge_edges__(self, x, edge_index, edge_score):
        nodes_remaining = set(range(x.shape[0]))

        cluster = np.zeros(x.shape[0], dtype=edge_index.dtype)
        # argsort in ascending order and reverse
        edge_argsort = np.argsort(edge_score)[::-1]

        # Iterate through all edges
        i = 0
        new_edge_indices = []
        for edge_idx in edge_argsort.tolist():
            source = edge_index[0, edge_idx]
            target = edge_index[1, edge_idx]

            if source in nodes_remaining and target in nodes_remaining:
                # contract the edge if it is not incident to a chosen node
                new_edge_indices.append(edge_idx)

                cluster[source] = i
                nodes_remaining.remove(source)

                if source != target:
                    cluster[target] = i
                    nodes_remaining.remove(target)

                i += 1
            else:
                continue

        # The remaining nodes are simply kept.
        for node_idx in nodes_remaining:
            cluster[node_idx] = i
            i += 1

        # We compute the new features as an addition of the old ones.
        new_num_nodes = np.max(cluster) + 1
        new_x = np.zeros((new_num_nodes, x.shape[1]), dtype=x.dtype)
        new_x = scatter_add(new_x, cluster, x)

        N = new_x.shape[0]
        new_edge_index = coalesce(cluster[edge_index], None, N, N)
        new_edge_index = np.array(new_edge_index.nonzero(),
                                  dtype=edge_index.dtype)

        return new_x, new_edge_index
Beispiel #15
0
def report_awards():
    data = read_yaml('../data/awards')

    print("\n\n## {}\n".format(data['about']))

    awards = data['entries']
    awards.sort(key=lambda i: i['dated'], reverse=True)

    for award in awards:

        type = award['type']
        if award['type'] == 'Bonus Award':  # be more specific
            award['category']
        row = {
            'type': type,
            'date': award['dated'].strftime("%Y"),
            'indent': tab_indent(len(type), left_margin=32),
            'summary': coalesce(award, 'summary', 'team', 'category'),
        }
        print("{type} ({date}){indent}{summary}".format(**row))
    def create(cls, diagnostic_info):
        if coalesce(diagnostic_info, ('Metadata', 'platform')) != 'windows':
            return None

        obj = cls()

        base = coalesce(diagnostic_info, ('DiagnosticInfo', 'SystemInformation', 'OSInfo'))
        obj.os_name = coalesce(base, 'name')
        obj.os_version = coalesce(base, 'version')
        obj.os_architecture = coalesce(base, 'architecture')
        obj.os_servicePackMajor = coalesce(base, 'servicePackMajor')
        obj.os_servicePackMinor = coalesce(base, 'servicePackMinor')
        obj.os_freePhysicalMemoryKB = coalesce(base, 'freePhysicalMemoryKB')
        obj.os_freeVirtualMemoryKB = coalesce(base, 'freeVirtualMemoryKB')
        obj.os_language_lcid = coalesce(base, ('LanguageInfo', 'lcid_string'))
        obj.os_locale_lcid = coalesce(base, ('LocaleInfo', 'lcid_string'))

        # This is an array of country info, and we'll use the last one.
        # (For the hacky reason that in the ['CA', 'PR', 'US'] case we want 'US'.)
        country_code_info = coalesce(base, 'CountryCodeInfo', [None])
        obj.os_country_code = coalesce(country_code_info[-1], 'country_code')

        base = coalesce(diagnostic_info, ('DiagnosticInfo', 'SystemInformation', 'NetworkInfo', 'Current', 'Internet'))
        obj.net_current_internet_connected = coalesce(base, 'internetConnected')
        obj.net_current_internet_conn_modem = coalesce(base, 'internetConnectionModem')
        obj.net_current_internet_conn_configured = coalesce(base, 'internetConnectionConfigured')
        obj.net_current_internet_conn_lan = coalesce(base, 'internetConnectionLAN')
        obj.net_current_internet_conn_proxy = coalesce(base, 'internetConnectionProxy')
        obj.net_current_internet_conn_offline = coalesce(base, 'internetConnectionOffline')
        obj.net_current_internet_ras_installed = coalesce(base, 'internetRASInstalled')

        base = coalesce(diagnostic_info, ('DiagnosticInfo', 'SystemInformation', 'NetworkInfo', 'Original', 'Internet'))
        obj.net_original_internet_connected = coalesce(base, 'internetConnected')
        obj.net_original_internet_conn_modem = coalesce(base, 'internetConnectionModem')
        obj.net_original_internet_conn_configured = coalesce(base, 'internetConnectionConfigured')
        obj.net_original_internet_conn_lan = coalesce(base, 'internetConnectionLAN')
        obj.net_original_internet_conn_proxy = coalesce(base, 'internetConnectionProxy')
        obj.net_original_internet_conn_offline = coalesce(base, 'internetConnectionOffline')
        obj.net_original_internet_ras_installed = coalesce(base, 'internetRASInstalled')

        base = coalesce(diagnostic_info, ('DiagnosticInfo', 'SystemInformation', 'NetworkInfo', 'Original', 'Proxy'), [])
        # There's an array of proxy info, typically one per network connection.
        # We don't need to export all of them, so we'll take the one that doesn't
        # have a named network connection.
        proxy = filter(lambda p: not coalesce(p, 'connectionName'), base)
        proxy = proxy[0] if proxy else (base[0] if base else None)
        obj.net_original_proxy_flags = coalesce(proxy, 'flags')
        obj.net_original_proxy_address = coalesce(proxy, 'proxy')
        obj.net_original_proxy_bypass = coalesce(proxy, 'bypass')
        obj.net_original_proxy_connectionName = coalesce(proxy, 'connectionName')

        base = coalesce(diagnostic_info, ('DiagnosticInfo', 'SystemInformation', 'Misc'))
        obj.misc_slowMachine = coalesce(base, 'slowMachine')
        obj.misc_mideastEnabled = coalesce(base, 'mideastEnabled')

        base = coalesce(diagnostic_info, ('DiagnosticInfo', 'SystemInformation', 'UserInfo'))
        obj.user_group_users = coalesce(base, 'inUsersGroup')
        obj.user_group_power = coalesce(base, 'inPowerUsersGroup')
        obj.user_group_guest = coalesce(base, 'inGuestsGroup')
        obj.user_group_admin = coalesce(base, 'inAdminsGroup')

        base = coalesce(diagnostic_info, ('DiagnosticInfo', 'SystemInformation', 'PsiphonInfo'))
        obj.psiphon_info_propagationChannelID = coalesce(base, 'PROPAGATION_CHANNEL_ID')
        obj.psiphon_info_sponsorID = coalesce(base, 'SPONSOR_ID')
        obj.psiphon_info_clientVersion = coalesce(base, 'CLIENT_VERSION')
        obj.psiphon_info_transport = coalesce(base, 'selectedTransport')
        obj.psiphon_info_splitTunnel = coalesce(base, 'splitTunnel')

        return obj
Beispiel #17
0
def _get_response_content(response_id, diagnostic_info):
    """Gets the response for the given response_id. diagnostic_info will be
    used to determine language and some content, but may be None.

    Returns a dict of the form:
        {
            subject: <subject text>,
            body_text: <body text>,
            body_html: <rich html body>,
            attachments: <attachments list, may be None>
        }

    Returns None if no response content can be derived.
    """

    logger.debug_log('_get_response_content: enter')

    sponsor_name = utils.coalesce(diagnostic_info,
                                  ['DiagnosticInfo', 'SystemInformation', 'PsiphonInfo', 'SPONSOR_ID'],
                                  required_types=utils.string_types)

    prop_channel_name = utils.coalesce(diagnostic_info,
                                       ['DiagnosticInfo', 'SystemInformation', 'PsiphonInfo', 'PROPAGATION_CHANNEL_ID'],
                                       required_types=utils.string_types)

    # Use default values if we couldn't get good user-specific values
    sponsor_name = sponsor_name or config['defaultSponsorName']
    prop_channel_name = prop_channel_name or config['defaultPropagationChannelName']

    lang_id = _get_lang_id_from_diagnostic_info(diagnostic_info)
    # lang_id may be None, if the language could not be determined

    # Read in all translations HTML
    response_translations = []
    for root, _, files in os.walk(_RESPONSES_DIR):
        for name in files:
            lang, ext = os.path.splitext(name)
            if ext != '.html':
                continue

            if lang == 'master':
                lang = 'en'

            with open(os.path.join(root, name)) as translation_file:
                translation = translation_file.read().decode('utf-8')

            # Strip leading and trailing whitespace so that we don't get extra
            # text elements in our BeautifulSoup
            translation = translation.strip()

            response_translations.append((lang, translation.strip()))

    # Reorder the array according to the detected language and _TOP_LANGS
    def lang_sorter(item):
        lang, _ = item
        rank = 999
        try:
            if lang == lang_id:
                rank = -1
            else:
                rank = _TOP_LANGS.index(lang)
        except ValueError:
            pass
        return rank

    response_translations.sort(key=lang_sorter)

    # Gather the info we'll need for formatting the email
    bucketname, email_address = psi_ops_helpers.get_bucket_name_and_email_address(sponsor_name, prop_channel_name)

    # Use default values if we couldn't get good user-specific values
    if not bucketname or not email_address:
        default_bucketname, default_email_address = \
            psi_ops_helpers.get_bucket_name_and_email_address(config['defaultSponsorName'],
                                                              config['defaultPropagationChannelName'])
        bucketname = bucketname or default_bucketname
        email_address = email_address or default_email_address

    # If, despite our best efforts, we still don't have a bucketname and
    # email address, just bail.
    if not bucketname or not email_address:
        logger.debug_log('_get_response_content: exiting due to no bucketname or address')
        return None

    # Collect the translations of the specific response we're sending

    subject = None
    bodies = []
    for lang_id, html in response_translations:
        soup = BeautifulSoup(html)
        if not subject:
            subject = soup.find(id='default_response_subject')
            if subject:
                # Strip outer element
                subject = u''.join(unicode(elem) for elem in subject.contents).strip()
        body = soup.find(id=response_id)
        if body:
            # Strip outer element
            body = u''.join(unicode(elem) for elem in body.contents).strip()

            # The user might be using a language for which there isn't a
            # download page. Fall back to English if that's the case.
            home_page_url = psi_ops_helpers.get_s3_bucket_home_page_url(
                bucketname,
                lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en')
            download_page_url = psi_ops_helpers.get_s3_bucket_download_page_url(
                bucketname,
                lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en')

            format_dict = {
                '0': email_address,
                '1': download_page_url,
                '2': home_page_url
            }
            body = unicode(body) % format_dict
            bodies.append(body)

    # Render the email body from the Mako template
    body_html = _render_email({
        'lang_id': lang_id,
        'response_id': response_id,
        'responses': bodies
    })

    # Get attachments.
    # This depends on which response we're returning.
    attachments = None
    if response_id == 'download_new_version_links':
        pass
    elif response_id == 'download_new_version_attachments':
        fp_windows = aws_helpers.get_s3_attachment('attachments',
                                                   bucketname,
                                                   psi_ops_helpers.DOWNLOAD_SITE_WINDOWS_BUILD_FILENAME)
        fp_android = aws_helpers.get_s3_attachment('attachments',
                                                   bucketname,
                                                   psi_ops_helpers.DOWNLOAD_SITE_ANDROID_BUILD_FILENAME)
        attachments = [(fp_windows, psi_ops_helpers.EMAIL_RESPONDER_WINDOWS_ATTACHMENT_FILENAME),
                       (fp_android, psi_ops_helpers.EMAIL_RESPONDER_ANDROID_ATTACHMENT_FILENAME)]
    else:
        pass

    logger.debug_log('_get_response_content: exit')

    return {
        'subject': subject,
        'body_text': _html_to_text(body_html),
        'body_html': body_html,
        'attachments': attachments
    }
Beispiel #18
0
def _get_response_content(response_id, diagnostic_info):
    """Gets the response for the given response_id. diagnostic_info will be
    used to determine language and some content, but may be None.

    Returns a dict of the form:
        {
            subject: <subject text>,
            body_text: <body text>,
            body_html: <rich html body>,
            attachments: <attachments list, may be None>
        }

    Returns None if no response content can be derived.
    """

    logger.debug_log('_get_response_content: enter')

    sponsor_name = utils.coalesce(diagnostic_info,
                                  ['DiagnosticInfo', 'SystemInformation', 'PsiphonInfo', 'SPONSOR_ID'],
                                  required_types=utils.string_types)

    prop_channel_name = utils.coalesce(diagnostic_info,
                                       ['DiagnosticInfo', 'SystemInformation', 'PsiphonInfo', 'PROPAGATION_CHANNEL_ID'],
                                       required_types=utils.string_types)

    # Use default values if we couldn't get good user-specific values
    sponsor_name = sponsor_name or config['defaultSponsorName']
    prop_channel_name = prop_channel_name or config['defaultPropagationChannelName']

    lang_id = _get_lang_id_from_diagnostic_info(diagnostic_info)
    # lang_id may be None, if the language could not be determined

    # Read in all translations HTML
    response_translations = []
    for root, _, files in os.walk(_RESPONSES_DIR):
        for name in files:
            lang, ext = os.path.splitext(name)
            if ext != '.html':
                continue

            if lang == 'master':
                lang = 'en'

            with open(os.path.join(root, name)) as translation_file:
                translation = translation_file.read().decode('utf-8')

            # Strip leading and trailing whitespace so that we don't get extra
            # text elements in our BeautifulSoup
            translation = translation.strip()

            response_translations.append((lang, translation.strip()))

    # Reorder the array according to the detected language and _TOP_LANGS
    def lang_sorter(item):
        lang, _ = item
        rank = 999
        try:
            if lang == lang_id:
                rank = -1
            else:
                rank = _TOP_LANGS.index(lang)
        except ValueError:
            pass
        return rank

    response_translations.sort(key=lang_sorter)

    # Gather the info we'll need for formatting the email
    bucketname, email_address = psi_ops_helpers.get_bucket_name_and_email_address(sponsor_name, prop_channel_name)

    # Use default values if we couldn't get good user-specific values
    if not bucketname or not email_address:
        default_bucketname, default_email_address = \
            psi_ops_helpers.get_bucket_name_and_email_address(config['defaultSponsorName'],
                                                              config['defaultPropagationChannelName'])
        bucketname = bucketname or default_bucketname
        email_address = email_address or default_email_address

    # If, despite our best efforts, we still don't have a bucketname and
    # email address, just bail.
    if not bucketname or not email_address:
        logger.debug_log('_get_response_content: exiting due to no bucketname or address')
        return None

    # Collect the translations of the specific response we're sending

    subject = None
    bodies = []
    for lang_id, html in response_translations:
        soup = BeautifulSoup(html)
        if not subject:
            subject = soup.find(id='default_response_subject')
            if subject:
                # Strip outer element
                subject = u''.join(unicode(elem) for elem in subject.contents).strip()
        body = soup.find(id=response_id)
        if body:
            # Strip outer element
            body = u''.join(unicode(elem) for elem in body.contents).strip()

            # The user might be using a language for which there isn't a
            # download page. Fall back to English if that's the case.
            home_page_url = psi_ops_helpers.get_s3_bucket_home_page_url(
                bucketname,
                lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en')
            download_page_url = psi_ops_helpers.get_s3_bucket_download_page_url(
                bucketname,
                lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en')
            faq_page_url = psi_ops_helpers.get_s3_bucket_faq_url(
                bucketname,
                lang_id if lang_id in psi_ops_helpers.WEBSITE_LANGS else 'en')

            # We're using numbers rather than more readable names here because
            # they're less likely to be accidentally modified by translators
            # (we think).
            format_dict = {
                '0': email_address,
                '1': download_page_url,
                '2': home_page_url,
                '3': faq_page_url
            }
            body = unicode(body) % format_dict
            bodies.append(body)

    # Render the email body from the Mako template
    body_html = _render_email({
        'lang_id': lang_id,
        'response_id': response_id,
        'responses': bodies
    })

    # Get attachments.
    # This depends on which response we're returning.
    attachments = None
    if response_id == 'download_new_version_links':
        pass
    elif response_id == 'download_new_version_attachments':
        fp_windows = aws_helpers.get_s3_attachment('attachments',
                                                   bucketname,
                                                   psi_ops_helpers.DOWNLOAD_SITE_WINDOWS_BUILD_FILENAME)
        fp_android = aws_helpers.get_s3_attachment('attachments',
                                                   bucketname,
                                                   psi_ops_helpers.DOWNLOAD_SITE_ANDROID_BUILD_FILENAME)
        attachments = [(fp_windows, psi_ops_helpers.EMAIL_RESPONDER_WINDOWS_ATTACHMENT_FILENAME),
                       (fp_android, psi_ops_helpers.EMAIL_RESPONDER_ANDROID_ATTACHMENT_FILENAME)]
    else:
        pass

    logger.debug_log('_get_response_content: exit')

    return {
        'subject': subject,
        'body_text': _html_to_text(body_html),
        'body_html': body_html,
        'attachments': attachments
    }
Beispiel #19
0
def _ensure_field_is_string(stringtype, data, fieldpath):
    prev_val = utils.coalesce(data, fieldpath)
    if prev_val is not None:
        utils.assign_value_to_obj_at_path(data, fieldpath,
                                          stringtype(prev_val))
    def create(cls, diagnostic_info):
        # Android feedback gets submitted to the individual servers
        if coalesce(diagnostic_info, ('Metadata', 'platform')) != 'windows':
            return None

        results = coalesce(diagnostic_info, ('Feedback', 'Survey', 'results'))

        if not results:
            return None

        connectivity = filter(lambda r: coalesce(r, 'title') == 'Connectivity', results)
        speed = filter(lambda r: coalesce(r, 'title') == 'Speed', results)
        compatibility = filter(lambda r: coalesce(r, 'title') == 'Compatibility', results)

        obj = cls()
        obj.connectivity = coalesce(connectivity[0], 'answer') if connectivity else None
        obj.speed = coalesce(speed[0], 'answer') if speed else None
        obj.compatibility = coalesce(compatibility[0], 'answer') if compatibility else None

        # If the user was never able to connect to a server, then this feedback
        # can't be considered to be for a particular server. But if the user
        # has been (or is) connected to a server, we'll use the one they last
        # connected to.
        diagnostic_history = coalesce(diagnostic_info, ('DiagnosticInfo', 'DiagnosticHistory'), [])
        connected_msg = 'ConnectedServer' if coalesce(diagnostic_info, ('Metadata', 'platform')) != 'windows' else 'ConnectingServer'
        connected_entries = [entry for entry in diagnostic_history if coalesce(entry, 'msg') == connected_msg]
        if connected_entries:
            obj.server_id = coalesce(connected_entries[-1], ('data', 'ipAddress'))

        return obj
    def create(cls, diagnostic_info):
        if coalesce(diagnostic_info, ('Metadata', 'platform')) != 'android':
            return None

        obj = cls()

        base = coalesce(diagnostic_info, ('DiagnosticInfo', 'SystemInformation'))
        obj.isRooted = coalesce(base, 'isRooted')
        obj.language = coalesce(base, 'language')
        obj.networkTypeName = coalesce(base, 'networkTypeName')
        obj.sys_build_tags = coalesce(base, ('Build', 'TAGS'))
        obj.sys_build_brand = coalesce(base, ('Build', 'BRAND'))
        obj.sys_build_version_release = coalesce(base, ('Build', 'VERSION__RELEASE'))
        obj.sys_build_version_codename = coalesce(base, ('Build', 'VERSION__CODENAME'))
        obj.sys_build_version_sdk = coalesce(base, ('Build', 'VERSION__SDK_INT'))
        obj.sys_build_cpu_abi = coalesce(base, ('Build', 'CPU_ABI'))
        obj.sys_build_model = coalesce(base, ('Build', 'MODEL'))
        obj.sys_build_manufacturer = coalesce(base, ('Build', 'MANUFACTURER'))
        obj.psiphon_info_sponsorID = coalesce(base, ('PsiphonInfo', 'SPONSOR_ID'))
        obj.psiphon_info_propagationChannelID = coalesce(base, ('PsiphonInfo', 'PROPAGATION_CHANNEL_ID'))
        obj.psiphon_info_clientVersion = coalesce(base, ('PsiphonInfo', 'CLIENT_VERSION'))

        BROWSER_ONLY_STATUS_ID = 'psiphon_running_browser_only'
        CONNECTION_STATUS_IDS = (BROWSER_ONLY_STATUS_ID, 'psiphon_running_whole_device')
        connection_log_ids = [coalesce(s, 'id') for s in coalesce(diagnostic_info,
                                                                  ('DiagnosticInfo', 'StatusHistory'),
                                                                  default_value=[],
                                                                  required_types=(list, tuple))
                              if coalesce(s, 'id') in CONNECTION_STATUS_IDS]
        if connection_log_ids:
            # The last connection log is the one we'll use as representative
            obj.browserOnly = connection_log_ids[-1] == BROWSER_ONLY_STATUS_ID

        return obj
Beispiel #22
0
def test_coalesce_should_return_none_if_no_non_empty_values():
    result = coalesce(*EMPTY_VALUES)
    assert result is None
Beispiel #23
0
def test_coalesce_should_return_first_non_empty_value():
    result = coalesce(*EMPTY_VALUES, 1)
    assert result == 1
Beispiel #24
0
def _ensure_field_is_string(stringtype, data, fieldpath):
    prev_val = utils.coalesce(data, fieldpath)
    if prev_val is not None:
        utils.assign_value_to_obj_at_path(data, fieldpath, stringtype(prev_val))
Beispiel #25
0
 def __init__(self, operators=None, allow_null=True):
     self.allow_null = allow_null
     self.operators = coalesce(operators, self.operators)