예제 #1
0
def update_network_dict():
    _remove_old_zoneinfo()
    _update_zoneinfo()
    load_network_conversions()

    d = {}

    # network timezones are stored on github pages
    url = 'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/network_timezones.txt'

    url_data = helpers.getURL(url)
    if url_data is None:
        # When urlData is None, trouble connecting to github
        logger.log(u'Updating network timezones failed, this can happen from time to time. URL: %s' % url, logger.WARNING)
        load_network_dict()
        return

    try:
        for line in url_data.splitlines():
            (key, val) = line.decode('utf-8').strip().rsplit(u':', 1)
            if key is None or val is None:
                continue
            d[key] = val
    except (IOError, OSError):
        pass

    my_db = db.DBConnection('cache.db')

    # load current network timezones
    old_d = dict(my_db.select('SELECT * FROM network_timezones'))

    # list of sql commands to update the network_timezones table
    cl = []
    for cur_d, cur_t in iteritems(d):
        h_k = cur_d in old_d
        if h_k and cur_t != old_d[cur_d]:
            # update old record
            cl.append(
                ['UPDATE network_timezones SET network_name=?, timezone=? WHERE network_name=?', [cur_d, cur_t, cur_d]])
        elif not h_k:
            # add new record
            cl.append(['INSERT INTO network_timezones (network_name, timezone) VALUES (?,?)', [cur_d, cur_t]])
        if h_k:
            del old_d[cur_d]

    # remove deleted records
    if len(old_d) > 0:
        old_items = list(va for va in old_d)
        cl.append(['DELETE FROM network_timezones WHERE network_name IN (%s)' % ','.join(['?'] * len(old_items)), old_items])

    # change all network timezone infos at once (much faster)
    if len(cl) > 0:
        my_db.mass_action(cl)
        load_network_dict()
예제 #2
0
def iter_fields(fields):
    """
    Iterate over fields.

    .. deprecated ::

      The addition of `~urllib3.fields.RequestField` makes this function
      obsolete. Instead, use :func:`iter_field_objects`, which returns
      `~urllib3.fields.RequestField` objects, instead.

    Supports list of (k, v) tuples and dicts.

    """
    if isinstance(fields, dict):
        return ((k, v) for k, v in six.iteritems(fields))

    return ((k, v) for k, v in fields)
예제 #3
0
def iter_field_objects(fields):
    """
    Iterate over fields.

    Supports list of (k, v) tuples and dicts, and lists of
    :class:`~urllib3.fields.RequestField`.

    """
    if isinstance(fields, dict):
        i = six.iteritems(fields)
    else:
        i = iter(fields)

    for field in i:
      if isinstance(field, RequestField):
        yield field
      else:
        yield RequestField.from_tuples(*field)
예제 #4
0
def update_network_dict():
    _remove_old_zoneinfo()
    _update_zoneinfo()
    load_network_conversions()

    d = {}

    # network timezones are stored on github pages
    url = 'https://raw.githubusercontent.com/Prinz23/sb_network_timezones/master/network_timezones.txt'

    url_data = helpers.getURL(url)
    if url_data is None:
        # When urlData is None, trouble connecting to github
        logger.log(
            u'Updating network timezones failed, this can happen from time to time. URL: %s'
            % url, logger.WARNING)
        load_network_dict()
        return

    try:
        for line in url_data.splitlines():
            (key, val) = line.decode('utf-8').strip().rsplit(u':', 1)
            if key is None or val is None:
                continue
            d[key] = val
    except (IOError, OSError):
        pass

    my_db = db.DBConnection('cache.db')

    # load current network timezones
    old_d = dict(my_db.select('SELECT * FROM network_timezones'))

    # list of sql commands to update the network_timezones table
    cl = []
    for cur_d, cur_t in iteritems(d):
        h_k = cur_d in old_d
        if h_k and cur_t != old_d[cur_d]:
            # update old record
            cl.append([
                'UPDATE network_timezones SET network_name=?, timezone=? WHERE network_name=?',
                [cur_d, cur_t, cur_d]
            ])
        elif not h_k:
            # add new record
            cl.append([
                'INSERT INTO network_timezones (network_name, timezone) VALUES (?,?)',
                [cur_d, cur_t]
            ])
        if h_k:
            del old_d[cur_d]

    # remove deleted records
    if len(old_d) > 0:
        old_items = list(va for va in old_d)
        cl.append([
            'DELETE FROM network_timezones WHERE network_name IN (%s)' %
            ','.join(['?'] * len(old_items)), old_items
        ])

    # change all network timezone infos at once (much faster)
    if len(cl) > 0:
        my_db.mass_action(cl)
        load_network_dict()
예제 #5
0
파일: setting.py 프로젝트: lygntx/scrapyc
 def setdict(self, values, priority='project'):
     for name, value in six.iteritems(values):
         self.set(name, value, priority)
예제 #6
0
 def _repr_attributes(self):
     return ', '.join("%s=%r" % (key, value) for key, value in six.iteritems(self.attributes))