def get_db_prep_save(self, value): if value is not None and not isinstance(value, basestring): if isinstance(value, dict): value = b64encode(compress(wbin.serialize(value))) else: raise TypeError('This field can only store dictionaries. Use PickledObjectField to store a wide(r) range of data types.') return value
def save(self, lang): """ Save the accumulated data into DB """ from itertools import islice iterator = self.iteritems() step = 100000 for _ in xrange(0, len(self), step): data = [{'username': user, 'lang': lang, 'normal_edits': d.normal_count, 'namespace_edits': b64encode( compress(serialize(d.namespace_count.tolist()))) if hasattr(d, 'namespace_count') else None, 'first_edit': d.first_time, 'last_edit': d.last_time, 'comments_count': d.comment_count, 'comments_avg': d.comment_length, 'minor': d.minor, 'welcome': d.welcome, 'npov': d.npov, 'please': d.please, 'thanks': d.thanks, 'revert': d.revert } for user, d in islice(iterator, step)] self.connection.execute(self.insert, data)
def get_db_prep_save(self, value): if value is not None and not isinstance(value, basestring): if isinstance(value, dict): value = b64encode(compress(wbin.serialize(value))) else: raise TypeError('This field can only store dictionaries. ' 'Use PickledObjectField to store a wide(r) ' 'range of data types.') return value
def flush(self): data = [{'title': page['title'], 'lang': self.lang, 'talk': (page['type'] == 'talk'), 'data': b64encode(compress(serialize(page['counter']))), 'desired': page['desired'], 'total_editors': page['total_editors'], 'bot_editors': page['bot_editors'], 'anonymous_editors': page['anon_editors'] } for page in self.queue] self.connection.execute(self.insert, data) self.queue = []
def flush(self): data = [{ 'title': page['title'], 'lang': self.lang, 'talk': (page['type'] == 'talk'), 'data': b64encode(compress(serialize(page['counter']))), 'desired': page['desired'], 'total_editors': page['total_editors'], 'bot_editors': page['bot_editors'], 'anonymous_editors': page['anon_editors'] } for page in self.queue] self.connection.execute(self.insert, data) self.queue = []
def flush(self): data = [ { "title": page["title"], "lang": self.lang, "talk": (page["type"] == "talk"), "data": b64encode(compress(serialize(page["counter"]))), "desired": page["desired"], "total_editors": page["total_editors"], "bot_editors": page["bot_editors"], "anonymous_editors": page["anon_editors"], } for page in self.queue ] self.connection.execute(self.insert, data) self.queue = []
def save(self, lang): """ Save the accumulated data into DB """ from itertools import islice iterator = self.iteritems() step = 100000 for _ in xrange(0, len(self), step): data = [{ 'username': user, 'lang': lang, 'normal_edits': d.normal_count, 'namespace_edits': b64encode(compress(serialize(d.namespace_count.tolist()))) if hasattr(d, 'namespace_count') else None, 'first_edit': d.first_time, 'last_edit': d.last_time, 'comments_count': d.comment_count, 'comments_avg': d.comment_length, 'minor': d.minor, 'welcome': d.welcome, 'npov': d.npov, 'please': d.please, 'thanks': d.thanks, 'revert': d.revert } for user, d in islice(iterator, step)] self.connection.execute(self.insert, data)
def _serialize(self, o): if self._ser_func is not None: return wbin.serialize(o, self._ser_func, (o,), self._ser_size) else: return wbin.serialize(o)
print len(pickle.dumps(d, 1)) start = time.time() for i in xrange(1000000): pickle.dumps(d, 1) print 'done:', time.time() - start print 'Starting pickle 2 ...' print pickle.dumps(d, 2) print len(pickle.dumps(d, 2)) start = time.time() for i in xrange(1000000): pickle.dumps(d, 2) print 'done:', time.time() - start print 'Starting wbin...' print wbin.serialize(d) print len(wbin.serialize(d)) start = time.time() for i in xrange(1000000): wbin.serialize(d) print 'done:', time.time() - start #print 'Starting demjson...' #print demjson.encode(d) #print len(demjson.encode(d)) #start = time.time() #for i in xrange(1000000): # demjson.encode(d) #print 'done:', time.time() - start