def get_db_prep_save(self, value):
if value is not None and not isinstance(value, basestring):
if isinstance(value, dict):
value = b64encode(compress(wbin.serialize(value)))
else:
raise TypeError('This field can only store dictionaries. Use PickledObjectField to store a wide(r) range of data types.')
return value
def save(self, lang):
"""
Save the accumulated data into DB
"""
from itertools import islice
iterator = self.iteritems()
step = 100000
for _ in xrange(0, len(self), step):
data = [{'username': user,
'lang': lang,
'normal_edits': d.normal_count,
'namespace_edits': b64encode(
compress(serialize(d.namespace_count.tolist())))
if hasattr(d, 'namespace_count') else None,
'first_edit': d.first_time,
'last_edit': d.last_time,
'comments_count': d.comment_count,
'comments_avg': d.comment_length,
'minor': d.minor,
'welcome': d.welcome,
'npov': d.npov,
'please': d.please,
'thanks': d.thanks,
'revert': d.revert
}
for user, d in islice(iterator, step)]
self.connection.execute(self.insert, data)
def get_db_prep_save(self, value):
if value is not None and not isinstance(value, basestring):
if isinstance(value, dict):
value = b64encode(compress(wbin.serialize(value)))
else:
raise TypeError('This field can only store dictionaries. '
'Use PickledObjectField to store a wide(r) '
'range of data types.')
return value
def flush(self):
data = [{'title': page['title'],
'lang': self.lang,
'talk': (page['type'] == 'talk'),
'data': b64encode(compress(serialize(page['counter']))),
'desired': page['desired'],
'total_editors': page['total_editors'],
'bot_editors': page['bot_editors'],
'anonymous_editors': page['anon_editors']
} for page in self.queue]
self.connection.execute(self.insert, data)
self.queue = []
def flush(self):
data = [{
'title': page['title'],
'lang': self.lang,
'talk': (page['type'] == 'talk'),
'data': b64encode(compress(serialize(page['counter']))),
'desired': page['desired'],
'total_editors': page['total_editors'],
'bot_editors': page['bot_editors'],
'anonymous_editors': page['anon_editors']
} for page in self.queue]
self.connection.execute(self.insert, data)
self.queue = []
def flush(self):
data = [
{
"title": page["title"],
"lang": self.lang,
"talk": (page["type"] == "talk"),
"data": b64encode(compress(serialize(page["counter"]))),
"desired": page["desired"],
"total_editors": page["total_editors"],
"bot_editors": page["bot_editors"],
"anonymous_editors": page["anon_editors"],
}
for page in self.queue
]
self.connection.execute(self.insert, data)
self.queue = []
def save(self, lang):
"""
Save the accumulated data into DB
"""
from itertools import islice
iterator = self.iteritems()
step = 100000
for _ in xrange(0, len(self), step):
data = [{
'username':
user,
'lang':
lang,
'normal_edits':
d.normal_count,
'namespace_edits':
b64encode(compress(serialize(d.namespace_count.tolist())))
if hasattr(d, 'namespace_count') else None,
'first_edit':
d.first_time,
'last_edit':
d.last_time,
'comments_count':
d.comment_count,
'comments_avg':
d.comment_length,
'minor':
d.minor,
'welcome':
d.welcome,
'npov':
d.npov,
'please':
d.please,
'thanks':
d.thanks,
'revert':
d.revert
} for user, d in islice(iterator, step)]
self.connection.execute(self.insert, data)
def _serialize(self, o):
if self._ser_func is not None:
return wbin.serialize(o, self._ser_func, (o,), self._ser_size)
else:
return wbin.serialize(o)
print len(pickle.dumps(d, 1))
start = time.time()
for i in xrange(1000000):
pickle.dumps(d, 1)
print 'done:', time.time() - start
print 'Starting pickle 2 ...'
print pickle.dumps(d, 2)
print len(pickle.dumps(d, 2))
start = time.time()
for i in xrange(1000000):
pickle.dumps(d, 2)
print 'done:', time.time() - start
print 'Starting wbin...'
print wbin.serialize(d)
print len(wbin.serialize(d))
start = time.time()
for i in xrange(1000000):
wbin.serialize(d)
print 'done:', time.time() - start
#print 'Starting demjson...'
#print demjson.encode(d)
#print len(demjson.encode(d))
#start = time.time()
#for i in xrange(1000000):
# demjson.encode(d)
#print 'done:', time.time() - start
