def by_id(self, id):
if len(id) == 32:
if sys.version_info[0] == 2:
id = id.decode('hex')
else:
id = codecs.decode(id, "hex_codec")
row = self.connection.get('SELECT * FROM entities WHERE id = %s', id)
return Entity.from_row(row, use_zlib=self.use_zlib) if row else None
def by_id(self, id):
if len(id) == 32:
if sys.version_info[0] == 2:
id = id.decode('hex')
else:
id = codecs.decode(id, "hex_codec")
row = self.connection.get('SELECT * FROM entities WHERE id = %s', id)
return Entity.from_row(row, use_zlib=self.use_zlib) if row else None
def _do_query(self, exprs, order_by, limit):
values = []
where_clause = []
for e in exprs:
if e.name not in self.properties:
raise ValueError('This index has no column named %r' %
(e.name, ))
expr_string, vals = e.build()
where_clause.append(expr_string)
values.extend(vals)
if self.table == 'entities':
# XXX: this is a bit hacky
q = 'SELECT * FROM entities WHERE ' + ' AND '.join(where_clause)
if order_by:
q += ' ORDER BY %s %s' % (order_by.name, order_by.order)
if limit:
q += ' LIMIT %d' % (limit, )
entity_rows = self.connection.query(q, *values)
else:
q = 'SELECT entity_id FROM %s' % self.table
if where_clause:
q += ' WHERE ' + ' AND '.join(where_clause)
if order_by:
q += ' ORDER BY %s %s' % (order_by.name, order_by.order)
if limit:
q += ' LIMIT %d' % (limit, )
rows = self.connection.query(q, *values)
if rows:
entity_ids = [r['entity_id'] for r in rows]
q = 'SELECT * FROM entities WHERE id IN ('
q += ', '.join('%s' for x in rows)
q += ')'
entity_rows = self.connection.query(q, *entity_ids)
else:
return []
if not order_by:
#sorted_entities = sorted(entity_rows, key=lambda x: x['updated'], reverse=True)
sorted_entities = sorted(entity_rows, key=lambda x: x['updated'])
else:
# XXX: this is O(n^2), bad
sorted_entities = []
for row_id in (row['entity_id'] for row in rows):
for e in entity_rows:
if e['id'] == row_id:
sorted_entities.append(e)
break
else:
assert False
return [
Entity.from_row(row, use_zlib=self.use_zlib)
for row in sorted_entities
]
def _do_query(self, exprs, order_by, limit):
values = []
where_clause = []
for e in exprs:
if e.name not in self.properties:
raise ValueError('This index has no column named %r' % (e.name,))
expr_string, vals = e.build()
where_clause.append(expr_string)
values.extend(vals)
if self.table == 'entities':
# XXX: this is a bit hacky
q = 'SELECT * FROM entities WHERE ' + ' AND '.join(where_clause)
if order_by:
q += ' ORDER BY %s %s' % (order_by.name, order_by.order)
if limit:
q += ' LIMIT %d' % (limit,)
entity_rows = self.connection.query(q, *values)
else:
q = 'SELECT entity_id FROM %s' % self.table
if where_clause:
q += ' WHERE ' + ' AND '.join(where_clause)
if order_by:
q += ' ORDER BY %s %s' % (order_by.name, order_by.order)
if limit:
q += ' LIMIT %d' % (limit,)
rows = self.connection.query(q, *values)
if rows:
entity_ids = [r['entity_id'] for r in rows]
q = 'SELECT * FROM entities WHERE id IN ('
q += ', '.join('%s' for x in rows)
q += ')'
entity_rows = self.connection.query(q, *entity_ids)
else:
return []
if not order_by:
#sorted_entities = sorted(entity_rows, key=lambda x: x['updated'], reverse=True)
sorted_entities = sorted(entity_rows, key=lambda x: x['updated'])
else:
# XXX: this is O(n^2), bad
sorted_entities = []
for row_id in (row['entity_id'] for row in rows):
for e in entity_rows:
if e['id'] == row_id:
sorted_entities.append(e)
break
else:
assert False
return [Entity.from_row(row, use_zlib=self.use_zlib) for row in sorted_entities]
def run(self):
rows_processed = 0
self.log.info('starting run loop')
try:
for row in self.row_iterator():
entity = Entity.from_row(row, use_zlib=self.use_zlib)
self.process_row(row, entity)
self.rows_processed += 1
self.last_id_processed = row['added_id']
except:
self.log.exception('exception during run loop!')
finally:
elapsed_time = time.time() - self.start_run
self.log.info('finished run loop, elapsed time = %1.2f seconds, processed %d rows, last added_id was %d' % (elapsed_time, self.rows_processed, self.last_id_processed))
def run(self):
rows_processed = 0
self.log.info('starting run loop')
try:
for row in self.row_iterator():
entity = Entity.from_row(row, use_zlib=self.use_zlib)
self.process_row(row, entity)
self.rows_processed += 1
self.last_id_processed = row['added_id']
except:
self.log.exception('exception during run loop!')
finally:
elapsed_time = time.time() - self.start_run
self.log.info(
'finished run loop, elapsed time = %1.2f seconds, processed %d rows, last added_id was %d'
% (elapsed_time, self.rows_processed, self.last_id_processed))
def by_id(self, id):
if len(id) == 32:
id = id.decode('hex')
row = self.connection.get('SELECT * FROM entities WHERE id = %s', id)
return Entity.from_row(row, use_zlib=self.use_zlib) if row else None
