def by_id(self, id): if len(id) == 32: if sys.version_info[0] == 2: id = id.decode('hex') else: id = codecs.decode(id, "hex_codec") row = self.connection.get('SELECT * FROM entities WHERE id = %s', id) return Entity.from_row(row, use_zlib=self.use_zlib) if row else None
def _do_query(self, exprs, order_by, limit): values = [] where_clause = [] for e in exprs: if e.name not in self.properties: raise ValueError('This index has no column named %r' % (e.name, )) expr_string, vals = e.build() where_clause.append(expr_string) values.extend(vals) if self.table == 'entities': # XXX: this is a bit hacky q = 'SELECT * FROM entities WHERE ' + ' AND '.join(where_clause) if order_by: q += ' ORDER BY %s %s' % (order_by.name, order_by.order) if limit: q += ' LIMIT %d' % (limit, ) entity_rows = self.connection.query(q, *values) else: q = 'SELECT entity_id FROM %s' % self.table if where_clause: q += ' WHERE ' + ' AND '.join(where_clause) if order_by: q += ' ORDER BY %s %s' % (order_by.name, order_by.order) if limit: q += ' LIMIT %d' % (limit, ) rows = self.connection.query(q, *values) if rows: entity_ids = [r['entity_id'] for r in rows] q = 'SELECT * FROM entities WHERE id IN (' q += ', '.join('%s' for x in rows) q += ')' entity_rows = self.connection.query(q, *entity_ids) else: return [] if not order_by: #sorted_entities = sorted(entity_rows, key=lambda x: x['updated'], reverse=True) sorted_entities = sorted(entity_rows, key=lambda x: x['updated']) else: # XXX: this is O(n^2), bad sorted_entities = [] for row_id in (row['entity_id'] for row in rows): for e in entity_rows: if e['id'] == row_id: sorted_entities.append(e) break else: assert False return [ Entity.from_row(row, use_zlib=self.use_zlib) for row in sorted_entities ]
def _do_query(self, exprs, order_by, limit): values = [] where_clause = [] for e in exprs: if e.name not in self.properties: raise ValueError('This index has no column named %r' % (e.name,)) expr_string, vals = e.build() where_clause.append(expr_string) values.extend(vals) if self.table == 'entities': # XXX: this is a bit hacky q = 'SELECT * FROM entities WHERE ' + ' AND '.join(where_clause) if order_by: q += ' ORDER BY %s %s' % (order_by.name, order_by.order) if limit: q += ' LIMIT %d' % (limit,) entity_rows = self.connection.query(q, *values) else: q = 'SELECT entity_id FROM %s' % self.table if where_clause: q += ' WHERE ' + ' AND '.join(where_clause) if order_by: q += ' ORDER BY %s %s' % (order_by.name, order_by.order) if limit: q += ' LIMIT %d' % (limit,) rows = self.connection.query(q, *values) if rows: entity_ids = [r['entity_id'] for r in rows] q = 'SELECT * FROM entities WHERE id IN (' q += ', '.join('%s' for x in rows) q += ')' entity_rows = self.connection.query(q, *entity_ids) else: return [] if not order_by: #sorted_entities = sorted(entity_rows, key=lambda x: x['updated'], reverse=True) sorted_entities = sorted(entity_rows, key=lambda x: x['updated']) else: # XXX: this is O(n^2), bad sorted_entities = [] for row_id in (row['entity_id'] for row in rows): for e in entity_rows: if e['id'] == row_id: sorted_entities.append(e) break else: assert False return [Entity.from_row(row, use_zlib=self.use_zlib) for row in sorted_entities]
def run(self): rows_processed = 0 self.log.info('starting run loop') try: for row in self.row_iterator(): entity = Entity.from_row(row, use_zlib=self.use_zlib) self.process_row(row, entity) self.rows_processed += 1 self.last_id_processed = row['added_id'] except: self.log.exception('exception during run loop!') finally: elapsed_time = time.time() - self.start_run self.log.info('finished run loop, elapsed time = %1.2f seconds, processed %d rows, last added_id was %d' % (elapsed_time, self.rows_processed, self.last_id_processed))
def run(self): rows_processed = 0 self.log.info('starting run loop') try: for row in self.row_iterator(): entity = Entity.from_row(row, use_zlib=self.use_zlib) self.process_row(row, entity) self.rows_processed += 1 self.last_id_processed = row['added_id'] except: self.log.exception('exception during run loop!') finally: elapsed_time = time.time() - self.start_run self.log.info( 'finished run loop, elapsed time = %1.2f seconds, processed %d rows, last added_id was %d' % (elapsed_time, self.rows_processed, self.last_id_processed))
def by_id(self, id): if len(id) == 32: id = id.decode('hex') row = self.connection.get('SELECT * FROM entities WHERE id = %s', id) return Entity.from_row(row, use_zlib=self.use_zlib) if row else None