def create_instance(self, report_id, instance_id, rows, footer, expire): now = datetime.utcnow() expire = now + timedelta(seconds=expire) # Check if the instance already exists metas = self.conn.execute(''' select expires_ts from %s where report_id = ? and instance_id = ? ''' % self.METADATA_TABLE, (report_id, instance_id)) for meta in metas: if meta[0] > now: raise caches.InstanceExistsError('Instance already cached.') # Build the table for this instance (will not exist for zero rows) table_name = '%s_%s' % (report_id, instance_id) self.conn.execute('drop table if exists %s' % table_name) try: first_row = rows.next() except StopIteration: pass else: columns = ', '.join(sorted(first_row.keys())) self.conn.execute('create table %s (%s)' % (table_name, columns)) for column in first_row.keys(): self.conn.execute(''' create index ix_%s_%s on %s (%s) ''' % (table_name, column, table_name, column)) # Insert the rows into the table for row in itertools.chain([first_row], rows): columns, values = zip(*row.items()) columns = ','.join(columns) inserts = ','.join(['?' for value in values]) self.conn.execute(''' insert into %s (%s) values (%s) ''' % (table_name, columns, inserts), map(encode, values)) # Create the metadata row for the instance self.conn.execute(''' insert or replace into %s (report_id, instance_id, created_ts, expires_ts, footer) values (?, ?, ?, ?, ?) ''' % self.METADATA_TABLE, (report_id, instance_id, now, expire, encode(footer() or {})))
def create_instance(self, report_id, instance_id, rows, footer, expire): keys = set() table_name = '%s:%s' % (report_id, instance_id) # Really simple lock on writing to the table # NOTE: Cannot use expire on the table lock because until Redis 2.1.3 # any write operation (even setnx on an existing key) causes the key # to be deleted so setnx returns true, even on existing keys, if they # have an expire set. if not self.conn.setnx('%s:_lock:' % table_name, 'lock'): raise caches.InstanceLockError('Instance already locked') # Check if table already created, or set the timestamp if self.conn.exists('%s:' % table_name): # Release lock and raise an error self.conn.delete('%s:_lock:' % table_name) raise caches.InstanceExistsError('Instance already cached') self.conn['%s:' % table_name] = encode(datetime.utcnow()) keys.add('%s:' % table_name) # Pipeline the insert operations for speed p = self.conn.pipeline(False) for row_id, row in enumerate(rows): p.hmset('%s:%s' % (table_name, row_id), encode_dict(row)) keys.add('%s:%s' % (table_name, row_id)) p.sadd('%s:ids:' % table_name, row_id) keys.add('%s:ids:' % table_name) # Index the row key = '%s:index:%s:' % (table_name, row_id) data = {} for name, value in row.iteritems(): t = type(value) if t is unicode: data[name] = value.encode('utf-8') elif t is Decimal: data[name] = float(value) elif t in (int, float, long, str): data[name] = value elif t is type(None): data[name] = REDIS_MAX_INT # For sorting else: data[name] = str(value) p.hmset(key, data) keys.add(key) # Table footer if footer: footer_row = encode_dict(footer() or {}) p.hmset('%s:footer:' % table_name, footer_row) keys.add('%s:footer:' % table_name) # mark that the table is done. p.set('%s:_done:' % table_name, 'done') keys.add('%s:_done:' % table_name) # Table expiration. if expire: for key in keys: p.expire(key, expire) # Release the table lock p.delete('%s:_lock:' % table_name) p.execute()
def create_instance(self, report_id, instance_id, rows, footer, expire): keys = set() table_name = '%s:%s' % (report_id, instance_id) # Really simple lock on writing to the table # NOTE: Cannot use expire on the table lock because until Redis 2.1.3 # any write operation (even setnx on an existing key) causes the key # to be deleted so setnx returns true, even on existing keys, if they # have an expire set. if not self.conn.setnx('%s:_lock:' % table_name, 'lock'): raise caches.InstanceLockError('Instance already locked') # Check if table already created, or set the timestamp if self.conn.exists('%s:' % table_name): # Release lock and raise an error self.conn.delete('%s:_lock:' % table_name) raise caches.InstanceExistsError('Instance already cached') self.conn['%s:' % table_name] = encode(datetime.utcnow()) keys.add('%s:' % table_name) # Pipeline the insert operations for speed try: p = self.conn.pipeline(False) for row_id, row in enumerate(rows): p.hmset('%s:%s' % (table_name, row_id), encode_dict(row)) keys.add('%s:%s' % (table_name, row_id)) p.sadd('%s:ids:' % table_name, row_id) keys.add('%s:ids:' % table_name) # Index the row key = '%s:index:%s:' % (table_name, row_id) data = {} for name, value in row.iteritems(): t = type(value) if t is unicode: data[name] = value.encode('utf-8') elif t is Decimal: data[name] = float(value) elif t in (int, float, long, str): data[name] = value else: data[name] = str(value) p.hmset(key, data) keys.add(key) # Table footer if footer: footer_row = encode_dict(footer() or {}) p.hmset('%s:footer:' % table_name, footer_row) keys.add('%s:footer:' % table_name) # mark that the table is done. p.set('%s:_done:' % table_name, 'done') keys.add('%s:_done:' % table_name) # Table expiration. if expire: for key in keys: p.expire(key, expire) # Release the table lock p.delete('%s:_lock:' % table_name) p.execute() except: try: # failed to actually run the report, so cleanup initial cache data. self.conn.delete('%s:_lock:' % table_name) self.conn.delete('%s:' % table_name) except: # don't set a redis error mask the original exception pass raise