def create_instance(self, report_id, instance_id, rows, footer, expire):
now = datetime.utcnow()
expire = now + timedelta(seconds=expire)
# Check if the instance already exists
metas = self.conn.execute('''
select expires_ts from %s
where report_id = ? and instance_id = ?
''' % self.METADATA_TABLE, (report_id, instance_id))
for meta in metas:
if meta[0] > now:
raise caches.InstanceExistsError('Instance already cached.')
# Build the table for this instance (will not exist for zero rows)
table_name = '%s_%s' % (report_id, instance_id)
self.conn.execute('drop table if exists %s' % table_name)
try:
first_row = rows.next()
except StopIteration:
pass
else:
columns = ', '.join(sorted(first_row.keys()))
self.conn.execute('create table %s (%s)' % (table_name, columns))
for column in first_row.keys():
self.conn.execute('''
create index ix_%s_%s on %s (%s)
''' % (table_name, column, table_name, column))
# Insert the rows into the table
for row in itertools.chain([first_row], rows):
columns, values = zip(*row.items())
columns = ','.join(columns)
inserts = ','.join(['?' for value in values])
self.conn.execute('''
insert into %s (%s) values (%s)
''' % (table_name, columns, inserts), map(encode, values))
# Create the metadata row for the instance
self.conn.execute('''
insert or replace into %s
(report_id, instance_id, created_ts, expires_ts, footer)
values (?, ?, ?, ?, ?)
''' % self.METADATA_TABLE, (report_id, instance_id, now, expire, encode(footer() or {})))
def create_instance(self, report_id, instance_id, rows, footer, expire):
keys = set()
table_name = '%s:%s' % (report_id, instance_id)
# Really simple lock on writing to the table
# NOTE: Cannot use expire on the table lock because until Redis 2.1.3
# any write operation (even setnx on an existing key) causes the key
# to be deleted so setnx returns true, even on existing keys, if they
# have an expire set.
if not self.conn.setnx('%s:_lock:' % table_name, 'lock'):
raise caches.InstanceLockError('Instance already locked')
# Check if table already created, or set the timestamp
if self.conn.exists('%s:' % table_name):
# Release lock and raise an error
self.conn.delete('%s:_lock:' % table_name)
raise caches.InstanceExistsError('Instance already cached')
self.conn['%s:' % table_name] = encode(datetime.utcnow())
keys.add('%s:' % table_name)
# Pipeline the insert operations for speed
p = self.conn.pipeline(False)
for row_id, row in enumerate(rows):
p.hmset('%s:%s' % (table_name, row_id), encode_dict(row))
keys.add('%s:%s' % (table_name, row_id))
p.sadd('%s:ids:' % table_name, row_id)
keys.add('%s:ids:' % table_name)
# Index the row
key = '%s:index:%s:' % (table_name, row_id)
data = {}
for name, value in row.iteritems():
t = type(value)
if t is unicode:
data[name] = value.encode('utf-8')
elif t is Decimal:
data[name] = float(value)
elif t in (int, float, long, str):
data[name] = value
elif t is type(None):
data[name] = REDIS_MAX_INT # For sorting
else:
data[name] = str(value)
p.hmset(key, data)
keys.add(key)
# Table footer
if footer:
footer_row = encode_dict(footer() or {})
p.hmset('%s:footer:' % table_name, footer_row)
keys.add('%s:footer:' % table_name)
# mark that the table is done.
p.set('%s:_done:' % table_name, 'done')
keys.add('%s:_done:' % table_name)
# Table expiration.
if expire:
for key in keys:
p.expire(key, expire)
# Release the table lock
p.delete('%s:_lock:' % table_name)
p.execute()
def create_instance(self, report_id, instance_id, rows, footer, expire):
keys = set()
table_name = '%s:%s' % (report_id, instance_id)
# Really simple lock on writing to the table
# NOTE: Cannot use expire on the table lock because until Redis 2.1.3
# any write operation (even setnx on an existing key) causes the key
# to be deleted so setnx returns true, even on existing keys, if they
# have an expire set.
if not self.conn.setnx('%s:_lock:' % table_name, 'lock'):
raise caches.InstanceLockError('Instance already locked')
# Check if table already created, or set the timestamp
if self.conn.exists('%s:' % table_name):
# Release lock and raise an error
self.conn.delete('%s:_lock:' % table_name)
raise caches.InstanceExistsError('Instance already cached')
self.conn['%s:' % table_name] = encode(datetime.utcnow())
keys.add('%s:' % table_name)
# Pipeline the insert operations for speed
try:
p = self.conn.pipeline(False)
for row_id, row in enumerate(rows):
p.hmset('%s:%s' % (table_name, row_id), encode_dict(row))
keys.add('%s:%s' % (table_name, row_id))
p.sadd('%s:ids:' % table_name, row_id)
keys.add('%s:ids:' % table_name)
# Index the row
key = '%s:index:%s:' % (table_name, row_id)
data = {}
for name, value in row.iteritems():
t = type(value)
if t is unicode:
data[name] = value.encode('utf-8')
elif t is Decimal:
data[name] = float(value)
elif t in (int, float, long, str):
data[name] = value
else:
data[name] = str(value)
p.hmset(key, data)
keys.add(key)
# Table footer
if footer:
footer_row = encode_dict(footer() or {})
p.hmset('%s:footer:' % table_name, footer_row)
keys.add('%s:footer:' % table_name)
# mark that the table is done.
p.set('%s:_done:' % table_name, 'done')
keys.add('%s:_done:' % table_name)
# Table expiration.
if expire:
for key in keys:
p.expire(key, expire)
# Release the table lock
p.delete('%s:_lock:' % table_name)
p.execute()
except:
try:
# failed to actually run the report, so cleanup initial cache data.
self.conn.delete('%s:_lock:' % table_name)
self.conn.delete('%s:' % table_name)
except:
# don't set a redis error mask the original exception
pass
raise
