def _column_comparison(self, column, value, force_null=True):
if isinstance(value, list):
value = list(set(value))
contains_null = any([x is None for x in value])
if contains_null:
value = [x for x in value if x is not None]
null_contains_return = (column + ' IS NULL OR ') if contains_null else ''
if len(value) > 0:
sorted_values = self._sorted(value)
if is_on_postgresql():
return '({} {} = ANY(VALUES {}))'.format(
null_contains_return,
column,
','.join(['(%s)' for i in value])
), sorted_values
else:
return '({} {} IN ({}))'.format(
null_contains_return,
column,
','.join(['%s' for i in value])
), sorted_values
else:
return '(' + null_contains_return + DATABASE_TRUE + ')', []
elif value is not None:
return column + ' = %s', [value]
elif (isinstance(force_null, bool) and force_null) or (isinstance(force_null, list) and column in force_null):
return column + ' IS NULL', []
else:
return DATABASE_TRUE, []
def handle_gc(self, options):
timer('recompute_gc')
print(' -- collecting garbage')
to_gc = [
str(x.id) for x in EnvironmentInfo.objects.filter(
status=EnvironmentInfo.STATUS_DISABLED).all()
]
if not to_gc:
print(' -- no environment info to collect')
return
to_gc_str = ','.join(to_gc)
with closing(connection.cursor()) as cursor:
cursor.execute(
'DELETE FROM proso_models_variable WHERE info_id IN (%s)' %
to_gc_str)
variables = cursor.rowcount
cursor.execute(
'DELETE FROM proso_models_audit WHERE info_id IN (%s)' %
to_gc_str)
audits = cursor.rowcount
cursor.execute(
'DELETE FROM proso_models_environmentinfo WHERE id IN (%s)' %
to_gc_str)
infos = cursor.rowcount
if is_on_postgresql():
timer('recompute_vacuum')
cursor.execute(
'VACUUM FULL ANALYZE VERBOSE proso_models_variable')
cursor.execute(
'VACUUM FULL ANALYZE VERBOSE proso_models_audit')
print(' -- vacuum phase, time:', timer('recompute_vacuum'),
'seconds')
print(' -- collecting garbage, time:', timer('recompute_gc'),
'seconds, deleted', variables, 'variables,', audits,
'audit records,', infos, 'environment info records')
def handle_one_export(self, name, sql, batch_size):
if is_on_postgresql():
sql = '({})'.format(sql)
table_name = 'tmp_{}'.format(str(uuid.uuid1()).replace('-', '_'))
with transaction.atomic():
with closing(connection.cursor()) as cursor:
print('processing {}'.format(name))
cursor.execute('CREATE TABLE {} AS {}'.format(table_name, sql))
cursor.execute('SELECT COUNT(*) FROM {}'.format(table_name))
count, = cursor.fetchone()
dest_file = settings.DATA_DIR + '/' + name + '.csv'
proso.django.db.dump_table(table_name, 'id', batch_size, dest_file)
cursor.execute('DROP TABLE {}'.format(table_name))
def handle_one_export(self, name, sql, batch_size):
if is_on_postgresql():
sql = '({})'.format(sql)
table_name = 'tmp_{}'.format(str(uuid.uuid1()).replace('-', '_'))
with transaction.atomic():
with closing(connection.cursor()) as cursor:
print('processing {}'.format(name))
cursor.execute('CREATE TABLE {} AS {}'.format(table_name, sql))
cursor.execute('SELECT COUNT(*) FROM {}'.format(table_name))
count, = cursor.fetchone()
dest_file = settings.DATA_DIR + '/' + name + '.csv'
proso.django.db.dump_table(table_name, 'id', batch_size,
dest_file)
cursor.execute('DROP TABLE {}'.format(table_name))
def handle_gc(self, options):
timer('recompute_gc')
print(' -- collecting garbage')
to_gc = [str(x.id) for x in EnvironmentInfo.objects.filter(status=EnvironmentInfo.STATUS_DISABLED).all()]
if not to_gc:
print(' -- no environment info to collect')
return
to_gc_str = ','.join(to_gc)
with closing(connection.cursor()) as cursor:
cursor.execute('DELETE FROM proso_models_variable WHERE info_id IN (%s)' % to_gc_str)
variables = cursor.rowcount
cursor.execute('DELETE FROM proso_models_environmentinfo WHERE id IN (%s)' % to_gc_str)
infos = cursor.rowcount
if is_on_postgresql():
timer('recompute_vacuum')
cursor.execute('VACUUM FULL ANALYZE VERBOSE proso_models_variable')
print(' -- vacuum phase, time:', timer('recompute_vacuum'), 'seconds')
print(' -- collecting garbage, time:', timer('recompute_gc'), 'seconds, deleted', variables, 'variables,', infos, 'environment info records')