def handle(self, *args, **options): if options['user']: usernames = [options['user']] else: usernames = ['anonymous'] + list(User.objects.values_list('username', flat=True)) adapter = DatabaseAdapter() stale_tables = [] for username in usernames: schema_name = settings.QUERY_USER_SCHEMA_PREFIX + username tables = adapter.fetch_tables(schema_name) jobs = QueryJob.objects.filter(schema_name=schema_name) for table in tables: job = QueryJob.objects.filter( schema_name=schema_name, table_name=table['name'] ).first() if job and job.phase not in [QueryJob.PHASE_EXECUTING, QueryJob.PHASE_COMPLETED]: stale_tables.append((schema_name, table['name'], job.phase if job else None)) if stale_tables: print('The following database tables have no associated QueryJob:') for stale_table in stale_tables: print('%s.%s -> %s' % stale_table) if options['delete']: for schema_name, table_name, phase in stale_tables: adapter.drop_table(schema_name, table_name) print('The tables have been deleted.') else: print('No tables without associated QueryJob have been found.')
def ingest_table(schema_name, table_name, file_path, drop_table=False): adapter = DatabaseAdapter() table = parse_single_table(file_path, pedantic=False) columns = [] for field in table.fields: columns.append({ 'name': field.name, 'datatype': field.datatype, 'ucd': field.ucd, 'unit': str(field.unit), }) if drop_table: adapter.drop_table(schema_name, table_name) adapter.create_table(schema_name, table_name, columns) adapter.insert_rows(schema_name, table_name, columns, table.array, table.array.mask) os.remove(file_path) return columns