def post(self): params = request.json if params['ttl'] == 0: query_result = None else: query_result = data_manager.get_query_result(params['query'], int(params['ttl'])) if query_result: return {'query_result': query_result.to_dict(parse_data=True)} else: job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY) return {'job': job.to_dict()}
def post(self): params = request.json if params['ttl'] == 0: query_result = None else: query_result = data_manager.get_query_result( params['query'], int(params['ttl'])) if query_result: return {'query_result': query_result.to_dict(parse_data=True)} else: job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY) return {'job': job.to_dict()}
def post(self): params = request.json models.ActivityLog( user=self.current_user, type=models.ActivityLog.QUERY_EXECUTION, activity=params['query'] ).save() if params['ttl'] == 0: query_result = None else: query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], int(params['ttl'])) if query_result: return {'query_result': query_result.to_dict()} else: data_source = models.DataSource.get_by_id(params['data_source_id']) job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY, data_source) return {'job': job.to_dict()}
def post(self): params = request.json models.ActivityLog(user=self.current_user, type=models.ActivityLog.QUERY_EXECUTION, activity=params['query']).save() if params['ttl'] == 0: query_result = None else: query_result = models.QueryResult.get_latest( params['data_source_id'], params['query'], int(params['ttl'])) if query_result: return {'query_result': query_result.to_dict()} else: data_source = models.DataSource.get_by_id(params['data_source_id']) job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY, data_source) return {'job': job.to_dict()}
def post(self): params = request.json if settings.FEATURE_TABLES_PERMISSIONS: metadata = utils.SQLMetaData(params['query']) if metadata.has_non_select_dml_statements or metadata.has_ddl_statements: return {'job': {'error': 'Only SELECT statements are allowed'}} if len(metadata.used_tables - current_user.allowed_tables ) > 0 and '*' not in current_user.allowed_tables: logging.warning('Permission denied for user %s to table %s', self.current_user.name, metadata.used_tables) return { 'job': { 'error': 'Access denied for table(s): %s' % (metadata.used_tables) } } models.ActivityLog(user=self.current_user, type=models.ActivityLog.QUERY_EXECUTION, activity=params['query']).save() if params['ttl'] == 0: query_result = None else: query_result = models.QueryResult.get_latest( params['data_source_id'], params['query'], int(params['ttl'])) if query_result: return {'query_result': query_result.to_dict()} else: data_source = models.DataSource.get_by_id(params['data_source_id']) job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY, data_source) return {'job': job.to_dict()}
def post(self): params = request.json if settings.FEATURE_TABLES_PERMISSIONS: metadata = utils.SQLMetaData(params['query']) if metadata.has_non_select_dml_statements or metadata.has_ddl_statements: return { 'job': { 'error': 'Only SELECT statements are allowed' } } if len(metadata.used_tables - current_user.allowed_tables) > 0 and '*' not in current_user.allowed_tables: logging.warning('Permission denied for user %s to table %s', self.current_user.name, metadata.used_tables) return { 'job': { 'error': 'Access denied for table(s): %s' % (metadata.used_tables) } } models.ActivityLog( user=self.current_user, type=models.ActivityLog.QUERY_EXECUTION, activity=params['query'] ).save() if params['ttl'] == 0: query_result = None else: query_result = models.QueryResult.get_latest(params['data_source_id'], params['query'], int(params['ttl'])) if query_result: return {'query_result': query_result.to_dict()} else: data_source = models.DataSource.get_by_id(params['data_source_id']) job = data_manager.add_job(params['query'], data.Job.HIGH_PRIORITY, data_source) return {'job': job.to_dict()}
print "Clear jobs/hashes:" redis_connection.delete("jobs") query_hashes = redis_connection.keys("query_hash_*") if query_hashes: redis_connection.delete(*query_hashes) starting_query_results_count = models.QueryResult.select().count() jobs_count = 5000 workers_count = 10 print "Creating jobs..." for i in xrange(jobs_count): query = "SELECT {}".format(i) print "Inserting: {}".format(query) data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY, data_source=data_source) print "Starting workers..." workers = data_manager.start_workers(workers_count) print "Waiting for jobs to be done..." keep_waiting = True while keep_waiting: results_count = models.QueryResult.select().count( ) - starting_query_results_count print "QueryResults: {}".format(results_count) time.sleep(5) if results_count == jobs_count: print "Yay done..." keep_waiting = False
print "Clear jobs/hashes:" redis_connection.delete("jobs") query_hashes = redis_connection.keys("query_hash_*") if query_hashes: redis_connection.delete(*query_hashes) starting_query_results_count = models.QueryResult.select().count() jobs_count = 5000 workers_count = 10 print "Creating jobs..." for i in xrange(jobs_count): query = "SELECT {}".format(i) print "Inserting: {}".format(query) data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY, data_source=data_source) print "Starting workers..." workers = data_manager.start_workers(workers_count) print "Waiting for jobs to be done..." keep_waiting = True while keep_waiting: results_count = models.QueryResult.select().count() - starting_query_results_count print "QueryResults: {}".format(results_count) time.sleep(5) if results_count == jobs_count: print "Yay done..." keep_waiting = False data_manager.stop_workers()