def get_jobs(self, notebook, snippet, logs):
jobs = []
patch_snippet_for_connector(snippet)
if snippet['dialect'] == 'hive':
engine = self._get_hive_execution_engine(notebook, snippet)
jobs_with_state = parse_out_jobs(logs,
engine=engine,
with_state=True)
queries_with_state = parse_out_queries(logs,
engine=engine,
with_state=True)
jobs = [{
'name':
job.get('job_id', ''),
'url':
reverse('jobbrowser.views.single_job',
kwargs={'job': job.get('job_id', '')})
if has_jobbrowser else '',
'started':
job.get('started', False),
'finished':
job.get('finished', False)
} for job in jobs_with_state]
if has_hive_query_browser:
jobs += [{
'name': job.get('job_id', ''),
'url': 'api/job/queries-hive/',
'started': job.get('started', False),
'finished': job.get('finished', False)
} for job in queries_with_state]
elif snippet['dialect'] == 'impala' and has_query_browser:
guid = snippet['result']['handle']['guid']
if isinstance(guid, str):
guid = guid.encode('utf-8')
query_id = unpack_guid_base64(guid)
progress = min(self.progress(notebook, snippet, logs),
99) if snippet['status'] != 'available' and snippet[
'status'] != 'success' else 100
jobs = [{
'name': query_id,
'url': '/hue/jobbrowser#!id=%s' % query_id,
'started': True,
'finished': False,
'percentJob': progress
}]
return jobs
def describe(request, database, table=None, column=None):
response = {'status': -1, 'message': ''}
notebook = json.loads(request.POST.get('notebook', '{}'))
source_type = request.POST.get('source_type', '')
snippet = {'type': source_type}
patch_snippet_for_connector(snippet)
describe = get_api(request, snippet).describe(notebook,
snippet,
database,
table,
column=column)
response.update(describe)
return JsonResponse(response)
def progress(self, notebook, snippet, logs=''):
patch_snippet_for_connector(snippet)
if snippet['dialect'] == 'hive':
match = re.search('Total jobs = (\d+)', logs, re.MULTILINE)
total = int(match.group(1)) if match else 1
started = logs.count('Starting Job')
ended = logs.count('Ended Job')
progress = int((started + ended) * 100 / (total * 2))
return max(progress, 5) # Return 5% progress as a minimum
elif snippet['dialect'] == 'impala':
match = re.findall('(\d+)% Complete', logs, re.MULTILINE)
# Retrieve the last reported progress percentage if it exists
return int(match[-1]) if match and isinstance(match, list) else 0
else:
return 50
