def get_jobs(self, notebook, snippet, logs): jobs = [] patch_snippet_for_connector(snippet) if snippet['dialect'] == 'hive': engine = self._get_hive_execution_engine(notebook, snippet) jobs_with_state = parse_out_jobs(logs, engine=engine, with_state=True) queries_with_state = parse_out_queries(logs, engine=engine, with_state=True) jobs = [{ 'name': job.get('job_id', ''), 'url': reverse('jobbrowser.views.single_job', kwargs={'job': job.get('job_id', '')}) if has_jobbrowser else '', 'started': job.get('started', False), 'finished': job.get('finished', False) } for job in jobs_with_state] if has_hive_query_browser: jobs += [{ 'name': job.get('job_id', ''), 'url': 'api/job/queries-hive/', 'started': job.get('started', False), 'finished': job.get('finished', False) } for job in queries_with_state] elif snippet['dialect'] == 'impala' and has_query_browser: guid = snippet['result']['handle']['guid'] if isinstance(guid, str): guid = guid.encode('utf-8') query_id = unpack_guid_base64(guid) progress = min(self.progress(notebook, snippet, logs), 99) if snippet['status'] != 'available' and snippet[ 'status'] != 'success' else 100 jobs = [{ 'name': query_id, 'url': '/hue/jobbrowser#!id=%s' % query_id, 'started': True, 'finished': False, 'percentJob': progress }] return jobs
def describe(request, database, table=None, column=None): response = {'status': -1, 'message': ''} notebook = json.loads(request.POST.get('notebook', '{}')) source_type = request.POST.get('source_type', '') snippet = {'type': source_type} patch_snippet_for_connector(snippet) describe = get_api(request, snippet).describe(notebook, snippet, database, table, column=column) response.update(describe) return JsonResponse(response)
def progress(self, notebook, snippet, logs=''): patch_snippet_for_connector(snippet) if snippet['dialect'] == 'hive': match = re.search('Total jobs = (\d+)', logs, re.MULTILINE) total = int(match.group(1)) if match else 1 started = logs.count('Starting Job') ended = logs.count('Ended Job') progress = int((started + ended) * 100 / (total * 2)) return max(progress, 5) # Return 5% progress as a minimum elif snippet['dialect'] == 'impala': match = re.findall('(\d+)% Complete', logs, re.MULTILINE) # Retrieve the last reported progress percentage if it exists return int(match[-1]) if match and isinstance(match, list) else 0 else: return 50