def test_impala_flags(): test_impala_conf_dir = tempfile.mkdtemp() resets = [] try: if conf.QUERYCACHE_ROWS.get() != 50000: resets.append(conf.QUERYCACHE_ROWS.set_for_testing(50000)) assert_equal(conf.QUERYCACHE_ROWS.get(), 50000) assert_equal(conf.IMPERSONATION_ENABLED.get(), False) flags = """ -webserver_certificate_file=/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem -ssl_server_certificate=/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem -max_result_cache_size=100000 -authorized_proxy_user_config=hue=* """ file(os.path.join(test_impala_conf_dir, 'impalad_flags'), 'w').write(flags) resets.append(conf.IMPALA_CONF_DIR.set_for_testing(test_impala_conf_dir)) impala_flags.reset() assert_equal(impala_flags.get_webserver_certificate_file(), '/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem') assert_equal(impala_flags.get_ssl_server_certificate(), '/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem') assert_equal(impala_flags.get_max_result_cache_size(), 100000) assert_equal(impala_flags.get_authorized_proxy_user_config(), 'hue=*') # From Config assert_equal(conf.QUERYCACHE_ROWS.get(), 100000) assert_equal(conf.IMPERSONATION_ENABLED.get(), True) finally: impala_flags.reset() for reset in resets: reset()
def test_impala_flags(): test_impala_conf_dir = tempfile.mkdtemp() finish = None try: assert_equal(conf.QUERYCACHE_ROWS.get(), 50000) assert_equal(conf.IMPERSONATION_ENABLED.get(), False) flags = """ -webserver_certificate_file=/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem -ssl_server_certificate=/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem -max_result_cache_size=100000 -authorized_proxy_user_config=hue=* """ file(os.path.join(test_impala_conf_dir, 'impalad_flags'), 'w').write(flags) finish = conf.IMPALA_CONF_DIR.set_for_testing(test_impala_conf_dir) impala_flags.reset() assert_equal(impala_flags.get_webserver_certificate_file(), '/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem') assert_equal(impala_flags.get_ssl_server_certificate(), '/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem') assert_equal(impala_flags.get_max_result_cache_size(), 100000) assert_equal(impala_flags.get_authorized_proxy_user_config(), 'hue=*') # From Config assert_equal(conf.QUERYCACHE_ROWS.get(), 100000) assert_equal(conf.IMPERSONATION_ENABLED.get(), True) finally: impala_flags.reset() if finish: finish()
def test_impala_flags(): test_impala_conf_dir = tempfile.mkdtemp() resets = [] expected_rows = 50000 try: if conf.QUERYCACHE_ROWS.get() != expected_rows: resets.append(conf.QUERYCACHE_ROWS.set_for_testing(expected_rows)) assert_equal(conf.QUERYCACHE_ROWS.get(), expected_rows) assert_equal(conf.IMPERSONATION_ENABLED.get(), False) flags = """ -webserver_certificate_file=/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem -ssl_server_certificate=/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem -max_result_cache_size=%d -authorized_proxy_user_config=hue=* """ % expected_rows open_file(os.path.join(test_impala_conf_dir, 'impalad_flags'), 'w').write(flags) resets.append( conf.IMPALA_CONF_DIR.set_for_testing(test_impala_conf_dir)) impala_flags.reset() assert_equal(impala_flags.get_webserver_certificate_file(), '/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem') assert_equal(impala_flags.get_ssl_server_certificate(), '/etc/test-ssl-conf/CA_STANDARD/impala-cert.pem') assert_equal(impala_flags.get_max_result_cache_size(), expected_rows) assert_equal(impala_flags.get_authorized_proxy_user_config(), 'hue=*') # From Config assert_equal(conf.QUERYCACHE_ROWS.get(), expected_rows) assert_equal(conf.IMPERSONATION_ENABLED.get(), True) finally: impala_flags.reset() for reset in resets: reset()
class Submission(object): """ Represents one unique Oozie submission. Actions are: - submit - rerun """ def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None): self.job = job self.user = user self.fs = fs self.jt = jt # Deprecated with YARN, we now use logical names only for RM self.oozie_id = oozie_id self.api = get_oozie(self.user) if properties is not None: self.properties = properties else: self.properties = {} if local_tz and isinstance(self.job.data, dict): local_tz = self.job.data.get('properties')['timezone'] # Modify start_date & end_date only when it's a coordinator from oozie.models2 import Coordinator if type(self.job) is Coordinator: if 'start_date' in self.properties: properties['start_date'] = convert_to_server_timezone(self.properties['start_date'], local_tz) if 'end_date' in self.properties: properties['end_date'] = convert_to_server_timezone(self.properties['end_date'], local_tz) if 'nominal_time' in self.properties: properties['nominal_time'] = convert_to_server_timezone(self.properties['nominal_time'], local_tz) self.properties['security_enabled'] = self.api.security_enabled def __str__(self): if self.oozie_id: res = "Submission for job '%s'." % (self.oozie_id,) else: res = "Submission for job '%s' (id %s, owner %s)." % (self.job.name, self.job.id, self.user) if self.oozie_id: res += " -- " + self.oozie_id return res @submit_dryrun def run(self, deployment_dir=None): """ Take care of all the actions of submitting a Oozie workflow. Returns the oozie job id if all goes well. """ if self.properties and 'oozie.use.system.libpath' not in self.properties: self.properties['oozie.use.system.libpath'] = 'true' self.oozie_id = self.api.submit_job(self.properties) LOG.info("Submitted: %s" % (self,)) if self._is_workflow(): self.api.job_control(self.oozie_id, 'start') LOG.info("Started: %s" % (self,)) return self.oozie_id def rerun(self, deployment_dir, fail_nodes=None, skip_nodes=None): jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties(jt_address, deployment_dir) self.properties.update({'oozie.wf.application.path': deployment_dir}) if 'oozie.coord.application.path' in self.properties: self.properties.pop('oozie.coord.application.path') if 'oozie.bundle.application.path' in self.properties: self.properties.pop('oozie.bundle.application.path') if fail_nodes: self.properties.update({'oozie.wf.rerun.failnodes': fail_nodes}) elif not skip_nodes: self.properties.update({'oozie.wf.rerun.failnodes': 'false'}) # Case empty 'skip_nodes' list else: self.properties.update({'oozie.wf.rerun.skip.nodes': skip_nodes}) self.api.rerun(self.oozie_id, properties=self.properties) LOG.info("Rerun: %s" % (self,)) return self.oozie_id def rerun_coord(self, deployment_dir, params): jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties(jt_address, deployment_dir) self.properties.update({'oozie.coord.application.path': deployment_dir}) self.api.job_control(self.oozie_id, action='coord-rerun', properties=self.properties, parameters=params) LOG.info("Rerun: %s" % (self,)) return self.oozie_id def update_coord(self): self.api = get_oozie(self.user, api_version="v2") self.api.job_control(self.oozie_id, action='update', properties=self.properties, parameters=None) LOG.info("Update: %s" % (self,)) return self.oozie_id def rerun_bundle(self, deployment_dir, params): jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties(jt_address, deployment_dir) self.properties.update({'oozie.bundle.application.path': deployment_dir}) self.api.job_control(self.oozie_id, action='bundle-rerun', properties=self.properties, parameters=params) LOG.info("Rerun: %s" % (self,)) return self.oozie_id def deploy(self, deployment_dir=None): try: if not deployment_dir: deployment_dir = self._create_deployment_dir() except Exception, ex: msg = _("Failed to create deployment directory: %s" % ex) LOG.exception(msg) raise PopupException(message=msg, detail=str(ex)) if self.api.security_enabled: jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties(jt_address) # Needed for coordinator deploying workflows with credentials if hasattr(self.job, 'nodes'): for action in self.job.nodes: # Make sure XML is there # Don't support more than one level sub-workflow if action.data['type'] == 'subworkflow': from oozie.models2 import Workflow workflow = Workflow(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['workflow'])) sub_deploy = Submission(self.user, workflow, self.fs, self.jt, self.properties) workspace = sub_deploy.deploy() self.job.override_subworkflow_id(action, workflow.id) # For displaying the correct graph self.properties['workspace_%s' % workflow.uuid] = workspace # For pointing to the correct workspace elif action.data['type'] == 'impala' or action.data['type'] == 'impala-document': from oozie.models2 import _get_impala_url from impala.impala_flags import get_ssl_server_certificate if action.data['type'] == 'impala-document': from notebook.models import Notebook if action.data['properties'].get('uuid'): notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid'])) statements = notebook.get_str() statements = Template(statements).safe_substitute(**self.properties) script_name = action.data['name'] + '.sql' self._create_file(deployment_dir, script_name, statements) else: script_name = os.path.basename(action.data['properties'].get('script_path')) if self.api.security_enabled: kinit = 'kinit -k -t *.keytab %(user_principal)s' % { 'user_principal': self.properties.get('user_principal', action.data['properties'].get('user_principal')) } else: kinit = '' shell_script = """#!/bin/bash # Needed to launch impala shell in oozie export PYTHON_EGG_CACHE=./myeggs %(kinit)s impala-shell %(kerberos_option)s %(ssl_option)s -i %(impalad_host)s -f %(query_file)s""" % { 'impalad_host': action.data['properties'].get('impalad_host') or _get_impala_url(), 'kerberos_option': '-k' if self.api.security_enabled else '', 'ssl_option': '--ssl' if get_ssl_server_certificate() else '', 'query_file': script_name, 'kinit': kinit } self._create_file(deployment_dir, action.data['name'] + '.sh', shell_script) elif action.data['type'] == 'hive-document': from notebook.models import Notebook if action.data['properties'].get('uuid'): notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid'])) statements = notebook.get_str() else: statements = action.data['properties'].get('statements') if self.properties.get('send_result_path'): statements = """ INSERT OVERWRITE DIRECTORY '%s' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' WITH SERDEPROPERTIES ( "separatorChar" = "\t", "quoteChar" = "'", "escapeChar" = "\\" ) STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.join([snippet['statement_raw'] for snippet in notebook.get_data()['snippets']])) if statements is not None: self._create_file(deployment_dir, action.data['name'] + '.sql', statements) elif action.data['type'] in ('java-document', 'java', 'mapreduce-document'): if action.data['type'] == 'java-document' or action.data['type'] == 'mapreduce-document': from notebook.models import Notebook notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid'])) properties = notebook.get_data()['snippets'][0]['properties'] else: properties = action.data['properties'] if properties.get('app_jar'): LOG.debug("Adding to oozie.libpath %s" % properties['app_jar']) paths = [properties['app_jar']] if self.properties.get('oozie.libpath'): paths.append(self.properties['oozie.libpath']) self.properties['oozie.libpath'] = ','.join(paths) elif action.data['type'] == 'pig-document': from notebook.models import Notebook notebook = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=action.data['properties']['uuid'])) statements = notebook.get_data()['snippets'][0]['statement_raw'] self._create_file(deployment_dir, action.data['name'] + '.pig', statements) oozie_xml = self.job.to_xml(self.properties) self._do_as(self.user.username, self._copy_files, deployment_dir, oozie_xml, self.properties) return deployment_dir
class Submission(object): """ Represents one unique Oozie submission. Actions are: - submit - rerun """ def __init__(self, user, job=None, fs=None, jt=None, properties=None, oozie_id=None, local_tz=None): self.job = job self.user = user self.fs = fs self.jt = jt # Deprecated with YARN, we now use logical names only for RM self.oozie_id = oozie_id self.api = get_oozie(self.user) if properties is not None: self.properties = properties else: self.properties = {} if local_tz and isinstance(self.job.data, dict): local_tz = self.job.data.get('properties')['timezone'] # Modify start_date & end_date only when it's a coordinator from oozie.models2 import Coordinator if type(self.job) is Coordinator: if 'start_date' in self.properties: properties['start_date'] = convert_to_server_timezone( self.properties['start_date'], local_tz) if 'end_date' in self.properties: properties['end_date'] = convert_to_server_timezone( self.properties['end_date'], local_tz) if 'nominal_time' in self.properties: properties['nominal_time'] = convert_to_server_timezone( self.properties['nominal_time'], local_tz) self.properties['security_enabled'] = self.api.security_enabled def __str__(self): if self.oozie_id: res = "Submission for job '%s'." % (self.oozie_id, ) else: res = "Submission for job '%s' (id %s, owner %s)." % ( self.job.name, self.job.id, self.user) if self.oozie_id: res += " -- " + self.oozie_id return res @submit_dryrun def run(self, deployment_dir=None): """ Take care of all the actions of submitting a Oozie workflow. Returns the oozie job id if all goes well. """ if self.properties and 'oozie.use.system.libpath' not in self.properties: self.properties['oozie.use.system.libpath'] = 'true' self.oozie_id = self.api.submit_job(self.properties) LOG.info("Submitted: %s" % (self, )) if self._is_workflow(): self.api.job_control(self.oozie_id, 'start') LOG.info("Started: %s" % (self, )) return self.oozie_id def rerun(self, deployment_dir, fail_nodes=None, skip_nodes=None): jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties(jt_address, deployment_dir) self.properties.update({'oozie.wf.application.path': deployment_dir}) if 'oozie.coord.application.path' in self.properties: self.properties.pop('oozie.coord.application.path') if 'oozie.bundle.application.path' in self.properties: self.properties.pop('oozie.bundle.application.path') if fail_nodes: self.properties.update({'oozie.wf.rerun.failnodes': fail_nodes}) elif not skip_nodes: self.properties.update({'oozie.wf.rerun.failnodes': 'false'}) # Case empty 'skip_nodes' list else: self.properties.update({'oozie.wf.rerun.skip.nodes': skip_nodes}) self.api.rerun(self.oozie_id, properties=self.properties) LOG.info("Rerun: %s" % (self, )) return self.oozie_id def rerun_coord(self, deployment_dir, params): jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties(jt_address, deployment_dir) self.properties.update( {'oozie.coord.application.path': deployment_dir}) self.api.job_control(self.oozie_id, action='coord-rerun', properties=self.properties, parameters=params) LOG.info("Rerun: %s" % (self, )) return self.oozie_id def update_coord(self): self.api = get_oozie(self.user, api_version="v2") self.api.job_control(self.oozie_id, action='update', properties=self.properties, parameters=None) LOG.info("Update: %s" % (self, )) return self.oozie_id def rerun_bundle(self, deployment_dir, params): jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties(jt_address, deployment_dir) self.properties.update( {'oozie.bundle.application.path': deployment_dir}) self.api.job_control(self.oozie_id, action='bundle-rerun', properties=self.properties, parameters=params) LOG.info("Rerun: %s" % (self, )) return self.oozie_id def deploy(self, deployment_dir=None): try: if not deployment_dir: deployment_dir = self._create_deployment_dir() except Exception, ex: msg = _("Failed to create deployment directory: %s" % ex) LOG.exception(msg) raise PopupException(message=msg, detail=str(ex)) if self.api.security_enabled: jt_address = cluster.get_cluster_addr_for_job_submission() self._update_properties( jt_address ) # Needed for coordinator deploying workflows with credentials if hasattr(self.job, 'nodes'): for action in self.job.nodes: # Make sure XML is there # Don't support more than one level sub-workflow if action.data['type'] == 'subworkflow': from oozie.models2 import Workflow workflow = Workflow(document=Document2.objects.get_by_uuid( user=self.user, uuid=action.data['properties']['workflow'])) sub_deploy = Submission(self.user, workflow, self.fs, self.jt, self.properties) workspace = sub_deploy.deploy() self.job.override_subworkflow_id( action, workflow.id) # For displaying the correct graph self.properties[ 'workspace_%s' % workflow. uuid] = workspace # For pointing to the correct workspace elif action.data['type'] == 'altus': self._create_file( deployment_dir, action.data['name'] + '.sh', '''#!/usr/bin/env bash export PYTHONPATH=`pwd` echo 'Starting Altus command...' python altus.py ''') shell_script = self._generate_altus_action_script( service=action.data['properties'].get('service'), command=action.data['properties'].get('command'), arguments=dict([ arg.split('=', 1) for arg in action.data['properties'].get( 'arguments', []) ]), auth_key_id=ALTUS.AUTH_KEY_ID.get(), auth_key_secret=ALTUS.AUTH_KEY_SECRET.get().replace( '\\n', '\n')) self._create_file(deployment_dir, 'altus.py', shell_script) ext_py_lib_path = os.path.join(get_desktop_root(), 'core', 'ext-py') lib_dir_path = os.path.join(self.job.deployment_dir, 'lib') libs = [ (os.path.join(ext_py_lib_path, 'navoptapi-0.1.0'), 'navoptapi'), (os.path.join(ext_py_lib_path, 'navoptapi-0.1.0'), 'altuscli'), (os.path.join(ext_py_lib_path, 'asn1crypto-0.24.0'), 'asn1crypto'), (os.path.join(ext_py_lib_path, 'rsa-3.4.2'), 'rsa'), (os.path.join(ext_py_lib_path, 'pyasn1-0.1.8'), 'pyasn1'), ] for source_path, name in libs: destination_path = os.path.join(lib_dir_path, name) if not self.fs.do_as_user(self.user, self.fs.exists, destination_path): # Note: would be much faster to have only one zip archive self.fs.do_as_user(self.user, self.fs.copyFromLocal, os.path.join(source_path, name), destination_path) elif action.data['type'] == 'impala' or action.data[ 'type'] == 'impala-document': from oozie.models2 import _get_impala_url from impala.impala_flags import get_ssl_server_certificate if action.data['type'] == 'impala-document': from notebook.models import Notebook if action.data['properties'].get('uuid'): notebook = Notebook( document=Document2.objects.get_by_uuid( user=self.user, uuid=action.data['properties']['uuid'])) statements = notebook.get_str() statements = Template(statements).safe_substitute( **self.properties) script_name = action.data['name'] + '.sql' self._create_file(deployment_dir, script_name, statements) else: script_name = os.path.basename( action.data['properties'].get('script_path')) if self.api.security_enabled: kinit = 'kinit -k -t *.keytab %(user_principal)s' % { 'user_principal': self.properties.get( 'user_principal', action.data['properties']. get('user_principal')) } else: kinit = '' shell_script = """#!/bin/bash # Needed to launch impala shell in oozie export PYTHON_EGG_CACHE=./myeggs %(kinit)s impala-shell %(kerberos_option)s %(ssl_option)s -i %(impalad_host)s -f %(query_file)s""" % { 'impalad_host': action.data['properties'].get('impalad_host') or _get_impala_url(), 'kerberos_option': '-k' if self.api.security_enabled else '', 'ssl_option': '--ssl' if get_ssl_server_certificate() else '', 'query_file': script_name, 'kinit': kinit } self._create_file(deployment_dir, action.data['name'] + '.sh', shell_script) elif action.data['type'] == 'hive-document': from notebook.models import Notebook if action.data['properties'].get('uuid'): notebook = Notebook( document=Document2.objects.get_by_uuid( user=self.user, uuid=action.data['properties']['uuid'])) statements = notebook.get_str() else: statements = action.data['properties'].get( 'statements') if self.properties.get('send_result_path'): statements = """ INSERT OVERWRITE DIRECTORY '%s' ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde' WITH SERDEPROPERTIES ( "separatorChar" = "\t", "quoteChar" = "'", "escapeChar" = "\\" ) STORED AS TEXTFILE %s""" % (self.properties.get('send_result_path'), '\n\n\n'.join([ snippet['statement_raw'] for snippet in notebook.get_data()['snippets'] ])) if statements is not None: self._create_file(deployment_dir, action.data['name'] + '.sql', statements) elif action.data['type'] in ('java-document', 'java', 'mapreduce-document'): if action.data['type'] == 'java-document' or action.data[ 'type'] == 'mapreduce-document': from notebook.models import Notebook notebook = Notebook( document=Document2.objects.get_by_uuid( user=self.user, uuid=action.data['properties']['uuid'])) properties = notebook.get_data( )['snippets'][0]['properties'] else: properties = action.data['properties'] if properties.get('app_jar'): LOG.debug("Adding to oozie.libpath %s" % properties['app_jar']) paths = [properties['app_jar']] if self.properties.get('oozie.libpath'): paths.append(self.properties['oozie.libpath']) self.properties['oozie.libpath'] = ','.join(paths) elif action.data['type'] == 'pig-document': from notebook.models import Notebook notebook = Notebook(document=Document2.objects.get_by_uuid( user=self.user, uuid=action.data['properties'] ['uuid'])) statements = notebook.get_data( )['snippets'][0]['statement_raw'] self._create_file(deployment_dir, action.data['name'] + '.pig', statements) elif action.data['type'] in ('spark', 'spark-document') or ( action.data['type'] in ('sqoop', 'sqoop-document') and action.data['properties']['statement'] in '--hive-import'): if not [ f for f in action.data.get('properties').get( 'files', []) if f.get('value').endswith('hive-site.xml') ]: hive_site_lib = Hdfs.join(deployment_dir + '/lib/', 'hive-site.xml') hive_site_content = get_hive_site_content() if not self.fs.do_as_user( self.user, self.fs.exists, hive_site_lib) and hive_site_content: self.fs.do_as_user( self.user, self.fs.create, hive_site_lib, overwrite=True, permission=0700, data=smart_str(hive_site_content)) if action.data['type'] in ('sqoop', 'sqoop-document'): if CONFIG_JDBC_LIBS_PATH.get( ) and CONFIG_JDBC_LIBS_PATH.get( ) not in self.properties.get('oozie.libpath', ''): LOG.debug("Adding to oozie.libpath %s" % CONFIG_JDBC_LIBS_PATH.get()) paths = [CONFIG_JDBC_LIBS_PATH.get()] if self.properties.get('oozie.libpath'): paths.append(self.properties['oozie.libpath']) self.properties['oozie.libpath'] = ','.join(paths) oozie_xml = self.job.to_xml(self.properties) self._do_as(self.user.username, self._copy_files, deployment_dir, oozie_xml, self.properties) return deployment_dir