def create_session(self, lang='scala', properties=None): if not properties and USE_DEFAULT_CONFIGURATION.get(): user_config = DefaultConfiguration.objects.get_configuration_for_user( app='spark', user=self.user) if user_config is not None: properties = user_config.properties_list props = self.get_livy_props(lang, properties) api = get_spark_api(self.user) response = api.create_session(**props) status = api.get_session(response['id']) count = 0 while status['state'] == 'starting' and count < 120: status = api.get_session(response['id']) count += 1 time.sleep(1) if status['state'] != 'idle': info = '\n'.join(status['log']) if status['log'] else 'timeout' raise QueryError( _('The Spark session is %s and could not be created in the cluster: %s' ) % (status['state'], info)) return { 'type': lang, 'id': response['id'], 'properties': self.to_properties(props) }
def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' else lang session = Session.objects.get_session(self.user, application=application) if session is None: session = dbms.get(self.user, query_server=get_query_server_config( name=lang)).open_session(self.user) response = {'type': lang, 'id': session.id} if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user( app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties if lang == 'impala': http_addr = self._get_impala_server_url(session) response['http_addr'] = http_addr return response
def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' else lang session = Session.objects.get_session(self.user, application=application) if session is None: session = dbms.get(self.user, query_server=get_query_server_config(name=lang)).open_session(self.user) response = { 'type': lang, 'id': session.id } if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user(app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties if lang == 'impala': impala_settings = session.get_formatted_properties() http_addr = next((setting['value'] for setting in impala_settings if setting['key'].lower() == 'http_addr'), None) response['http_addr'] = http_addr return response
def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' else lang session = Session.objects.get_session(self.user, application=application) reuse_session = session is not None if not reuse_session: session = dbms.get(self.user, query_server=get_query_server_config( name=lang)).open_session(self.user) response = {'type': lang, 'id': session.id} if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user( app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties response['reuse_session'] = reuse_session response['session_id'] = '' try: decoded_guid = session.get_handle().sessionId.guid response['session_id'] = "%x:%x" % struct.unpack( b"QQ", decoded_guid) except Exception, e: LOG.warn('Failed to decode session handle: %s' % e)
def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang if has_session_pool(): session = Session.objects.get_tez_session( self.user, application, MAX_NUMBER_OF_SESSIONS.get()) elif not has_multiple_sessions(): session = Session.objects.get_session(self.user, application=application) else: session = None reuse_session = session is not None if not reuse_session: db = dbms.get(self.user, query_server=get_query_server_config( name=lang, connector=self.interpreter)) session = db.open_session(self.user) response = {'type': lang, 'id': session.id} if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user( app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties response['configuration'] = json.loads(session.properties) response['reuse_session'] = reuse_session response['session_id'] = '' try: decoded_guid = session.get_handle().sessionId.guid response['session_id'] = unpack_guid(decoded_guid) except Exception as e: LOG.warn('Failed to decode session handle: %s' % e) if lang == 'impala' and session: http_addr = _get_impala_server_url(session) response['http_addr'] = http_addr return response
def create_session(self, lang='scala', properties=None): if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user(app='spark', user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties() props = dict([(p['name'], p['value']) for p in properties]) if properties is not None else {} props['kind'] = lang api = get_spark_api(self.user) response = api.create_session(**props) status = api.get_session(response['id']) count = 0 while status['state'] == 'starting' and count < 120: status = api.get_session(response['id']) count += 1 time.sleep(1) if status['state'] != 'idle': info = '\n'.join(status['log']) if status['log'] else 'timeout' raise QueryError(_('The Spark session could not be created in the cluster: %s') % info) return { 'type': lang, 'id': response['id'], 'properties': properties }
def create_session(self, lang='scala', properties=None): if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user( app='spark', user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties() props = dict([(p['name'], p['value']) for p in properties]) if properties is not None else {} # HUE-4761: Hue's session request is causing Livy to fail with "JsonMappingException: Can not deserialize # instance of scala.collection.immutable.List out of VALUE_STRING token" due to List type values # not being formed properly, they are quoted csv strings (without brackets) instead of proper List # types, this is for keys; archives, jars, files and pyFiles. The Mako frontend probably should be # modified to pass the values as Livy expects but for now we coerce these types to be Lists. # Issue only occurs when non-default values are used because the default path properly sets the # empty list '[]' for these four values. # Note also that Livy has a 90 second timeout for the session request to complete, this needs to # be increased for requests that take longer, for example when loading large archives. tmparchives = props['archives'] if type(tmparchives) is not list: props['archives'] = tmparchives.split(",") LOG.debug("Check List type: archives was not a list") tmpjars = props['jars'] if type(tmpjars) is not list: props['jars'] = tmpjars.split(",") LOG.debug("Check List type: jars was not a list") tmpfiles = props['files'] if type(tmpfiles) is not list: props['files'] = tmpfiles.split(",") LOG.debug("Check List type: files was not a list") tmppyFiles = props['pyFiles'] if type(tmppyFiles) is not list: props['pyFiles'] = tmppyFiles.split(",") LOG.debug("Check List type: pyFiles was not a list") # Convert the conf list to a dict for Livy listitems = props['conf'] LOG.debug("Property Spark Conf kvp list from UI is: " + str(listitems)) confDict = {} for i in range(len(listitems)): kvp = listitems[i] LOG.debug("Property Spark Conf key " + str(i) + " = " + str(kvp.get('key'))) LOG.debug("Property Spark Conf value " + str(i) + " = " + str(kvp.get('value'))) confDict[kvp.get('key')] = kvp.get('value') props['conf'] = confDict LOG.debug("Property Spark Conf dictionary is: " + str(confDict)) props['kind'] = lang api = get_spark_api(self.user) response = api.create_session(**props) status = api.get_session(response['id']) count = 0 while status['state'] == 'starting' and count < 120: status = api.get_session(response['id']) count += 1 time.sleep(1) if status['state'] != 'idle': info = '\n'.join(status['log']) if status['log'] else 'timeout' raise QueryError( _('The Spark session could not be created in the cluster: %s') % info) return {'type': lang, 'id': response['id'], 'properties': properties}
def create_session(self, lang='hive', properties=None): application = 'beeswax' if lang == 'hive' or lang == 'llap' else lang uses_session_pool = has_session_pool() uses_multiple_sessions = has_multiple_sessions() if lang == 'impala': uses_session_pool = False uses_multiple_sessions = False try: if uses_session_pool: session = Session.objects.get_tez_session( self.user, application, MAX_NUMBER_OF_SESSIONS.get()) elif not uses_multiple_sessions: session = Session.objects.get_session(self.user, application=application) else: session = None except Exception as e: if 'Connection refused' in str( e) or 'Name or service not known' in str(e): LOG.exception( 'Connection being refused or service is not available in either session or in multiple sessions' '- HA failover') reset_ha() reuse_session = session is not None if not reuse_session: db = dbms.get(self.user, query_server=get_query_server_config( name=lang, connector=self.interpreter)) try: session = db.open_session(self.user) except Exception as e: if 'Connection refused' in str( e) or 'Name or service not known' in str(e): LOG.exception( 'Connection being refused or service is not available in reuse session - HA failover' ) reset_ha() response = {'type': lang, 'id': session.id} if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user( app=lang, user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties(lang) response['properties'] = properties response['configuration'] = json.loads(session.properties) response['reuse_session'] = reuse_session response['session_id'] = '' try: decoded_guid = session.get_handle().sessionId.guid response['session_id'] = unpack_guid(decoded_guid) except Exception as e: LOG.warning('Failed to decode session handle: %s' % e) if lang == 'impala' and session: http_addr = _get_impala_server_url(session) response['http_addr'] = http_addr return response
def create_session(self, lang='scala', properties=None): if not properties: config = None if USE_DEFAULT_CONFIGURATION.get(): config = DefaultConfiguration.objects.get_configuration_for_user(app='spark', user=self.user) if config is not None: properties = config.properties_list else: properties = self.get_properties() props = dict([(p['name'], p['value']) for p in properties]) if properties is not None else {} # HUE-4761: Hue's session request is causing Livy to fail with "JsonMappingException: Can not deserialize # instance of scala.collection.immutable.List out of VALUE_STRING token" due to List type values # not being formed properly, they are quoted csv strings (without brackets) instead of proper List # types, this is for keys; archives, jars, files and pyFiles. The Mako frontend probably should be # modified to pass the values as Livy expects but for now we coerce these types to be Lists. # Issue only occurs when non-default values are used because the default path properly sets the # empty list '[]' for these four values. # Note also that Livy has a 90 second timeout for the session request to complete, this needs to # be increased for requests that take longer, for example when loading large archives. tmparchives = props['archives'] if type(tmparchives) is not list: props['archives'] = tmparchives.split(",") LOG.debug("Check List type: archives was not a list") tmpjars = props['jars'] if type(tmpjars) is not list: props['jars'] = tmpjars.split(",") LOG.debug("Check List type: jars was not a list") tmpfiles = props['files'] if type(tmpfiles) is not list: props['files'] = tmpfiles.split(",") LOG.debug("Check List type: files was not a list") tmppyFiles = props['pyFiles'] if type(tmppyFiles) is not list: props['pyFiles'] = tmppyFiles.split(",") LOG.debug("Check List type: pyFiles was not a list") props['kind'] = lang api = get_spark_api(self.user) response = api.create_session(**props) status = api.get_session(response['id']) count = 0 while status['state'] == 'starting' and count < 120: status = api.get_session(response['id']) count += 1 time.sleep(1) if status['state'] != 'idle': info = '\n'.join(status['log']) if status['log'] else 'timeout' raise QueryError(_('The Spark session could not be created in the cluster: %s') % info) return { 'type': lang, 'id': response['id'], 'properties': properties }