def get_query_server_config(server=None): if not server or server not in DATABASES: keys = DATABASES.keys() name = keys and keys[0] or None else: name = server if name: query_server = { 'server_name': DATABASES[name].ENGINE.get().split('.')[-1], 'server_host': DATABASES[name].HOST.get(), 'server_port': DATABASES[name].PORT.get(), 'username': DATABASES[name].USER.get(), 'password': get_database_password(name), 'options': force_dict_to_strings(DATABASES[name].OPTIONS.get()), 'alias': name } if DATABASES[name].NAME.get(): query_server['name'] = DATABASES[name].NAME.get() else: query_server = {} LOG.debug("Query Server: %s" % query_server) return query_server
def get_query_server_config(server=None): if not server or server not in DATABASES: keys = DATABASES.keys() name = keys and keys[0] or None else: name = server if name: query_server = { "server_name": DATABASES[name].ENGINE.get().split(".")[-1], "server_host": DATABASES[name].HOST.get(), "server_port": DATABASES[name].PORT.get(), "username": DATABASES[name].USER.get(), "password": get_database_password(name), "options": force_dict_to_strings(DATABASES[name].OPTIONS.get()), "alias": name, } if DATABASES[name].NAME.get(): query_server["name"] = DATABASES[name].NAME.get() else: query_server = {} LOG.debug("Query Server: %s" % query_server) return query_server
def run_sqoop(request, source, destination, start_time): rdbms_mode = source['rdbmsMode'] rdbms_name = source['rdbmsType'] rdbms_database_name = source['rdbmsDatabaseName'] rdbms_all_tables_selected = source['rdbmsAllTablesSelected'] destination_type = destination['outputFormat'] destination_name = destination['name'] destination_table_name = destination['tableName'] destination_database_name = destination['databaseName'] destination_mappers_num = destination['numMappers'] if not rdbms_all_tables_selected: rdbms_table_name = source['rdbmsTableName'] if rdbms_mode == 'configRdbms': rdbms_host = DATABASES[rdbms_name].HOST.get() rdbms_port = DATABASES[rdbms_name].PORT.get() rdbms_user_name = DATABASES[rdbms_name].USER.get() rdbms_password = get_database_password(rdbms_name) else: rdbms_host = source['rdbmsHostname'] rdbms_port = source['rdbmsPort'] rdbms_user_name = source['rdbmsUsername'] rdbms_password = source['rdbmsPassword'] statement = '--connect jdbc:%(rdbmsName)s://%(rdbmsHost)s:%(rdbmsPort)s/%(rdbmsDatabaseName)s --username %(rdbmsUserName)s --password %(rdbmsPassword)s' % { 'rdbmsName': rdbms_name, 'rdbmsHost': rdbms_host, 'rdbmsPort': rdbms_port, 'rdbmsDatabaseName': rdbms_database_name, 'rdbmsUserName': rdbms_user_name, 'rdbmsPassword': rdbms_password } if destination_type == 'file': success_url = '/filebrowser/view/' + destination_name targetDir = request.fs.fs_defaultfs + destination_name if rdbms_all_tables_selected: statement = 'import-all-tables %(statement)s --warehouse-dir %(targetDir)s' % { 'statement': statement, 'targetDir': targetDir } else: statement = 'import %(statement)s --table %(rdbmsTableName)s --delete-target-dir --target-dir %(targetDir)s' % { 'statement': statement, 'rdbmsTableName': rdbms_table_name, 'targetDir': targetDir } elif destination_type == 'table': success_url = reverse('metastore:describe_table', kwargs={ 'database': destination_database_name, 'table': destination_table_name }) if rdbms_all_tables_selected: statement = 'import-all-tables %(statement)s --hive-import' % { 'statement': statement } else: statement = 'import %(statement)s --table %(rdbmsTableName)s --hive-import' % { 'statement': statement, 'rdbmsTableName': rdbms_table_name } elif destination_type == 'hbase': success_url = '/hbase/#HBase/' + destination_table_name # Todo statement = '%(statement)s --num-mappers %(numMappers)s' % { 'statement': statement, 'numMappers': destination_mappers_num } lib_files = [] if destination['sqoopJobLibPaths']: lib_files = [{ 'path': f['path'], 'type': 'jar' } for f in destination['sqoopJobLibPaths']] task = make_notebook(name=_( 'Indexer job for %(rdbmsDatabaseName)s.%(rdbmsDatabaseName)s to %(path)s' ) % { 'rdbmsDatabaseName': rdbms_database_name, 'rdbmsDatabaseName': rdbms_database_name, 'path': destination_name }, editor_type='sqoop1', statement=statement, files=lib_files, status='ready', on_success_url=success_url, last_executed=start_time, is_task=True) return task.execute(request, batch=False)
def run_sqoop(request, source, destination, start_time): rdbms_mode = source['rdbmsMode'] rdbms_name = source['rdbmsJdbcDriverName'] if source[ 'rdbmsType'] == 'jdbc' else source['rdbmsType'] rdbms_database_name = source['rdbmsDatabaseName'] rdbms_all_tables_selected = source['rdbmsAllTablesSelected'] destination_type = destination['outputFormat'] destination_name = destination['name'] destination_table_name = destination['tableName'] destination_database_name = destination['databaseName'] destination_mappers_num = destination['numMappers'] destination_file_output_format = destination['rdbmsFileOutputFormat'] destination_custom_fields_delimiter = destination['customFieldsDelimiter'] destination_custom_line_delimiter = destination['customLineDelimiter'] destination_custom_enclosed_by_delimiter = destination[ 'customEnclosedByDelimiter'] destination_splitby_column = destination['rdbmsSplitByColumn'] if not rdbms_all_tables_selected: rdbms_table_name = source['rdbmsTableName'] if rdbms_mode == 'configRdbms' and rdbms_name == 'jdbc': username = '' password = '' url = '' interpreters = get_ordered_interpreters(request.user) key = [ key for key in interpreters if key['name'] == source['rdbmsJdbcDriverName'] ] if key: options = key[0]['options'] url = options['url'] password = options['password'] username = options['user'] statement = '--connect %(url)s --username %(username)s --password %(password)s' % { 'url': url, 'username': username, 'password': password } else: if rdbms_mode == 'configRdbms': rdbms_host = DATABASES[rdbms_name].HOST.get() rdbms_port = DATABASES[rdbms_name].PORT.get() rdbms_user_name = DATABASES[rdbms_name].USER.get() rdbms_password = get_database_password(rdbms_name) else: rdbms_host = source['rdbmsHostname'] rdbms_port = source['rdbmsPort'] rdbms_user_name = source['rdbmsUsername'] rdbms_password = source['rdbmsPassword'] password_file_path = request.fs.join( request.fs.get_home_dir() + '/sqoop/', uuid.uuid4().hex + '.password') request.fs.do_as_user(request.user, request.fs.create, password_file_path, overwrite=True, permission=0700, data=smart_str(rdbms_password)) lib_files = [] if destination['sqoopJobLibPaths']: lib_files = [{ 'path': f['path'], 'type': 'jar' } for f in destination['sqoopJobLibPaths']] statement = '--connect jdbc:%(rdbmsName)s://%(rdbmsHost)s:%(rdbmsPort)s/%(rdbmsDatabaseName)s --username %(rdbmsUserName)s --password-file %(passwordFilePath)s' % { 'rdbmsName': rdbms_name, 'rdbmsHost': rdbms_host, 'rdbmsPort': rdbms_port, 'rdbmsDatabaseName': rdbms_database_name, 'rdbmsUserName': rdbms_user_name, 'passwordFilePath': password_file_path } if destination_type == 'file': success_url = '/filebrowser/view/' + destination_name targetDir = request.fs.fs_defaultfs + destination_name if rdbms_all_tables_selected: statement = 'import-all-tables %(statement)s --warehouse-dir %(targetDir)s' % { 'statement': statement, 'targetDir': targetDir } else: statement = 'import %(statement)s --table %(rdbmsTableName)s --delete-target-dir --target-dir %(targetDir)s' % { 'statement': statement, 'rdbmsTableName': rdbms_table_name, 'targetDir': targetDir } if destination_file_output_format == 'text': statement = '%(statement)s --as-textfile --fields-terminated-by %(customFieldsDelimiter)s --lines-terminated-by %(customLineDelimiter)s --enclosed-by %(customEnclosedByDelimiter)s' % { 'statement': statement, 'customFieldsDelimiter': destination_custom_fields_delimiter, 'customLineDelimiter': destination_custom_line_delimiter, 'customEnclosedByDelimiter': destination_custom_enclosed_by_delimiter } elif destination_file_output_format == 'sequence': statement = '%(statement)s --as-sequencefile' % { 'statement': statement } elif destination_file_output_format == 'avro': statement = '%(statement)s --as-avrodatafile' % { 'statement': statement } statement = _splitby_column_check(statement, destination_splitby_column) elif destination_type == 'table': success_url = reverse('metastore:describe_table', kwargs={ 'database': destination_database_name, 'table': destination_table_name }) if rdbms_all_tables_selected: statement = 'import-all-tables %(statement)s --hive-import' % { 'statement': statement } else: statement = 'import %(statement)s --table %(rdbmsTableName)s --hive-import' % { 'statement': statement, 'rdbmsTableName': rdbms_table_name } statement = _splitby_column_check(statement, destination_splitby_column) elif destination_type == 'hbase': success_url = '/hbase/#HBase/' + destination_table_name # Todo statement = '%(statement)s --num-mappers %(numMappers)s' % { 'statement': statement, 'numMappers': destination_mappers_num } task = make_notebook(name=_( 'Indexer job for %(rdbmsDatabaseName)s.%(rdbmsDatabaseName)s to %(path)s' ) % { 'rdbmsDatabaseName': rdbms_database_name, 'rdbmsDatabaseName': rdbms_database_name, 'path': destination_name }, editor_type='sqoop1', statement=statement, files=lib_files, status='ready', on_success_url=success_url, last_executed=start_time, is_task=True) return task.execute(request, batch=False)