def get_query_server_config(server=None):
if not server or server not in DATABASES:
keys = DATABASES.keys()
name = keys and keys[0] or None
else:
name = server
if name:
query_server = {
'server_name': DATABASES[name].ENGINE.get().split('.')[-1],
'server_host': DATABASES[name].HOST.get(),
'server_port': DATABASES[name].PORT.get(),
'username': DATABASES[name].USER.get(),
'password': get_database_password(name),
'options': force_dict_to_strings(DATABASES[name].OPTIONS.get()),
'alias': name
}
if DATABASES[name].NAME.get():
query_server['name'] = DATABASES[name].NAME.get()
else:
query_server = {}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(server=None):
if not server or server not in DATABASES:
keys = DATABASES.keys()
name = keys and keys[0] or None
else:
name = server
if name:
query_server = {
"server_name": DATABASES[name].ENGINE.get().split(".")[-1],
"server_host": DATABASES[name].HOST.get(),
"server_port": DATABASES[name].PORT.get(),
"username": DATABASES[name].USER.get(),
"password": get_database_password(name),
"options": force_dict_to_strings(DATABASES[name].OPTIONS.get()),
"alias": name,
}
if DATABASES[name].NAME.get():
query_server["name"] = DATABASES[name].NAME.get()
else:
query_server = {}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(server=None):
if not server or server not in DATABASES:
keys = DATABASES.keys()
name = keys and keys[0] or None
else:
name = server
if name:
query_server = {
'server_name': DATABASES[name].ENGINE.get().split('.')[-1],
'server_host': DATABASES[name].HOST.get(),
'server_port': DATABASES[name].PORT.get(),
'username': DATABASES[name].USER.get(),
'password': get_database_password(name),
'options': force_dict_to_strings(DATABASES[name].OPTIONS.get()),
'alias': name
}
if DATABASES[name].NAME.get():
query_server['name'] = DATABASES[name].NAME.get()
else:
query_server = {}
LOG.debug("Query Server: %s" % query_server)
return query_server
def run_sqoop(request, source, destination, start_time):
rdbms_mode = source['rdbmsMode']
rdbms_name = source['rdbmsType']
rdbms_database_name = source['rdbmsDatabaseName']
rdbms_all_tables_selected = source['rdbmsAllTablesSelected']
destination_type = destination['outputFormat']
destination_name = destination['name']
destination_table_name = destination['tableName']
destination_database_name = destination['databaseName']
destination_mappers_num = destination['numMappers']
if not rdbms_all_tables_selected:
rdbms_table_name = source['rdbmsTableName']
if rdbms_mode == 'configRdbms':
rdbms_host = DATABASES[rdbms_name].HOST.get()
rdbms_port = DATABASES[rdbms_name].PORT.get()
rdbms_user_name = DATABASES[rdbms_name].USER.get()
rdbms_password = get_database_password(rdbms_name)
else:
rdbms_host = source['rdbmsHostname']
rdbms_port = source['rdbmsPort']
rdbms_user_name = source['rdbmsUsername']
rdbms_password = source['rdbmsPassword']
statement = '--connect jdbc:%(rdbmsName)s://%(rdbmsHost)s:%(rdbmsPort)s/%(rdbmsDatabaseName)s --username %(rdbmsUserName)s --password %(rdbmsPassword)s' % {
'rdbmsName': rdbms_name,
'rdbmsHost': rdbms_host,
'rdbmsPort': rdbms_port,
'rdbmsDatabaseName': rdbms_database_name,
'rdbmsUserName': rdbms_user_name,
'rdbmsPassword': rdbms_password
}
if destination_type == 'file':
success_url = '/filebrowser/view/' + destination_name
targetDir = request.fs.fs_defaultfs + destination_name
if rdbms_all_tables_selected:
statement = 'import-all-tables %(statement)s --warehouse-dir %(targetDir)s' % {
'statement': statement,
'targetDir': targetDir
}
else:
statement = 'import %(statement)s --table %(rdbmsTableName)s --delete-target-dir --target-dir %(targetDir)s' % {
'statement': statement,
'rdbmsTableName': rdbms_table_name,
'targetDir': targetDir
}
elif destination_type == 'table':
success_url = reverse('metastore:describe_table',
kwargs={
'database': destination_database_name,
'table': destination_table_name
})
if rdbms_all_tables_selected:
statement = 'import-all-tables %(statement)s --hive-import' % {
'statement': statement
}
else:
statement = 'import %(statement)s --table %(rdbmsTableName)s --hive-import' % {
'statement': statement,
'rdbmsTableName': rdbms_table_name
}
elif destination_type == 'hbase':
success_url = '/hbase/#HBase/' + destination_table_name
# Todo
statement = '%(statement)s --num-mappers %(numMappers)s' % {
'statement': statement,
'numMappers': destination_mappers_num
}
lib_files = []
if destination['sqoopJobLibPaths']:
lib_files = [{
'path': f['path'],
'type': 'jar'
} for f in destination['sqoopJobLibPaths']]
task = make_notebook(name=_(
'Indexer job for %(rdbmsDatabaseName)s.%(rdbmsDatabaseName)s to %(path)s'
) % {
'rdbmsDatabaseName': rdbms_database_name,
'rdbmsDatabaseName': rdbms_database_name,
'path': destination_name
},
editor_type='sqoop1',
statement=statement,
files=lib_files,
status='ready',
on_success_url=success_url,
last_executed=start_time,
is_task=True)
return task.execute(request, batch=False)
def run_sqoop(request, source, destination, start_time):
rdbms_mode = source['rdbmsMode']
rdbms_name = source['rdbmsJdbcDriverName'] if source[
'rdbmsType'] == 'jdbc' else source['rdbmsType']
rdbms_database_name = source['rdbmsDatabaseName']
rdbms_all_tables_selected = source['rdbmsAllTablesSelected']
destination_type = destination['outputFormat']
destination_name = destination['name']
destination_table_name = destination['tableName']
destination_database_name = destination['databaseName']
destination_mappers_num = destination['numMappers']
destination_file_output_format = destination['rdbmsFileOutputFormat']
destination_custom_fields_delimiter = destination['customFieldsDelimiter']
destination_custom_line_delimiter = destination['customLineDelimiter']
destination_custom_enclosed_by_delimiter = destination[
'customEnclosedByDelimiter']
destination_splitby_column = destination['rdbmsSplitByColumn']
if not rdbms_all_tables_selected:
rdbms_table_name = source['rdbmsTableName']
if rdbms_mode == 'configRdbms' and rdbms_name == 'jdbc':
username = ''
password = ''
url = ''
interpreters = get_ordered_interpreters(request.user)
key = [
key for key in interpreters
if key['name'] == source['rdbmsJdbcDriverName']
]
if key:
options = key[0]['options']
url = options['url']
password = options['password']
username = options['user']
statement = '--connect %(url)s --username %(username)s --password %(password)s' % {
'url': url,
'username': username,
'password': password
}
else:
if rdbms_mode == 'configRdbms':
rdbms_host = DATABASES[rdbms_name].HOST.get()
rdbms_port = DATABASES[rdbms_name].PORT.get()
rdbms_user_name = DATABASES[rdbms_name].USER.get()
rdbms_password = get_database_password(rdbms_name)
else:
rdbms_host = source['rdbmsHostname']
rdbms_port = source['rdbmsPort']
rdbms_user_name = source['rdbmsUsername']
rdbms_password = source['rdbmsPassword']
password_file_path = request.fs.join(
request.fs.get_home_dir() + '/sqoop/',
uuid.uuid4().hex + '.password')
request.fs.do_as_user(request.user,
request.fs.create,
password_file_path,
overwrite=True,
permission=0700,
data=smart_str(rdbms_password))
lib_files = []
if destination['sqoopJobLibPaths']:
lib_files = [{
'path': f['path'],
'type': 'jar'
} for f in destination['sqoopJobLibPaths']]
statement = '--connect jdbc:%(rdbmsName)s://%(rdbmsHost)s:%(rdbmsPort)s/%(rdbmsDatabaseName)s --username %(rdbmsUserName)s --password-file %(passwordFilePath)s' % {
'rdbmsName': rdbms_name,
'rdbmsHost': rdbms_host,
'rdbmsPort': rdbms_port,
'rdbmsDatabaseName': rdbms_database_name,
'rdbmsUserName': rdbms_user_name,
'passwordFilePath': password_file_path
}
if destination_type == 'file':
success_url = '/filebrowser/view/' + destination_name
targetDir = request.fs.fs_defaultfs + destination_name
if rdbms_all_tables_selected:
statement = 'import-all-tables %(statement)s --warehouse-dir %(targetDir)s' % {
'statement': statement,
'targetDir': targetDir
}
else:
statement = 'import %(statement)s --table %(rdbmsTableName)s --delete-target-dir --target-dir %(targetDir)s' % {
'statement': statement,
'rdbmsTableName': rdbms_table_name,
'targetDir': targetDir
}
if destination_file_output_format == 'text':
statement = '%(statement)s --as-textfile --fields-terminated-by %(customFieldsDelimiter)s --lines-terminated-by %(customLineDelimiter)s --enclosed-by %(customEnclosedByDelimiter)s' % {
'statement':
statement,
'customFieldsDelimiter':
destination_custom_fields_delimiter,
'customLineDelimiter':
destination_custom_line_delimiter,
'customEnclosedByDelimiter':
destination_custom_enclosed_by_delimiter
}
elif destination_file_output_format == 'sequence':
statement = '%(statement)s --as-sequencefile' % {
'statement': statement
}
elif destination_file_output_format == 'avro':
statement = '%(statement)s --as-avrodatafile' % {
'statement': statement
}
statement = _splitby_column_check(statement,
destination_splitby_column)
elif destination_type == 'table':
success_url = reverse('metastore:describe_table',
kwargs={
'database': destination_database_name,
'table': destination_table_name
})
if rdbms_all_tables_selected:
statement = 'import-all-tables %(statement)s --hive-import' % {
'statement': statement
}
else:
statement = 'import %(statement)s --table %(rdbmsTableName)s --hive-import' % {
'statement': statement,
'rdbmsTableName': rdbms_table_name
}
statement = _splitby_column_check(statement,
destination_splitby_column)
elif destination_type == 'hbase':
success_url = '/hbase/#HBase/' + destination_table_name
# Todo
statement = '%(statement)s --num-mappers %(numMappers)s' % {
'statement': statement,
'numMappers': destination_mappers_num
}
task = make_notebook(name=_(
'Indexer job for %(rdbmsDatabaseName)s.%(rdbmsDatabaseName)s to %(path)s'
) % {
'rdbmsDatabaseName': rdbms_database_name,
'rdbmsDatabaseName': rdbms_database_name,
'path': destination_name
},
editor_type='sqoop1',
statement=statement,
files=lib_files,
status='ready',
on_success_url=success_url,
last_executed=start_time,
is_task=True)
return task.execute(request, batch=False)