def _create_optional_descriptors(self, staged_component_path, component,
properties):
result = {}
if 'hbase.json' in component['component_detail']:
logging.debug("creating hbase.json")
self._fill_properties(
'%s/%s' % (staged_component_path, 'hbase.json'), properties)
hbase_descriptor.create(
'%s/%s' % (staged_component_path, 'hbase.json'),
self._environment)
if 'hdfs.json' in component['component_detail']:
logging.debug("creating hdfs.json")
descriptor_path = '%s/%s' % (staged_component_path, 'hdfs.json')
self._fill_properties(descriptor_path, properties)
with open(descriptor_path) as descriptor_file:
contents = descriptor_file.read()
hdfs_descriptor = json.loads(contents)
for element in hdfs_descriptor:
properties['hdfspath_' + element['name']] = element['path']
self._hdfs_client.make_dir(element['path'])
result['hdfs.json'] = hdfs_descriptor
if 'opentsdb.json' in component['component_detail']:
logging.debug("creating opentsdb.json")
self._fill_properties(
'%s/%s' % (staged_component_path, 'opentsdb.json'), properties)
opentsdb_descriptor.create(
'%s/%s' % (staged_component_path, 'opentsdb.json'),
self._environment)
return result
def test_with_impala_schema(self, pyhs_mock, hbase_mock):
my_text = '[{"table":"a_table", "col_family":"a_cf", "hive_schema":["some ddl","more ddl"]}]'
mocked_open_function = mock_open(read_data=my_text)
with patch("__builtin__.open", mocked_open_function):
environment = {
'hbase_rest_server': '1.2.3.4',
'hbase_rest_port': '2231',
'hive_server': '5.6.7.8',
'hive_port': '9876'}
hbase_descriptor.create("", environment)
hbase_mock.assert_called_once_with(host='1.2.3.4', port=2231)
hbase_mock.return_value.table.assert_called_once_with('a_table')
hbase_mock.return_value.table.return_value.create.assert_called_once_with('a_cf')
pyhs_mock.return_value.cursor.return_value.execute.assert_any_call('some ddl')
pyhs_mock.return_value.cursor.return_value.execute.assert_any_call('more ddl')
def test_normal_use(self, pyhs_mock, hbase_mock):
pyhs_mock.return_value = Mock()
my_text = '[{"table":"a_table", "col_family":"a_cf", "hive_schema":[]}]'
mocked_open_function = mock_open(read_data=my_text)
with patch("__builtin__.open", mocked_open_function):
environment = {
'hbase_rest_server': '1.2.3.4',
'hbase_rest_port': '2231',
'hive_server': '5.6.7.8',
'hive_port': '9876'}
hbase_descriptor.create("", environment)
hbase_mock.assert_called_once_with(host='1.2.3.4', port=2231)
hbase_mock.return_value.table.assert_called_once_with('a_table')
hbase_mock.return_value.table.return_value.create.assert_called_once_with('a_cf')
def test_normal_use(self, hbase_mock):
my_text = '[{"table":"a_table", "col_family":"a_cf", "hive_schema":[]}]'
mocked_open_function = mock_open(read_data=my_text)
with patch("__builtin__.open", mocked_open_function):
environment = {
'hbase_rest_server': '1.2.3.4',
'hbase_rest_port': '2231',
'hive_server': '5.6.7.8',
'hive_port': '9876'
}
hbase_descriptor.create("", environment)
hbase_mock.assert_called_once_with(host='1.2.3.4', port=2231)
hbase_mock.return_value.table.assert_called_once_with('a_table')
hbase_mock.return_value.table.return_value.create.assert_called_once_with(
'a_cf')
def test_with_impala_schema(self, pyhs_mock, hbase_mock):
my_text = '[{"table":"a_table", "col_family":"a_cf", "hive_schema":["some ddl","more ddl"]}]'
mocked_open_function = mock_open(read_data=my_text)
with patch("__builtin__.open", mocked_open_function):
environment = {
'hbase_rest_server': '1.2.3.4',
'hbase_rest_port': '2231',
'hive_server': '5.6.7.8',
'hive_port': '9876'
}
hbase_descriptor.create("", environment)
hbase_mock.assert_called_once_with(host='1.2.3.4', port=2231)
hbase_mock.return_value.table.assert_called_once_with('a_table')
hbase_mock.return_value.table.return_value.create.assert_called_once_with(
'a_cf')
pyhs_mock.return_value.cursor.return_value.execute.assert_any_call(
'some ddl')
pyhs_mock.return_value.cursor.return_value.execute.assert_any_call(
'more ddl')
def test_with_impala_schema(self, subprocess_mock, hbase_mock):
my_text = '[{"table":"a_table", "col_family":"a_cf", "hive_schema":["some ddl"]}]'
mocked_open_function = mock_open(read_data=my_text)
subprocess_mock.return_value = 'something'
with patch("__builtin__.open", mocked_open_function):
environment = {
'hbase_rest_server': '1.2.3.4',
'hbase_rest_port': '2231',
'hive_server': '5.6.7.8',
'hive_port': '9876'
}
hbase_descriptor.create("", environment)
hbase_mock.assert_called_once_with(host='1.2.3.4', port=2231)
hbase_mock.return_value.table.assert_called_once_with('a_table')
hbase_mock.return_value.table.return_value.create.assert_called_once_with(
'a_cf')
subprocess_mock.assert_called_once_with([
"beeline", "-u",
"jdbc:hive2://5.6.7.8:9876/;transportMode=http;httpPath=cliservice",
"-e", "some ddl"
])