def test_submit(self):
"""
Tests to verify that from connection extra option the options are added to the Sqoop command.
"""
hook = SqoopHook(**self._config)
cmd = ' '.join(hook._prepare_command())
# Check if the config has been extracted from the json
if self._config_json['namenode']:
self.assertIn("-fs {}".format(self._config_json['namenode']), cmd)
if self._config_json['job_tracker']:
self.assertIn("-jt {}".format(self._config_json['job_tracker']),
cmd)
if self._config_json['libjars']:
self.assertIn("-libjars {}".format(self._config_json['libjars']),
cmd)
if self._config_json['files']:
self.assertIn("-files {}".format(self._config_json['files']), cmd)
if self._config_json['archives']:
self.assertIn("-archives {}".format(self._config_json['archives']),
cmd)
self.assertIn(
"--hcatalog-database {}".format(self._config['hcatalog_database']),
cmd)
self.assertIn(
"--hcatalog-table {}".format(self._config['hcatalog_table']), cmd)
# Check the regulator stuff passed by the default constructor
if self._config['verbose']:
self.assertIn("--verbose", cmd)
if self._config['num_mappers']:
self.assertIn(
"--num-mappers {}".format(self._config['num_mappers']), cmd)
for key, value in self._config['properties'].items():
self.assertIn("-D {}={}".format(key, value), cmd)
# We don't have the sqoop binary available, and this is hard to mock,
# so just accept an exception for now.
with self.assertRaises(OSError):
hook.export_table(**self._config_export)
with self.assertRaises(OSError):
hook.import_table(table='schema.table',
target_dir='/sqoop/example/path')
with self.assertRaises(OSError):
hook.import_query(query='SELECT * FROM sometable',
target_dir='/sqoop/example/path')
def test_submit_none_mappers(self):
"""
Test to check that if value of num_mappers is None, then it shouldn't be in the cmd built.
"""
_config_without_mappers = self._config.copy()
_config_without_mappers['num_mappers'] = None
hook = SqoopHook(**_config_without_mappers)
cmd = ' '.join(hook._prepare_command())
self.assertNotIn('--num-mappers', cmd)
def test_submit_none_mappers(self):
"""
Test to check that if value of num_mappers is None, then it shouldn't be in the cmd built.
"""
_config_without_mappers = self._config.copy()
_config_without_mappers['num_mappers'] = None
hook = SqoopHook(**_config_without_mappers)
cmd = ' '.join(hook._prepare_command())
self.assertNotIn('--num-mappers', cmd)
def test_submit(self):
hook = SqoopHook(**self._config)
cmd = ' '.join(hook._prepare_command())
# Check if the config has been extracted from the json
if self._config_json['namenode']:
self.assertIn("-fs {}".format(self._config_json['namenode']), cmd)
if self._config_json['job_tracker']:
self.assertIn("-jt {}".format(self._config_json['job_tracker']),
cmd)
if self._config_json['libjars']:
self.assertIn("-libjars {}".format(self._config_json['libjars']),
cmd)
if self._config_json['files']:
self.assertIn("-files {}".format(self._config_json['files']), cmd)
if self._config_json['archives']:
self.assertIn(
"-archives {}".format(self._config_json['archives']), cmd
)
self.assertIn("--hcatalog-database {}".format(self._config['hcatalog_database']), cmd)
self.assertIn("--hcatalog-table {}".format(self._config['hcatalog_table']), cmd)
# Check the regulator stuff passed by the default constructor
if self._config['verbose']:
self.assertIn("--verbose", cmd)
if self._config['num_mappers']:
self.assertIn(
"--num-mappers {}".format(self._config['num_mappers']), cmd
)
print(self._config['properties'])
for key, value in self._config['properties'].items():
self.assertIn("-D {}={}".format(key, value), cmd)
# We don't have the sqoop binary available, and this is hard to mock,
# so just accept an exception for now.
with self.assertRaises(OSError):
hook.export_table(**self._config_export)
with self.assertRaises(OSError):
hook.import_table(table='schema.table',
target_dir='/sqoop/example/path')
with self.assertRaises(OSError):
hook.import_query(query='SELECT * FROM sometable',
target_dir='/sqoop/example/path')