def test_submit(self): """ Tests to verify that from connection extra option the options are added to the Sqoop command. """ hook = SqoopHook(**self._config) cmd = ' '.join(hook._prepare_command()) # Check if the config has been extracted from the json if self._config_json['namenode']: self.assertIn("-fs {}".format(self._config_json['namenode']), cmd) if self._config_json['job_tracker']: self.assertIn("-jt {}".format(self._config_json['job_tracker']), cmd) if self._config_json['libjars']: self.assertIn("-libjars {}".format(self._config_json['libjars']), cmd) if self._config_json['files']: self.assertIn("-files {}".format(self._config_json['files']), cmd) if self._config_json['archives']: self.assertIn("-archives {}".format(self._config_json['archives']), cmd) self.assertIn( "--hcatalog-database {}".format(self._config['hcatalog_database']), cmd) self.assertIn( "--hcatalog-table {}".format(self._config['hcatalog_table']), cmd) # Check the regulator stuff passed by the default constructor if self._config['verbose']: self.assertIn("--verbose", cmd) if self._config['num_mappers']: self.assertIn( "--num-mappers {}".format(self._config['num_mappers']), cmd) for key, value in self._config['properties'].items(): self.assertIn("-D {}={}".format(key, value), cmd) # We don't have the sqoop binary available, and this is hard to mock, # so just accept an exception for now. with self.assertRaises(OSError): hook.export_table(**self._config_export) with self.assertRaises(OSError): hook.import_table(table='schema.table', target_dir='/sqoop/example/path') with self.assertRaises(OSError): hook.import_query(query='SELECT * FROM sometable', target_dir='/sqoop/example/path')
def test_submit_none_mappers(self): """ Test to check that if value of num_mappers is None, then it shouldn't be in the cmd built. """ _config_without_mappers = self._config.copy() _config_without_mappers['num_mappers'] = None hook = SqoopHook(**_config_without_mappers) cmd = ' '.join(hook._prepare_command()) self.assertNotIn('--num-mappers', cmd)
def test_submit(self): hook = SqoopHook(**self._config) cmd = ' '.join(hook._prepare_command()) # Check if the config has been extracted from the json if self._config_json['namenode']: self.assertIn("-fs {}".format(self._config_json['namenode']), cmd) if self._config_json['job_tracker']: self.assertIn("-jt {}".format(self._config_json['job_tracker']), cmd) if self._config_json['libjars']: self.assertIn("-libjars {}".format(self._config_json['libjars']), cmd) if self._config_json['files']: self.assertIn("-files {}".format(self._config_json['files']), cmd) if self._config_json['archives']: self.assertIn( "-archives {}".format(self._config_json['archives']), cmd ) self.assertIn("--hcatalog-database {}".format(self._config['hcatalog_database']), cmd) self.assertIn("--hcatalog-table {}".format(self._config['hcatalog_table']), cmd) # Check the regulator stuff passed by the default constructor if self._config['verbose']: self.assertIn("--verbose", cmd) if self._config['num_mappers']: self.assertIn( "--num-mappers {}".format(self._config['num_mappers']), cmd ) print(self._config['properties']) for key, value in self._config['properties'].items(): self.assertIn("-D {}={}".format(key, value), cmd) # We don't have the sqoop binary available, and this is hard to mock, # so just accept an exception for now. with self.assertRaises(OSError): hook.export_table(**self._config_export) with self.assertRaises(OSError): hook.import_table(table='schema.table', target_dir='/sqoop/example/path') with self.assertRaises(OSError): hook.import_query(query='SELECT * FROM sometable', target_dir='/sqoop/example/path')