def test_update_properties(self): finish = [] finish.append(MR_CLUSTERS.set_for_testing({'default': {}})) finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True)) finish.append(YARN_CLUSTERS.set_for_testing({'default': {}})) finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True)) try: properties = { 'user.name': 'hue', 'test.1': 'http://localhost/test?test1=test&test2=test', 'nameNode': 'hdfs://curacao:8020', 'jobTracker': 'jtaddress', 'security_enabled': False } final_properties = properties.copy() submission = Submission(None, properties=properties, oozie_id='test', fs=MockFs()) assert_equal(properties, submission.properties) submission._update_properties('jtaddress', 'deployment-directory') assert_equal(final_properties, submission.properties) cluster.clear_caches() fs = cluster.get_hdfs() jt = cluster.get_next_ha_mrcluster()[1] final_properties = properties.copy() final_properties.update({ 'jobTracker': 'jtaddress', 'nameNode': fs.fs_defaultfs }) submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt) assert_equal(properties, submission.properties) submission._update_properties('jtaddress', 'deployment-directory') assert_equal(final_properties, submission.properties) finish.append(HDFS_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('namenode')) finish.append(MR_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('jobtracker')) cluster.clear_caches() fs = cluster.get_hdfs() jt = cluster.get_next_ha_mrcluster()[1] final_properties = properties.copy() final_properties.update({ 'jobTracker': 'jobtracker', 'nameNode': 'namenode' }) submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt) assert_equal(properties, submission.properties) submission._update_properties('jtaddress', 'deployment-directory') assert_equal(final_properties, submission.properties) finally: cluster.clear_caches() for reset in finish: reset()
def decorate(api, *args, **kwargs): try: return funct(api, *args, **kwargs) except Exception, ex: if 'Could not connect to' in str(ex): LOG.info('JobTracker not available, trying JT plugin HA: %s.' % ex) jt_ha = get_next_ha_mrcluster() if jt_ha is not None: config, api.jt = jt_ha return funct(api, *args, **kwargs) raise ex
def test_update_properties(self): finish = [] finish.append(MR_CLUSTERS.set_for_testing({'default': {}})) finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True)) finish.append(YARN_CLUSTERS.set_for_testing({'default': {}})) finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True)) try: properties = { 'user.name': 'hue', 'test.1': 'http://localhost/test?test1=test&test2=test', 'nameNode': 'hdfs://curacao:8020', 'jobTracker': 'jtaddress', 'security_enabled': False } final_properties = properties.copy() submission = Submission(None, properties=properties, oozie_id='test', fs=MockFs()) assert_equal(properties, submission.properties) submission._update_properties('jtaddress', 'deployment-directory') assert_equal(final_properties, submission.properties) clear_sys_caches() fs = cluster.get_hdfs() jt = cluster.get_next_ha_mrcluster()[1] final_properties = properties.copy() final_properties.update({ 'jobTracker': 'jtaddress', 'nameNode': fs.fs_defaultfs }) submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt) assert_equal(properties, submission.properties) submission._update_properties('jtaddress', 'deployment-directory') assert_equal(final_properties, submission.properties) finish.append( HDFS_CLUSTERS['default'].LOGICAL_NAME.set_for_testing( 'namenode')) finish.append(MR_CLUSTERS['default'].LOGICAL_NAME.set_for_testing( 'jobtracker')) clear_sys_caches() fs = cluster.get_hdfs() jt = cluster.get_next_ha_mrcluster()[1] final_properties = properties.copy() final_properties.update({ 'jobTracker': 'jobtracker', 'nameNode': 'namenode' }) submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt) assert_equal(properties, submission.properties) submission._update_properties('jtaddress', 'deployment-directory') assert_equal(final_properties, submission.properties) finally: clear_sys_caches() for reset in finish: reset()