def test_config_validator_more():
# TODO: Setup DN to not load the plugin, which is a common user error.
# We don't actually use the mini_cluster. But the cluster sets up the correct
# configuration that forms the test basis.
minicluster = pseudo_hdfs4.shared_cluster()
cli = make_logged_in_client()
reset = (
conf.MR_CLUSTERS["default"].HOST.set_for_testing("localhost"),
conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(23),
)
old = cluster.clear_caches()
try:
resp = cli.get('/debug/check_config')
assert_false('Failed to access filesystem root' in resp.content)
assert_false('Failed to create' in resp.content)
assert_false('Failed to chown' in resp.content)
assert_false('Failed to delete' in resp.content)
assert_true('Failed to contact JobTracker plugin' in resp.content)
finally:
for old_conf in reset:
old_conf()
cluster.restore_caches(old)
def test_config_validator_more():
# TODO: Setup DN to not load the plugin, which is a common user error.
# We don't actually use the mini_cluster. But the cluster sets up the correct
# configuration that forms the test basis.
minicluster = pseudo_hdfs4.shared_cluster()
cli = make_logged_in_client()
reset = (
conf.MR_CLUSTERS["default"].HOST.set_for_testing("localhost"),
conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(23),
)
old = cluster.clear_caches()
try:
resp = cli.get('/debug/check_config')
assert_false('Failed to access filesystem root' in resp.content)
assert_false('Failed to create' in resp.content)
assert_false('Failed to chown' in resp.content)
assert_false('Failed to delete' in resp.content)
assert_true('Failed to contact JobTracker plugin' in resp.content)
finally:
for old_conf in reset:
old_conf()
cluster.restore_caches(old)
def test_update_properties():
finish = []
finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
try:
properties = {
'user.name': 'hue',
'test.1': 'http://localhost/test?test1=test&test2=test'
}
final_properties = properties.copy()
submission = Submission(None, properties=properties, oozie_id='test')
assert_equal(properties, submission.properties)
submission._update_properties('jtaddress', 'deployment-directory')
assert_equal(final_properties, submission.properties)
cluster.clear_caches()
fs = cluster.get_hdfs()
jt = cluster.get_next_ha_mrcluster()[1]
final_properties = properties.copy()
final_properties.update({
'jobTracker': 'jtaddress',
'nameNode': fs.fs_defaultfs
})
submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt)
assert_equal(properties, submission.properties)
submission._update_properties('jtaddress', 'deployment-directory')
assert_equal(final_properties, submission.properties)
finish.append(HDFS_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('namenode'))
finish.append(MR_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('jobtracker'))
cluster.clear_caches()
fs = cluster.get_hdfs()
jt = cluster.get_next_ha_mrcluster()[1]
final_properties = properties.copy()
final_properties.update({
'jobTracker': 'jobtracker',
'nameNode': 'namenode'
})
submission = Submission(None, properties=properties, oozie_id='test', fs=fs, jt=jt)
assert_equal(properties, submission.properties)
submission._update_properties('jtaddress', 'deployment-directory')
assert_equal(final_properties, submission.properties)
finally:
for reset in finish:
reset()
def test_non_default_cluster():
NON_DEFAULT_NAME = 'non_default'
cluster.clear_caches()
reset = (
conf.HDFS_CLUSTERS.set_for_testing({NON_DEFAULT_NAME: {}}),
conf.MR_CLUSTERS.set_for_testing({NON_DEFAULT_NAME: {}}),
)
try:
# This is indeed the only hdfs/mr cluster
assert_equal(1, len(cluster.get_all_hdfs()))
assert_equal(1, len(cluster.all_mrclusters()))
assert_true(cluster.get_hdfs(NON_DEFAULT_NAME))
assert_true(cluster.get_mrcluster(NON_DEFAULT_NAME))
cli = make_logged_in_client()
# That we can get to a view without errors means that the middlewares work
cli.get('/about')
finally:
for old_conf in reset:
old_conf()
def test_non_default_cluster():
NON_DEFAULT_NAME = 'non_default'
cluster.clear_caches()
reset = (
conf.HDFS_CLUSTERS.set_for_testing({ NON_DEFAULT_NAME: { } }),
conf.MR_CLUSTERS.set_for_testing({ NON_DEFAULT_NAME: { } }),
)
try:
# This is indeed the only hdfs/mr cluster
assert_equal(1, len(cluster.get_all_hdfs()))
assert_equal(1, len(cluster.all_mrclusters()))
assert_true(cluster.get_hdfs(NON_DEFAULT_NAME))
assert_true(cluster.get_mrcluster(NON_DEFAULT_NAME))
cli = make_logged_in_client()
# That we can get to a view without errors means that the middlewares work
cli.get('/about')
finally:
for old_conf in reset:
old_conf()
def test_massage_uri(self):
finish = HDFS_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('namenode')
cluster.clear_caches()
try:
assert_equal('', _massage_uri(''))
assert_equal('namenode/data', _massage_uri('hdfs:///data'))
assert_equal('hdfs://nn:11/data', _massage_uri('hdfs://nn:11/data'))
assert_equal('hdfs://logical/data', _massage_uri('hdfs://logical/data'))
assert_equal('namenode/data', _massage_uri('/data'))
assert_equal('file:///data', _massage_uri('file:///data'))
finally:
finish()
finish = HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing('hdfs://fs_defaultfs:8021')
cluster.clear_caches()
try:
assert_equal('', _massage_uri(''))
assert_equal('hdfs://fs_defaultfs:8021/data', _massage_uri('hdfs:///data'))
assert_equal('hdfs://nn:11/data', _massage_uri('hdfs://nn:11/data'))
assert_equal('hdfs://logical/data', _massage_uri('hdfs://logical/data'))
assert_equal('hdfs://fs_defaultfs:8021/data', _massage_uri('/data'))
assert_equal('file:///data', _massage_uri('file:///data'))
finally:
finish()
def test_config_validator_basic():
reset = (
conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing('http://not.the.re:50070/'),
conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(70000),
)
old = cluster.clear_caches()
try:
cli = make_logged_in_client()
resp = cli.get('/debug/check_config')
assert_true('hadoop.hdfs_clusters.default.webhdfs_url' in resp.content)
finally:
for old_conf in reset:
old_conf()
cluster.restore_caches(old)
def test_update_properties():
finish = []
finish.append(MR_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
finish.append(YARN_CLUSTERS['default'].SUBMIT_TO.set_for_testing(True))
try:
properties = {
'user.name': 'hue',
'test.1': 'http://localhost/test?test1=test&test2=test'
}
final_properties = properties.copy()
submission = Submission(None, properties=properties, oozie_id='test')
assert_equal(properties, submission.properties)
submission._update_properties('jtaddress', 'deployment-directory')
assert_equal(final_properties, submission.properties)
cluster.clear_caches()
fs = cluster.get_hdfs()
jt = cluster.get_next_ha_mrcluster()[1]
final_properties = properties.copy()
final_properties.update({
'jobTracker': 'jtaddress',
'nameNode': fs.fs_defaultfs
})
submission = Submission(None,
properties=properties,
oozie_id='test',
fs=fs,
jt=jt)
assert_equal(properties, submission.properties)
submission._update_properties('jtaddress', 'deployment-directory')
assert_equal(final_properties, submission.properties)
finish.append(
HDFS_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('namenode'))
finish.append(
MR_CLUSTERS['default'].LOGICAL_NAME.set_for_testing('jobtracker'))
cluster.clear_caches()
fs = cluster.get_hdfs()
jt = cluster.get_next_ha_mrcluster()[1]
final_properties = properties.copy()
final_properties.update({
'jobTracker': 'jobtracker',
'nameNode': 'namenode'
})
submission = Submission(None,
properties=properties,
oozie_id='test',
fs=fs,
jt=jt)
assert_equal(properties, submission.properties)
submission._update_properties('jtaddress', 'deployment-directory')
assert_equal(final_properties, submission.properties)
finally:
cluster.clear_caches()
for reset in finish:
reset()
def test_config_validator_basic():
reset = (
conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(
'http://not.the.re:50070/'),
conf.MR_CLUSTERS['default'].JT_THRIFT_PORT.set_for_testing(70000),
)
old = cluster.clear_caches()
try:
cli = make_logged_in_client()
resp = cli.get('/debug/check_config')
assert_true('hadoop.hdfs_clusters.default.webhdfs_url' in resp.content)
finally:
for old_conf in reset:
old_conf()
cluster.restore_caches(old)
def test_config_validator_basic():
reset = (
conf.HDFS_CLUSTERS["default"].WEBHDFS_URL.set_for_testing("http://not.the.re:50070/"),
conf.MR_CLUSTERS["default"].JT_THRIFT_PORT.set_for_testing(70000),
)
old = cluster.clear_caches()
try:
cli = make_logged_in_client()
resp = cli.get("/debug/check_config")
assert_true("hadoop.hdfs_clusters.default.webhdfs_url" in resp.content)
assert_true("hadoop.mapred_clusters.default.thrift_port" in resp.content)
assert_true("Port should be" in resp.content)
finally:
for old_conf in reset:
old_conf()
cluster.restore_caches(old)
def test_config_validator_more():
# TODO: Setup DN to not load the plugin, which is a common user error.
# We don't actually use the mini_cluster. But the cluster sets up the correct
# configuration that forms the test basis.
minicluster = pseudo_hdfs4.shared_cluster()
cli = make_logged_in_client()
old = cluster.clear_caches()
try:
resp = cli.get('/debug/check_config')
assert_false('Failed to access filesystem root' in resp.content)
assert_false('Failed to create' in resp.content)
assert_false('Failed to chown' in resp.content)
assert_false('Failed to delete' in resp.content)
finally:
cluster.restore_caches(old)
def test_config_validator_more():
# TODO: Setup DN to not load the plugin, which is a common user error.
# We don't actually use the mini_cluster. But the cluster sets up the correct
# configuration that forms the test basis.
minicluster = pseudo_hdfs4.shared_cluster()
cli = make_logged_in_client()
old = cluster.clear_caches()
try:
resp = cli.get('/debug/check_config')
assert_false('Failed to access filesystem root' in resp.content)
assert_false('Failed to create' in resp.content)
assert_false('Failed to chown' in resp.content)
assert_false('Failed to delete' in resp.content)
finally:
cluster.restore_caches(old)
def clear_sys_caches(): return cluster.clear_caches(), fsmanager.clear_cache()
def clear_sys_caches():
return cluster.clear_caches(), fsmanager.clear_cache()
def clear_sys_caches():
return [cluster.clear_caches()]
