def hdfs_client(kerberos, hdfs_server):
try:
client_id = "hdfs-client"
client = {
"id": client_id,
"mem": 1024,
"user": "******",
"container": {
"type":
"MESOS",
"docker": {
"image": "nvaziri/hdfs-client:stable",
"forcePullImage": True
},
"volumes": [{
"containerPath":
"/{}/hdfs.keytab".format(config.HADOOP_VERSION),
"secret":
"hdfs_keytab"
}]
},
"secrets": {
"hdfs_keytab": {
"source": kerberos.get_keytab_path()
}
},
"networks": [{
"mode": "host"
}],
"env": {
"REALM":
kerberos.get_realm(),
"KDC_ADDRESS":
kerberos.get_kdc_address(),
"JAVA_HOME":
"/usr/lib/jvm/default-java",
"KRB5_CONFIG":
"/etc/krb5.conf",
"HDFS_SERVICE_NAME":
sdk_hosts._safe_name(config.FOLDERED_SERVICE_NAME),
"HADOOP_VERSION":
config.HADOOP_VERSION
}
}
sdk_marathon.install_app(client)
krb5.write_krb5_config_file(client_id, "/etc/krb5.conf", kerberos)
yield client
finally:
sdk_marathon.destroy_app(client_id)
def get_hdfs_client_app(service_name, kerberos=None) -> dict:
"""
Returns a Marathon app definition for an HDFS client against the specified service.
This app should be installed AFTER the service is up and running, or else it may fail with an error like:
18/08/21 20:36:57 FATAL conf.Configuration: error parsing conf core-site.xml
org.xml.sax.SAXParseException; Premature end of file.
"""
app = {
"id": CLIENT_APP_NAME,
"mem": 1024,
"user": "******",
"container": {
"type": "MESOS",
"docker": {
"image": DOCKER_IMAGE_NAME,
"forcePullImage": True
},
},
"networks": [{
"mode": "host"
}],
"env": {
"JAVA_HOME": "/usr/lib/jvm/default-java",
"KRB5_CONFIG": "/etc/krb5.conf",
# for foldered name in test_kerberos_auth.py:
"HDFS_SERVICE_NAME": sdk_hosts._safe_name(service_name),
"HADOOP_VERSION": HADOOP_VERSION,
},
}
if kerberos:
# Insert kerberos-related configuration into the client:
app["env"]["REALM"] = kerberos.get_realm()
app["env"]["KDC_ADDRESS"] = kerberos.get_kdc_address()
app["secrets"] = {
"hdfs_keytab": {
"source": kerberos.get_keytab_path()
}
}
app["container"]["volumes"] = [{
"containerPath":
"/{}/hdfs.keytab".format(HADOOP_VERSION),
"secret":
"hdfs_keytab"
}]
return app
def get_hdfs_client_app(service_name, kerberos=None) -> dict:
"""
Returns a Marathon app definition for an HDFS client against the specified service.
This app should be installed AFTER the service is up and running, or else it may fail with an error like:
18/08/21 20:36:57 FATAL conf.Configuration: error parsing conf core-site.xml
org.xml.sax.SAXParseException; Premature end of file.
"""
app = {
"id": CLIENT_APP_NAME,
"mem": 1024,
"user": "******",
"container": {
"type": "MESOS",
"docker": {"image": DOCKER_IMAGE_NAME, "forcePullImage": True},
},
"networks": [{"mode": "host"}],
"env": {
"JAVA_HOME": "/usr/lib/jvm/default-java",
"KRB5_CONFIG": "/etc/krb5.conf",
# for foldered name in test_kerberos_auth.py:
"HDFS_SERVICE_NAME": sdk_hosts._safe_name(service_name),
"HADOOP_VERSION": HADOOP_VERSION,
},
}
if kerberos:
# Insert kerberos-related configuration into the client:
app["env"]["REALM"] = kerberos.get_realm()
app["env"]["KDC_ADDRESS"] = kerberos.get_kdc_address()
app["secrets"] = {
"hdfs_keytab": {
"source": kerberos.get_keytab_path()
}
}
app["container"]["volumes"] = [
{
"containerPath": "/{}/hdfs.keytab".format(HADOOP_VERSION),
"secret": "hdfs_keytab"
}
]
return app