def hpcprepWaiter(sessionToken, extraParameters, seconds_to_wait):
"""Creates all parameters required as input by the HPC launcher.
Most parameters are hard-coded here, but in other scenarios (for example
with user-defined input), some more string handling will happen here.
"""
# Validate session token
ep = ExtraParameters(extraParameters)
auth = AuthClient(ep.get_auth_WSDL_URL())
if not auth.validate_session_token(sessionToken):
logging.error("Token validation failed")
error_msg = "Session-token validation failed"
raise TokenValidationFailedFault(faultstring=error_msg)
# Prepare parameters the HPC launcher needs
image_name = "waiter_abortable.simg"
commandline = "python"
parameters = "/app/startup.py {} /app".format(seconds_to_wait)
queue = "qexp"
numNodes = os.environ["N_NODES"]
numCores = os.environ["N_CORES"]
maxDurationInMinutes = 5
SingularityVersion = "2.4.2"
return (image_name, commandline, parameters, queue, numNodes, numCores,
maxDurationInMinutes, SingularityVersion)
def showDialog(serviceID, sessionToken, extraParameters):
"""
Starts the dialog application.
"""
logging.info(
"startDialog() called with service ID {}".format(serviceID))
# Check that the session token is valid and abort with a SOAP fault if
# it's not. To that end, we use the clfpy library both to extract the
# authentication endpoint from the extraParameters input argument and
# to communicate with that endpoint.
ep = ExtraParameters(extraParameters)
auth = AuthClient(ep.get_auth_WSDL_URL())
if not auth.validate_session_token(sessionToken):
logging.error("Token validation failed")
error_msg = "Session-token validation failed"
raise TokenValidationFailedFault(faultstring=error_msg)
# The entire application, which will be visible to the user running the
# workflow, is packed in the status report created in this method.
# Here, we create a simple HTML page with a button to continue the
# workflow.
status = base64.b64encode(
create_html_dialog(serviceID, sessionToken,
ep.get_WFM_endpoint()).encode()).decode()
result = "UNSET"
return (status, result)
def abortService(ctx, serviceID, sessionToken):
"""Aborts the currently running service (not implemented, returns false)
"""
logging.info(
"abortService() called with service ID {}".format(serviceID))
# We obtain the authentication-manager endpoint from a class property
# and check that the session token is valid
# Read wsdl URL from a file
waiterdir = os.path.join(WAITER_LOG_FOLDER, serviceID)
wsdlfile = os.path.join(waiterdir, 'wsdl.txt')
with open(wsdlfile) as f:
auth_wsdl = f.read().strip()
auth = AuthClient(auth_wsdl)
if not auth.validate_session_token(sessionToken):
error_msg = "Session-token validation failed"
raise TokenValidationFailedFault(faultstring=error_msg)
# This method offers the option to abort long-running asynchronous
# services. In this example, we do not implement this functionality
# and thus always return False.
# In a more realistic scenario, this method would terminate the
# background computation process gracefully.
return False
def main():
port = 80
try:
context_root = os.environ["CONTEXT_ROOT"]
except KeyError:
print("Error: environment variable CONTEXT_ROOT not set.")
exit(1)
url = "http://localhost:{}{}/Dialog?wsdl".format(port, context_root)
print("wsdl URL is {}".format(url))
print("Obtaining session token")
user = input("Enter username: "******"Enter project: ")
password = getpass.getpass(prompt="Enter password: "******"Calling startDialog()")
response = soap_call(url, "showDialog", ["serviceID1", token, extra_pars])
html = base64.b64decode(response["status_base64"]).decode()
with open("test.html", 'w') as fout:
fout.write(html)
print("Result written to test.html")
def main():
"""Makes a series of test calls and prints their outputs."""
port = 80
try:
context_root = os.environ["CONTEXT_ROOT"]
except KeyError:
print("Error: environment variable CONTEXT_ROOT not set.")
exit(1)
print("Obtaining session token")
user = input("Enter username: "******"Enter project: ")
password = getpass.getpass(prompt="Enter password: "******"http://localhost:{}{}/WaiterPrep?wsdl".format(port, context_root)
print("wsdl URL is {}".format(url))
print("Calling preprocessor with 45 seconds waiting time:")
response = soap_call(url, "hpcprepWaiter", [token, extra_pars, 45])
print(response)
def main():
port = 80
try:
context_root = os.environ["CONTEXT_ROOT"]
except KeyError:
print("Error: environment variable CONTEXT_ROOT not set.")
exit(1)
url = "http://localhost:{}{}/Waiter?wsdl".format(port, context_root)
print("wsdl URL is {}".format(url))
if len(sys.argv) != 2:
print("Expected [start|status] as argument.")
exit(1)
print("Obtaining session token")
user = input("Enter username: "******"Enter project: ")
password = getpass.getpass(prompt="Enter password: ")
auth = AuthClient(auth_endpoint)
token = auth.get_session_token(user, project, password)
if sys.argv[1] == 'start':
start(url, token)
elif sys.argv[1] == 'status':
status(url, token)
else:
print('Unknown argument.')
def abortService(ctx, serviceID, sessionToken):
"""Aborts the currently running service (not implemented, returns false)
"""
logging.info(f"abortService() called with service ID {serviceID}")
# We obtain the authentication-manager endpoint from a class property
# and check that the session token is valid
# Read wsdl URL from a file
logdir = os.path.join(LOG_FOLDER, serviceID)
wsdlfile = os.path.join(logdir, 'wsdl.txt')
with open(wsdlfile) as f:
auth_wsdl = f.read().strip()
auth = AuthClient(auth_wsdl)
if not auth.validate_session_token(sessionToken):
error_msg = "Session-token validation failed"
raise TokenValidationFailedFault(faultstring=error_msg)
pidfile = os.path.join(logdir, 'pid.txt')
with open(pidfile) as f:
pid = f.read().strip()
try:
os.kill(int(pid), signal.SIGKILL)
except Exception as ex:
print('Exception while aborting consumer')
print(str(ex))
return False
return True
def getServiceStatus(ctx, serviceID, sessionToken):
"""Status-query method which is called regularly by WFM.
Here, a more realistic service would query the status of a calculation
etc. and process its log files to create a status page. Here, the log
contains only a single number, which we convert to an html progress
bar.
"""
logging.info(
"getServiceStatus() called with service ID {}".format(serviceID))
# We obtain the authentication-manager endpoint from a class property
# and check that the session token is valid
auth = AuthClient(ctx.descriptor.service_class.auth_wsdl)
if not auth.validate_session_token(sessionToken):
logging.error("Token validation failed")
error_msg = "Session-token validation failed"
raise TokenValidationFailedFault(faultstring=error_msg)
# Create correct file paths from service ID. By using the unique
# service ID, we can address the right waiter process in case this
# service is called several times in parallel.
waiterdir = os.path.join(WAITER_LOG_FOLDER, serviceID)
statusfile = os.path.join(waiterdir, 'status.txt')
resultfile = os.path.join(waiterdir, 'result.txt')
# Read the current status from the waiter logs. Here, that is only a
# single number between 0 and 100.
with open(statusfile) as f:
current_status = f.read().strip()
if current_status == "100":
logging.info("Waiting completed")
status = "COMPLETED"
# Read result page from waiter
with open(resultfile) as f:
result = f.read()
return (status, result)
# Note that the interface definition of getServiceStatus() specifies
# "UNCHANGED" as another option for the return value of
# 'status_base64'. In this case, the workflow manager will simply
# continue to display the last status page transmitted. This can be
# used when the status-page generation in itself is costly.
# If not finished, create a status page from the current status
# This could include more post-processing etc. in a more realistic
# service
result = "UNSET"
status = base64.b64encode(
create_html_progressbar(int(current_status)).encode()).decode()
return (status, result)
def parameterDebugger(serviceID,
sessionToken,
extraParameters,
in1="",
label1="in1",
in2="",
label2="in2",
in3="",
label3="in3",
in4="",
label4="in4",
in5="",
label5="in5"):
"""
Starts the debugger application.
"""
logging.info(
"parameterDebugger() called with service ID {}".format(serviceID))
# Validate token
ep = ExtraParameters(extraParameters)
auth = AuthClient(ep.get_auth_WSDL_URL())
if not auth.validate_session_token(sessionToken):
logging.error("Token validation failed")
error_msg = "Session-token validation failed"
raise TokenValidationFailedFault(faultstring=error_msg)
# Create application HTML
html = HTML.format(
sid=serviceID,
stk=sessionToken,
wfm_endpoint=ep.get_WFM_endpoint(),
eP=extraParameters,
in1=in1,
in2=in2,
in3=in3,
in4=in4,
in5=in5,
label1=label1,
label2=label2,
label3=label3,
label4=label4,
label5=label5,
result=RES_B64,
)
status = base64.b64encode(html.encode()).decode()
return status
def hpcPrepWithFile(sessionToken, extraParameters, filepath, textinput):
"""Creates parameters required as input by the HPC launcher.
"""
# Validate session token
ep = ExtraParameters(extraParameters)
auth = AuthClient(ep.get_auth_WSDL_URL())
if not auth.validate_session_token(sessionToken):
logging.error("Token validation failed")
error_msg = "Session-token validation failed"
raise TokenValidationFailedFault(faultstring=error_msg)
# Prepare parameters the HPC launcher needs
commandline = "python"
parameters = "/app/startup.py {} {}".format(filepath, textinput)
queue = "qexp"
numNodes = os.environ["N_NODES"]
numCores = os.environ["N_CORES"]
return (commandline, parameters, queue, numNodes, numCores)
def abortService(ctx, serviceID, sessionToken):
"""Aborts the currently running service (not implemented, returns false)
"""
logging.info(
"abortService() called with service ID {}".format(serviceID))
# We obtain the authentication-manager endpoint from a class property
# and check that the session token is valid
auth = AuthClient(ctx.descriptor.service_class.auth_wsdl)
if not auth.validate_session_token(sessionToken):
error_msg = "Session-token validation failed"
raise TokenValidationFailedFault(faultstring=error_msg)
# This method offers the option to abort long-running asynchronous
# services. In this example, we do not implement this functionality
# and thus always return False.
# In a more realistic scenario, this method would terminate the
# background computation process gracefully.
return False
def main():
try:
port = int(sys.argv[1])
print(f"Using port {port}")
except:
port = 821
print(f"Couldn't get port from commandline argument, using {port}.")
try:
context_root = os.environ["CONTEXT_ROOT"]
except KeyError:
print("Error: environment variable CONTEXT_ROOT not set.")
exit(1)
# host_adress = 'kafka_consumer'
host_adress = '193.175.65.88'
url = f"http://{host_adress}:{port}{context_root}/KafkaConsumerService?wsdl"
print(f"wsdl URL is {url}")
if len(sys.argv) != 2:
print("Expected [start|status] as argument.")
exit(1)
print("Obtaining session token")
user = input("Enter username: "******"Enter project: ")
password = getpass.getpass(prompt="Enter password: ")
auth = AuthClient(auth_endpoint)
token = auth.get_session_token(user, project, password)
if sys.argv[1] == 'start':
start(url, token)
elif sys.argv[1] == 'status':
status(url, token)
else:
print('Unknown argument.')
from clfpy import HpcImagesClient
from clfpy import GssClient
from clfpy import AuthClient
auth = AuthClient("https://api.hetcomp.org/authManager/AuthManager?wsdl")
user = "******"
project = "???"
password = "******"
print("Authenticating ...")
tk = auth.get_session_token(user, project, password)
print("Uploading image ...")
gss = GssClient("https://api.hetcomp.org/gss-0.1/FileUtilities?wsdl")
gss_ID = "it4i_anselm://home/abortable_waiter.simg"
# Change to gss.upload for the first upload
gss.update(gss_ID, tk, "abortable_waiter.simg")
print("Registering image ...")
images = HpcImagesClient("https://api.hetcomp.org/hpc-4-anselm/Images?wsdl")
# Change to images.upload_image for the first upload
images.update_image(tk, "waiter_abortable.simg", gss_ID)
print("Querying image information ...")
print(images.get_image_info(tk, "waiter_abortable.simg"))
def startWaiter(ctx, serviceID, sessionToken, extraParameters,
secondsToWait):
"""Starts a waiter script as a separate process and returns immediately.
In a more realistic scenario, this is where a longer computation etc.
would be started as a separate process. Here, we simply start a process
which waits for a while while regularly updating a status file.
"""
logging.info(
"startWaiter() called with service ID {}".format(serviceID))
# Check that the session token is valid and abort with a SOAP fault if
# it's not. To that end, we use the clfpy library both to extract the
# authentication endpoint from the extraParameters input argument and
# to communicate with that endpoint. We also save the authentication-
# manager endpoint in a class property which we can re-use in methods
# that don't have the extraParameters as an argument.
ep = ExtraParameters(extraParameters)
auth = AuthClient(ep.get_auth_WSDL_URL())
if not auth.validate_session_token(sessionToken):
logging.error("Token validation failed")
error_msg = "Session-token validation failed"
raise TokenValidationFailedFault(faultstring=error_msg)
# Add default value for waiting time
if secondsToWait is None:
logging.info("Setting default value for waiting time")
secondsToWait = 60
# Create a temporary folder to store the status files in.
# Note that we use the service ID as a unique identifier. Since this
# service is stateless, subsequent calls to getServiceStatus() need to
# be able to read the correct status files. (The service can be started
# several times in parallel.)
# In a more realistic setting, one would set up log directories for a
# computation here.
waiterdir = os.path.join(WAITER_LOG_FOLDER, serviceID)
if not os.path.exists(waiterdir):
os.mkdir(waiterdir)
statusfile = os.path.join(waiterdir, 'status.txt')
resultfile = os.path.join(waiterdir, 'result.txt')
# Store the auth-manager WSDL URL in a file for later use in
# getServiceStatus()
wsdlfile = os.path.join(waiterdir, 'wsdl.txt')
with open(wsdlfile, 'w') as f:
f.write(ep.get_auth_WSDL_URL())
logging.info("Stored auth-manager WSDL URL: {}".format(
ep.get_auth_WSDL_URL()))
# Spawn new process running the waiter script.
# We pass the status and result file to the script to ensure that the
# waiter logs to the correct place.
logging.info("Starting waiter script")
command = [
'python', 'wait_a_while.py',
str(secondsToWait), statusfile, resultfile
]
subprocess.Popen(command)
# We now create a first status page to be displayed while the
# workflow is executed. Since the waiter process was only just started,
# we don't have a proper status yet. So we simply start with an empty
# progress bar.
# The status page needs to be base64 encoded.
status = base64.b64encode(create_html_progressbar(0).encode()).decode()
result = "UNSET"
return (status, result)
def main():
"""Makes a series of test calls and prints their outputs."""
try:
host_address_service = int(sys.argv[1])
print(f"Using host address {host_address_service}")
except:
host_address_service = "localhost"
print(
"Couldn't get host address of the consumer service from commandline argument, using localhost."
)
try:
port_service = int(sys.argv[2])
print(f"Using port {port_service}")
except:
port_service = 821
print(
f"Couldn't get port of the consumer service from commandline argument, using {port_service}."
)
try:
serviceid = int(sys.argv[3])
print(f"Using serviceid {serviceid}")
except:
serviceid = "testid_" + str(int(time.time())) + "-" + str(
random.randint(0, 10000))
print(
f"Couldn't get serviceid from commandline argument, using {serviceid}."
)
try:
host_address = int(sys.argv[4])
print(f"Using host address {host_address}")
except:
host_address = "92.78.102.187"
print(
f"Couldn't get kafka broker host address from commandline argument, using default: {host_address}"
)
try:
port = int(sys.argv[5])
print(f"Using port {port}")
except:
port = 59092
print(
f"Couldn't get port from commandline argument, using default: {port}."
)
try:
timeout = int(sys.argv[6])
print(f"Using timeout {timeout}")
except:
timeout = 5
print(
f"Couldn't get timeout from commandline argument, using {timeout}s."
)
try:
topic = int(sys.argv[7])
print(f"Using topic {topic}")
except:
topic = "testtopic_" + str(random.randint(0, 10000))
print(f"Couldn't get topic from commandline argument, using {topic}.")
try:
context_root = os.environ["CONTEXT_ROOT"]
except KeyError:
context_root = '/demo-kafka-consumer'
print(
f"Error: environment variable CONTEXT_ROOT not set, using default: {context_root}."
)
print("Obtaining session token")
user = input("Enter username: "******"Enter project: ")
password = getpass.getpass(prompt="Enter password: "******"http://{host_address_service}:{port_service}{context_root}/KafkaConsumerService?wsdl"
print(f"Service URL is {url}")
print("Testing start of consumer:")
response = soap_call(
url, "startConsumer",
[id, token, extra_pars, host_address, port, topic,
int(timeout + 5)])
print(f"Start result = {response}")
print(
f"Waiting {timeout} seconds until update is retrieved and consumer is stopped"
)
time.sleep(timeout)
response2 = soap_call(url, "getServiceStatus", [id, token])
print(f"status = {response2}")
response3 = soap_call(url, "abortService", [id, token])
print(f"Consumer stop successful: {response3}")
print(f"Test finished")