def show(owner, repo, host, port, config_file_path, output_file_path, output_style='show'):
host, port = server_util.get_server_address(host, port)
client = ecflow.Client(host, port)
client.sync_local()
defs = client.get_defs()
if defs is None:
print("The server has no definition")
return
nodes = defs.get_all_nodes()
bunch = Bunch()
for node in nodes:
node_path = node.get_abs_node_path()
node_name = node.name()
node_status = NodeStatus(str(node.get_dstate()))
node = {
'path': node_path,
'status': node_status,
'name': node_name
}
bunch.add_node_status(node)
bunch.status = NodeStatus(str(defs.get_state()))
if output_style == 'show':
print(json.dumps(bunch.to_dict(), indent=2))
elif output_style == 'save_to_file':
with open(output_file_path, 'w') as f:
json.dump(bunch.to_dict(), f, indent=2)
def __init__(self, ecf_host, ecf_port, logfile):
if ecflow is None:
raise Exception("Ecflow was not found")
Server.__init__(self)
self.ecf_host = ecf_host
self.ecf_port = ecf_port
self.logfile = logfile
self.ecf_client = ecflow.Client(self.ecf_host, self.ecf_port)
self.suite_name = None
def replace(self, interactive=True):
if not self.checked:
print("suite " + self.name +
" has not been checked, refusing to replace")
return
if interactive:
if not ask_confirm("Replace suite " + self.name + " on server"):
return
client = ecflow.Client() # connect using environment
client.replace("/" + self.name, self.defs)
print("Suite " + self.name + " replaced on server")
def __init__(self, node, port, path):
self.node = node
self.port = port
self.path = path
self.client = ec.Client(self.node, port)
try:
if type(path) is str:
self.client.ch_register(False, [str(path.split('/')[1])])
elif type(path) is array:
# print(path, "array")
self.client.ch_register(False, [path])
else:
self.client.ch_register(False, [str(path[1])])
except:
pass
def fetch_ecflow_stats(self):
# Create the client. This will read the default environment variables
ecflow_host = os.getenv('ECF_HOST', "localhost")
ecflow_port = os.getenv('ECF_PORT', "3143")
ci = ecflow.Client(ecflow_host, ecflow_port)
# Get the node tree suite definition as stored in the server
# The definition is retrieved and stored on the variable 'ci'
ci.sync_local()
# access the definition retrieved from the server
server_defs = ci.get_defs()
if server_defs == None:
print "The server has no definition"
exit(1)
return server_defs
def main():
logger = set_logger(os.path.splitext(os.path.split(__file__)[-1])[0], 'INFO')
"""Print currently loaded suite definition"""
try:
ci = ecflow.Client()
# Get server definition by syncing with client defs.
ci.sync_local()
# Set print style to show structure of suite definition.
ecflow.PrintStyle.set_style(ecflow.Style.DEFS)
# Print the returned suite definition.
logger.info(ci.get_defs())
except RuntimeError:
msg = ('Error in {}\n'.format(__file__)+
'ecflow was not able to retrieve suite definition')
if sys.version_info[0] != 3:
import traceback
msg += '\n\nOriginal traceback:\n' + traceback.format_exc()
raise RuntimeError(msg)
return
def main():
"""Start ECFLOW from HALTED server.
Delete any currently loaded suite definitions and tasks. Load
a new suite definition file. Restart the server. Run new suite. Run
this version when starting with a HALTED server. Run from inside
def_files directory."""
logger = set_logger(
os.path.splitext(os.path.split(__file__)[-1])[0], 'INFO')
if len(sys.argv) == 2:
suite = sys.argv[1]
def_file = suite + ".def"
else:
logger.critical('Usage: python {} suite_name'.format(__file__))
sys.exit()
try:
logger.info('Loading definition in ' + def_file + ' into the server')
ci = ecflow.Client()
ci.delete_all()
# Read definition from disk and load into the server.
ci.load(def_file)
logger.info('Restarting the server. This starts job scheduling')
ci.restart_server()
logger.info('Begin the suite named ' + suite)
ci.begin_suite(suite)
except RuntimeError:
msg = ('Error in {}\n'.format(__file__) +
'ecflow was not able to begin suite {}'.format(suite))
if sys.version_info[0] != 3:
import traceback
msg += '\n\nOriginal traceback:\n' + traceback.format_exc()
raise RuntimeError(msg)
return
def main():
"""Replace definition file and run new suite.
Delete any currently loaded definition files and tasks. Load the
suite definition file, and run the suite.
Run this version to start when you are starting with a RUNNING server.
Run from inside the def_files/ directory.
"""
logger = set_logger(os.path.splitext(os.path.split(__file__)[-1])[0], 'INFO')
logger.info('Client -> Server: delete, then load a new definition')
if len(sys.argv) == 2:
suite = sys.argv[1]
def_file = suite + '.def'
else:
logger.critical('Usage: python {} suite_name'.format(__file__))
sys.exit()
try:
ci = ecflow.Client()
# Clear out the server
ci.delete_all()
# Load the definition into the server.
ci.load(def_file)
# Start the suite.
ci.begin_suite(suite)
except RuntimeError:
msg = ('Error in {}\n'.format(__file__)+
'ecflow was not able to begin suite {}'.format(suite))
if sys.version_info[0] != 3:
import traceback
msg += '\n\nOriginal traceback:\n' + traceback.format_exc()
raise RuntimeError(msg)
return
import os
from pathlib import Path
import ecflow
home = os.path.abspath(Path(Path(__file__).parent, "../../../build/course"))
try:
print("Loading definition in 'test.def' into the server")
ci = ecflow.Client('login05', '33083')
ci.delete("/test")
ci.load(str(Path(home, "test.def")))
ci.begin_suite("test")
except RuntimeError as e:
print("Failed:", e)
def process(node):
if isinstance(node, ecflow.Task):
print("a task", node.name())
elif isinstance(node, ecflow.Family):
print("a family", node.name())
elif isinstance(node, ecflow.Suite):
print("a suite", node.name())
elif isinstance(node, ecflow.Alias):
print("an alias")
else:
print("???")
print(node.get_abs_node_path(), node.get_state(), "T:", node.get_trigger(),
"C:", node.get_complete())
for kid in node.nodes:
process(kid)
if __name__ == '__main__':
CLIENT = ecflow.Client(os.getenv('ECF_HOST', "localhost"),
os.getenv('ECF_PORT', "2500"))
CLIENT.ch_register(False, [
"elearning",
])
CLIENT.sync_local()
DEFS = CLIENT.get_defs()
for item in DEFS.suites:
process(item)
def __call__(self,replace_path):
suite = self._suite
paths = self._config['path']
prefix = paths['prefix']
clock = ecflow.Clock(False)
suite.add_clock(clock)
suite.add_variable("ECF_JOB_CMD","/bin/bash %ECF_JOB% 1> %ECF_JOBOUT% 2>&1")
suite.add_variable("ECF_HOME",self.make_path(prefix,paths['server']))
suite.add_variable("ECF_FILES", self.make_path(prefix,paths['tasks']))
if paths['include'] is None:
suite.add_variable("ECF_INCLUDE", os.path.join(self._pwd,'include'))
else:
suite.add_variable("ECF_INCLUDE", self.make_path(prefix,paths['include']))
suite.add_variable("ECF_BIN", os.path.join(self._pwd,'bin'))
suite.add_variable("PYTHON_EXEC", self._config['preferences']['python_runtime'])
suite.add_variable("TMPDIR", self.make_path(prefix,paths['tmpdir']))
suite.add_variable("ECF_TRIES", self._config['preferences']['tries'])
suite.add_variable("DELTA", 0)
suite.add_variable("CONFIG_DIR",self.make_path(prefix,self._config['path']['config']))
# admin emails related variables
suite.add_variable("SCHEDULER_PATH",self._pwd)
suite.add_variable("SCHEDULER_EMAIL",self._config['preferences']['email']['from'])
suite.add_variable("SCHEDULER_ADMIN",','.join(self._config['preferences']['email']['to']))
# for customer emails
suite.add_variable("EMAIL_STATUS",'1')
# where tasks are supposed to save data
suite.add_variable("DATA_STORE",self.make_path(prefix,paths['datastore']))
for family,info in self.families.items():
with suite.add_family(family) as current:
if info['start_date'] is None:
startdate = Day(Date())
else:
startdate = Day(info['start_date'])
duration = self.duration
if info['duration'] is not None:
duration = info['duration']
enddate = startdate + duration
# adjust environment
current.add_variable("PYTHON_SCRIPTS", info['python_scripts'])
current.add_repeat(ecflow.RepeatDate("YMD", int(startdate.format('%Y%m%d')),int(enddate.format('%Y%m%d')),self.loop_increment[info['loop_type']]))
# if it exists, createFactory just returns the factory
factory = createFactory(family)
families = factory.registered()
for family in families:
factory.create(family)(PlatinFlow(current))
restart = current.add_task('restart')
restart.add_time('23:59')
cleanup = suite.add_task('cleanup')
cleanup.add_time('23:59')
client = ecflow.Client(self._config['preferences']['host'],
self._config['preferences']['port'])
if replace_path is None:
replace_path = '/' + self._config['preferences']['suite_name']
client.replace(replace_path,self._defs)
import ecflow
try:
# When no arguments specified uses ECF_HOST and/or ECF_PORT,
# otherwise defaults to localhost:3141
ci = ecflow.Client() # inherit from shell variables
ci.ping()
except RuntimeError as e:
print("ping failed: " + str(e))
try:
# Explicitly set host and port using the same client
# For alternative argument list see ecflow.Client.set_host_port()
ci.set_host_port(
"login05:33083"
) # actually set the host and port (change to your host and port)
ci.ping()
except RuntimeError as e:
print("ping failed: " + str(e))
try:
# Create a new client, Explicitly setting host and port.
# For alternative argument list see ecflow.Client
ci = ecflow.Client("localhost:1000") # another server
ci.ping()
except RuntimeError as e:
print("ping failed: " + str(e))
import ecflow
try:
# Create the client
ci = ecflow.Client("login05", "33083")
# Get the node tree suite definition as stored in the server
# The definition is retrieved and stored on the variable 'ci'
ci.sync_local()
# access the definition retrieved from the server
defs = ci.get_defs()
if defs is None:
print("The server has no definition")
exit(1)
# get the tasks, *alternatively* could use defs.get_all_nodes()
# to include suites, families and tasks.
task_vec = defs.get_all_tasks()
# iterate over tasks and print path and state
for task in task_vec:
print(task.get_abs_node_path() + " " + str(task.get_state()))
except RuntimeError as e:
print("Failed: ", str(e))
summary_task.add_trigger(trigger_str)
# add archive task
archive_task = suite.add_task("archive")
archive_trigger = "summary == complete"
archive_task.add_trigger(archive_trigger)
defs.save_as_defs("%s/check.def" % run_path)
print "Written definition file to %s/check.def" % run_path
job_ctrl = ecflow.JobCreationCtrl()
defs.check_job_creation(job_ctrl)
# now use the client to get things running
# NB This needs sorting out
cl = ecflow.Client()
cl.set_host_port(server_hostname, str(server_port))
if server_exists(cl) == False:
print "Server not running on %s:%s. Exiting." % (server_hostname,
str(server_port))
sys.exit(1)
try:
######################################cl.delete_all()
# get a copy of the server defs on the client
cl.sync_local()
current_defs = cl.get_defs()
if current_defs != None:
import ecflow
try:
ci = ecflow.Client('login05', 33083)
ci.sync_local() # get server definition, by sync with client defs
ecflow.PrintStyle.set_style(
ecflow.Style.STATE) # set printing to show structure
print(ci.get_defs()) # print the returned suite definition
ecflow.PrintStyle.set_style(
ecflow.Style.MIGRATE
) # set printing to show structure and state, and node history
print(ci.get_defs())
except RuntimeError as e:
print("Failed:", e)
#! /usr/bin/env python
from __future__ import absolute_import
import os, produtil.log
##@var __all__
# List of symbols to import during "from produtil.ecflow import all"
__all__ = [
'set_ecflow_event', 'set_ecflow_label', 'set_ecflow_meter', "in_ecflow"
]
ecflow_task_name = os.environ.get('ECF_NAME', '')
if ecflow_task_name:
import ecflow
ecflow_client = ecflow.Client()
ecflow_client.set_child_path(ecflow_task_name)
ecflow_client.set_child_pid(int(os.environ['ECF_RID']))
ecflow_client.set_child_password(os.environ['ECF_PASS'])
ecflow_client.set_child_try_no(int(os.environ['ECF_TRYNO']))
else:
ecflow_client = None
def in_ecflow():
"""!Are we running in ecflow?
Checks environment variables that were set at the initialization
of this module to decide whether the job is inside ecflow or not.
@returns True if the job is inside ecflow, False otherwise. """
return ecflow_client is not None
if trigger_expr:
tasks = re.findall(r'(\S*) ==', trigger_expr.get_expression())
for t in tasks:
task = self.__defs.find_abs_node(
self.__suite.get_abs_node_path() + "/" + t)
if task.get_state() == ecflow.State.aborted:
if node.get_state() != ecflow.State.aborted:
print("Will force aborted state for task",
node.get_abs_node_path())
self.__ci.force_state(node.get_abs_node_path(),
ecflow.State.aborted)
try:
# Create the client. This will read the default environment variables
ci = ecflow.Client()
# Get the node tree suite definition as stored in the server
# The definition is retrieved and stored on the variable 'ci'
ci.sync_local()
# access the definition retrieved from the server
server_defs = ci.get_defs()
if server_defs == None:
print("The server has no definition")
exit(1)
traverser = DefsTraverser(server_defs, ci)
traverser.force_abort()
def collect_status(owner, repo, host, port, config_file_path, disable_post, post_url, content_encoding, verbose):
"""
collect ecflow server's status and post it to nmp-broker.
POST data
message: json string
{
'app': 'ecflow_status_collector',
'type': 'ecflow_status',
'timestamp': current_time,
'data': {
'owner': owner,
'repo': repo,
'server_name': repo,
'ecflow_host': port,
'ecflow_port': port,
'time': current_time,
'status': bunch_dict
}
}
"""
start_time = datetime.utcnow()
if verbose:
logger.info("[{owner}/{repo}] Fetching ecflow status...".format(owner=owner, repo=repo))
if config_file_path:
config = get_config(config_file_path)
else:
config = None
host, port = server_util.get_server_address(host, port)
client = ecflow.Client(host, port)
client.sync_local()
defs = client.get_defs()
if defs is None:
logger.info("The server has no definition", file=sys.stderr)
return
nodes = defs.get_all_nodes()
bunch = Bunch()
for node in nodes:
node_path = node.get_abs_node_path()
node_name = node.name()
node_status = NodeStatus(str(node.get_dstate()))
node = {
'path': node_path,
'status': node_status,
'name': node_name
}
bunch.add_node_status(node)
bunch.status = NodeStatus(str(defs.get_state()))
bunch_dict = bunch.to_dict()
current_time = (datetime.utcnow() + timedelta(hours=8)).isoformat() # 北京时间
result = {
'app': 'ecflow_status_collector',
'type': 'ecflow_status',
'timestamp': current_time,
'data': {
'owner': owner,
'repo': repo,
'server_name': repo,
'ecflow_host': host,
'ecflow_port': port,
'time': current_time,
'status': bunch_dict
}
}
post_data = {
'message': json.dumps(result)
}
get_status_end_time = datetime.utcnow()
if verbose:
logger.info("[{owner}/{repo}] Fetching ecflow status...Done. Used {cost}".format(
owner=owner, repo=repo,
cost=get_status_end_time - start_time
))
if not disable_post:
if verbose:
logger.info("[{owner}/{repo}] Posting ecflow status...".format(owner=owner, repo=repo))
if post_url:
url = post_url.format(owner=owner, repo=repo)
elif config:
host = config['post']['host']
port = config['post']['port']
url = config['post']['url'].format(
host=host,
port=port,
owner=owner,
repo=repo
)
else:
raise Exception("post url is not set.")
if content_encoding is None:
if config:
if 'content-encoding' in config['post']['headers']:
content_encoding = config['post']['headers']['content-encoding']
if content_encoding == 'gzip':
gzipped_data = gzip.compress(bytes(json.dumps(post_data), 'utf-8'))
requests.post(url, data=gzipped_data, headers={
'content-encoding': 'gzip'
})
else:
requests.post(url, data=post_data)
if verbose:
post_end_time = datetime.utcnow()
logger.info("[{owner}/{repo}] Posting ecflow status...Done. Used {cost}".format(
owner=owner, repo=repo,
cost=post_end_time - get_status_end_time
))
end_time = datetime.utcnow()
if verbose:
logger.info("[{owner}/{repo}] Collect ecflow status used {time}".format(
owner=owner, repo=repo, time=end_time - start_time))
#!/usr/bin/env python2.7
from __future__ import print_function
import os
import ecflow
# When no arguments is specified, Client uses bash variables ECF_HOST, ECF_PORT
HOST = os.getenv("ECF_HOST", "localhost")
PORT = int(os.getenv("ECF_PORT", "%d" % (1500 + os.getuid())))
NAME = os.getenv("SUITE", "elearning")
# ecflow_start.sh gives port number 1500+uid:
CLIENT = ecflow.Client(HOST + ":%d" % PORT)
# multiple ways to create a client:
# python -c "import ecflow; help(ecflow.Client)"
try:
CLIENT.ping()
except RuntimeError as err:
print("#ERR: ping failed: " + str(err))
try: # read definition from disk and load into the server:
CLIENT.load("%s.def" % NAME)
except RuntimeError as err:
CLIENT.replace("/%s" % NAME, "%s.def" % NAME)
DEBUG = True # DEBUG = False
if DEBUG:
print("Checking job creation: .ecf -> .job0")
print(ecflow.Defs("%s.def" % NAME).check_job_creation())
#!/usr/bin/python3.6m
import ecflow
try:
ci = ecflow.Client("localhost:2500")
ci.ping()
except RuntimeError as e:
print("ping failed: ", str(e))
try:
print("Loading definition in 'run_rapid.def' into the server")
ci = ecflow.Client("localhost:2500")
ci.sync_local() # get the defs from the server, and place on ci
defs = ci.get_defs() # retrieve the defs from ci
if defs is None:
print("No definition in server, loading defs from disk")
ci.load("run_rapid.def")
else:
print("read definition from disk and load into the server")
ci.replace("/run_rapid", "run_rapid.def")
print("Restarting the server. This starts job scheduling")
ci.restart_server()
print("Begin the suite named 'run_rapid'")
ci.begin_suite("run_rapid")
except RuntimeError as e:
#!/usr/bin/env python
from __future__ import print_function
import os
import ecflow
# When no arguments specified, uses ECF_HOST and/or ECF_PORT,
HOST = os.getenv("ECF_HOST", "localhost")
PORT = int(os.getenv("ECF_PORT", "%d" % (1500 + os.getuid())))
KEY = HOST + ":%d" % PORT
# multiple ways to create a client:
# python -c "import ecflow; help(ecflow.Client)"
CLIENT = ecflow.Client(KEY)
try:
CLIENT.ping()
except RuntimeError as err:
print("#ERR: ping failed: %s" % err)
# ci.load("%s.def" % NAME) # read definition from disk, and load into server
try:
CLIENT.restart_server()
except RuntimeError as err:
print("#ERR: cannot restart server", err)
NAME = os.getenv("SUITE", "elearning")
try:
CLIENT.begin_suite("/%s" % NAME)
except RuntimeError as err:
print("#ERR: cannot begin suite %s" % NAME, err)
# try: CLIENT.resume("/%s" % NAME)
# except RuntimeError as err: print("#ERR: cannot resume suite %s" % NAME, err)
def collect_variable(owner, repo, host, port, node_path, config_file_path):
"""
POST data:
message: json string
normal message
{
'app': 'nwpc-ecflow-collector',
'type': 'ecflow_node_collector',
'data': {
'request': {
'command': 'variable',
'arguments': [],
'time': request_time_string
},
'response': {
'node': ecf_node.to_dict(),
'time': datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
}
}
}
error message
{
'app': 'nwpc-ecflow-collector',
'type': 'ecflow_node_collector',
'error': 'variable-error',
'data': {
'request': {
'command': 'variable',
'arguments': [],
'time': request_time_string
},
'response': {
'error': 'defs not found',
'time': datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
}
}
}
"""
request_date_time = datetime.datetime.utcnow()
request_time_string = request_date_time.strftime("%Y-%m-%d %H:%M:%S")
client = ecflow.Client(host, port)
client.sync_local()
defs = client.get_defs()
if defs is None:
result = {
'app': 'nwpc-ecflow-collector',
'type': 'ecflow_node_collector',
'error': 'variable-error',
'data': {
'request': {
'command': 'variable',
'arguments': [],
'time': request_time_string
},
'response': {
'error':
'defs not found',
'time':
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
}
}
}
print(json.dumps(result, indent=2))
return result
node = defs.find_abs_node(node_path)
if node is None:
result = {
'app': 'nwpc-ecflow-collector',
'type': 'ecflow_node_collector',
'error': 'variable-error',
'data': {
'request': {
'command': 'variable',
'arguments': [],
'time': request_time_string
},
'response': {
'error':
'node not found',
'time':
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
}
}
}
print(json.dumps(result, indent=2))
return result
ecf_node = get_node_variable(node)
result = {
'app': 'nwpc-ecflow-collector',
'type': 'ecflow_node_collector',
'data': {
'request': {
'command': 'variable',
'arguments': [],
'time': request_time_string
},
'response': {
'node': ecf_node.to_dict(),
'time':
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
}
}
}
print(json.dumps(result, indent=2))
return result
