def set_execute_settings(self, run_settings, local_settings):
self.set_domain_settings(run_settings, local_settings)
update(local_settings, run_settings,
'%s/stages/setup/payload_destination' % django_settings.SCHEMA_PREFIX,
'%s/stages/setup/filename_for_PIDs' % django_settings.SCHEMA_PREFIX,
'%s/stages/setup/process_output_dirname' % django_settings.SCHEMA_PREFIX,
'%s/stages/setup/smart_connector_input' % django_settings.SCHEMA_PREFIX,
'%s/system/contextid' % django_settings.SCHEMA_PREFIX,
'%s/system/random_numbers' % django_settings.SCHEMA_PREFIX,
'%s/system/id' % django_settings.SCHEMA_PREFIX
)
try:
local_settings['curate_data'] = getval(run_settings,
'%s/input/mytardis/curate_data' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
local_settings['curate_data'] = 0
local_settings['bdp_username'] = getval(run_settings,
'%s/bdp_userprofile/username' % django_settings.SCHEMA_PREFIX)
if '%s/input/system/compplatform/hadoop' % django_settings.SCHEMA_PREFIX in run_settings.keys():
from chiminey.platform import get_platform_settings
platform_url = run_settings['%s/platform/computation' % django_settings.SCHEMA_PREFIX]['platform_url']
pltf_settings = get_platform_settings(platform_url, local_settings['bdp_username'])
local_settings['root_path'] = '/home/%s' % pltf_settings['username']
local_settings['hadoop_home_path'] = pltf_settings['hadoop_home_path']
logger.debug('root_path=%s' % local_settings['root_path'])
else:
logger.debug('root_path not found')
def set_schedule_settings(self, run_settings, local_settings):
super(HadoopStrategy,
self).set_schedule_settings(run_settings, local_settings)
platform_url = run_settings['%s/platform/computation' %
RMIT_SCHEMA]['platform_url']
local_settings['root_path'] = '/home/%s' % (get_platform_settings(
platform_url, local_settings['bdp_username'])['username'])
logger.debug('out=%s' % local_settings)
def set_execute_settings(self, run_settings, local_settings):
self.set_domain_settings(run_settings, local_settings)
update(
local_settings, run_settings,
'%s/stages/setup/payload_destination' %
django_settings.SCHEMA_PREFIX,
'%s/stages/setup/filename_for_PIDs' %
django_settings.SCHEMA_PREFIX,
'%s/stages/setup/process_output_dirname' %
django_settings.SCHEMA_PREFIX,
'%s/stages/setup/smart_connector_input' %
django_settings.SCHEMA_PREFIX,
'%s/system/contextid' % django_settings.SCHEMA_PREFIX,
'%s/system/random_numbers' % django_settings.SCHEMA_PREFIX,
'%s/system/id' % django_settings.SCHEMA_PREFIX)
try:
local_settings['curate_data'] = getval(
run_settings, '%s/input/mytardis/curate_data' %
django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
local_settings['curate_data'] = 0
local_settings['bdp_username'] = getval(
run_settings,
'%s/bdp_userprofile/username' % django_settings.SCHEMA_PREFIX)
if '%s/input/system/compplatform/hadoop' % django_settings.SCHEMA_PREFIX in run_settings.keys(
):
from chiminey.platform import get_platform_settings
platform_url = run_settings[
'%s/platform/computation' %
django_settings.SCHEMA_PREFIX]['platform_url']
pltf_settings = get_platform_settings(
platform_url, local_settings['bdp_username'])
local_settings[
'root_path'] = '/home/%s' % pltf_settings['username']
local_settings['hadoop_home_path'] = pltf_settings[
'hadoop_home_path']
logger.debug('root_path=%s' % local_settings['root_path'])
else:
logger.debug('root_path not found')
def set_schedule_settings(self, run_settings, local_settings):
super(HadoopStrategy, self).set_schedule_settings(run_settings, local_settings)
platform_url = run_settings['%s/platform/computation' % RMIT_SCHEMA]['platform_url']
local_settings['root_path'] = '/home/%s' % (get_platform_settings(
platform_url, local_settings['bdp_username'])['username'])
logger.debug('out=%s' % local_settings)
def process(self, run_settings):
"""
Check all registered nodes to find whether
they are running, stopped or in error_nodes
"""
local_settings = getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS)
# local_settings = run_settings[models.UserProfile.PROFILE_SCHEMA_NS]
retrieve_local_settings(run_settings, local_settings)
logger.debug("local_settings=%s" % local_settings)
self.contextid = getval(run_settings, '%s/system/contextid' % django_settings.SCHEMA_PREFIX)
output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % django_settings.SCHEMA_PREFIX)
output_storage_settings = get_platform_settings(output_storage_url, local_settings['bdp_username'])
# FIXME: Need to be consistent with how we handle settings here. Prob combine all into
# single local_settings for simplicity.
output_storage_settings['bdp_username'] = local_settings['bdp_username']
offset = getval(run_settings, '%s/platform/storage/output/offset' % django_settings.SCHEMA_PREFIX)
self.job_dir = get_job_dir(output_storage_settings, offset)
try:
self.finished_nodes = getval(run_settings, '%s/stages/run/finished_nodes' % django_settings.SCHEMA_PREFIX)
except SettingNotFoundException:
self.finished_nodes = '[]'
try:
self.id = int(getval(run_settings, '%s/system/id' % django_settings.SCHEMA_PREFIX))
self.output_dir = "output_%s" % self.id
except (SettingNotFoundException, ValueError):
self.id = 0
self.output_dir = "output"
logger.debug("output_dir=%s" % self.output_dir)
logger.debug("run_settings=%s" % run_settings)
logger.debug("Wait stage process began")
#processes = self.executed_procs
processes = [x for x in self.current_processes if x['status'] == 'running']
self.error_nodes = []
# TODO: parse finished_nodes input
logger.debug('self.finished_nodes=%s' % self.finished_nodes)
self.finished_nodes = ast.literal_eval(self.finished_nodes)
computation_platform_url = getval(run_settings, '%s/platform/computation/platform_url' % django_settings.SCHEMA_PREFIX)
comp_pltf_settings = get_platform_settings(computation_platform_url, local_settings['bdp_username'])
local_settings.update(comp_pltf_settings)
comp_pltf_settings['bdp_username'] = local_settings['bdp_username']
wait_strategy = strategies.SynchronousWaitStrategy()
try:
payload_source = getval(run_settings, '%s/stages/setup/payload_source' % django_settings.SCHEMA_PREFIX)
if payload_source:
wait_strategy = strategies.AsynchronousWaitStrategy()
except SettingNotFoundException:
pass
for process in processes:
#instance_id = node.id
ip_address = process['ip_address']
process_id = process['id']
retry_left = process['retry_left']
#ip = botocloudconnector.get_instance_ip(instance_id, self.boto_settings)
#ssh = open_connection(ip_address=ip, settings=self.boto_settings)
#if not botocloudconnector.is_vm_running(node):
# An unlikely situation where the node crashed after is was
# detected as registered.
#FIXME: should error nodes be counted as finished?
# logging.error('Instance %s not running' % instance_id)
# self.error_nodes.append(node)
# continue
relative_path_suffix = self.get_relative_output_path(local_settings)
fin = wait_strategy.is_job_finished( self,
ip_address, process_id, retry_left,
local_settings, relative_path_suffix)
logger.debug("fin=%s" % fin)
if fin:
logger.debug("done. output is available")
logger.debug("node=%s" % str(process))
logger.debug("finished_nodes=%s" % self.finished_nodes)
#FIXME: for multiple nodes, if one finishes before the other then
#its output will be retrieved, but it may again when the other node fails, because
#we cannot tell whether we have prevous retrieved this output before and finished_nodes
# is not maintained between triggerings...
if not (int(process_id) in [int(x['id'])
for x in self.finished_nodes
if int(process_id) == int(x['id'])]):
self.get_output(ip_address, process_id, self.output_dir,
local_settings, comp_pltf_settings,
output_storage_settings, run_settings)
audit_url = get_url_with_credentials(
comp_pltf_settings, os.path.join(
self.output_dir, process_id, "audit.txt"),
is_relative_path=True)
fsys = storage.get_filesystem(audit_url)
logger.debug("Audit file url %s" % audit_url)
if fsys.exists(audit_url):
fsys.delete(audit_url)
self.finished_nodes.append(process)
logger.debug('finished_processes=%s' % self.finished_nodes)
for iterator, p in enumerate(self.all_processes):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.all_processes[iterator]['status'] = 'completed'
for iterator, p in enumerate(self.executed_procs):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.executed_procs[iterator]['status'] = 'completed'
for iterator, p in enumerate(self.current_processes):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.current_processes[iterator]['status'] = 'completed'
else:
logger.warn("We have already "
+ "processed output of %s on node %s" % (process_id, ip_address))
else:
print "job %s at %s not completed" % (process_id, ip_address)
failed_processes = [x for x in self.current_processes if x['status'] == 'failed']
logger.debug('failed_processes=%s' % failed_processes)
logger.debug('failed_processes=%d' % len(failed_processes))
messages.info(run_settings, "%d: Waiting %d processes (%d completed, %d failed) " % (
self.id + 1, len(self.current_processes), len(self.finished_nodes),
len(failed_processes)))
def process(self, run_settings):
"""
Check all registered nodes to find whether
they are running, stopped or in error_nodes
"""
local_settings = getvals(run_settings, models.UserProfile.PROFILE_SCHEMA_NS)
# local_settings = run_settings[models.UserProfile.PROFILE_SCHEMA_NS]
retrieve_local_settings(run_settings, local_settings)
logger.debug("local_settings=%s" % local_settings)
self.contextid = getval(run_settings, '%s/system/contextid' % RMIT_SCHEMA)
output_storage_url = getval(run_settings, '%s/platform/storage/output/platform_url' % RMIT_SCHEMA)
output_storage_settings = get_platform_settings(output_storage_url, local_settings['bdp_username'])
# FIXME: Need to be consistent with how we handle settings here. Prob combine all into
# single local_settings for simplicity.
output_storage_settings['bdp_username'] = local_settings['bdp_username']
offset = getval(run_settings, '%s/platform/storage/output/offset' % RMIT_SCHEMA)
self.job_dir = get_job_dir(output_storage_settings, offset)
try:
self.finished_nodes = getval(run_settings, '%s/stages/run/finished_nodes' % RMIT_SCHEMA)
# self.finished_nodes = smartconnectorscheduler.get_existing_key(run_settings,
# 'http://rmit.edu.au/schemas/stages/run/finished_nodes')
except SettingNotFoundException:
self.finished_nodes = '[]'
try:
self.id = int(getval(run_settings, '%s/system/id' % RMIT_SCHEMA))
# self.id = int(smartconnectorscheduler.get_existing_key(run_settings,
# 'http://rmit.edu.au/schemas/system/id'))
self.output_dir = "output_%s" % self.id
except (SettingNotFoundException, ValueError):
self.id = 0
self.output_dir = "output"
logger.debug("output_dir=%s" % self.output_dir)
logger.debug("run_settings=%s" % run_settings)
logger.debug("Wait stage process began")
#processes = self.executed_procs
processes = [x for x in self.current_processes if x['status'] == 'running']
self.error_nodes = []
# TODO: parse finished_nodes input
logger.debug('self.finished_nodes=%s' % self.finished_nodes)
self.finished_nodes = ast.literal_eval(self.finished_nodes)
computation_platform_url = getval(run_settings, '%s/platform/computation/platform_url' % RMIT_SCHEMA)
comp_pltf_settings = get_platform_settings(computation_platform_url, local_settings['bdp_username'])
local_settings.update(comp_pltf_settings)
comp_pltf_settings['bdp_username'] = local_settings['bdp_username']
wait_strategy = strategies.SynchronousWaitStrategy()
try:
synchronous_wait = getval(run_settings, '%s/stages/wait/synchronous' % RMIT_SCHEMA)
if not synchronous_wait:
wait_strategy = strategies.AsynchronousWaitStrategy()
except SettingNotFoundException:
pass
for process in processes:
#instance_id = node.id
ip_address = process['ip_address']
process_id = process['id']
retry_left = process['retry_left']
#ip = botocloudconnector.get_instance_ip(instance_id, self.boto_settings)
#ssh = open_connection(ip_address=ip, settings=self.boto_settings)
#if not botocloudconnector.is_vm_running(node):
# An unlikely situation where the node crashed after is was
# detected as registered.
#FIXME: should error nodes be counted as finished?
# logging.error('Instance %s not running' % instance_id)
# self.error_nodes.append(node)
# continue
relative_path_suffix = self.get_relative_output_path(local_settings)
fin = wait_strategy.is_job_finished( self,
ip_address, process_id, retry_left,
local_settings, relative_path_suffix)
logger.debug("fin=%s" % fin)
if fin:
logger.debug("done. output is available")
logger.debug("node=%s" % str(process))
logger.debug("finished_nodes=%s" % self.finished_nodes)
#FIXME: for multiple nodes, if one finishes before the other then
#its output will be retrieved, but it may again when the other node fails, because
#we cannot tell whether we have prevous retrieved this output before and finished_nodes
# is not maintained between triggerings...
if not (int(process_id) in [int(x['id'])
for x in self.finished_nodes
if int(process_id) == int(x['id'])]):
self.get_output(ip_address, process_id, self.output_dir,
local_settings, comp_pltf_settings,
output_storage_settings, run_settings)
audit_url = get_url_with_credentials(
comp_pltf_settings, os.path.join(
self.output_dir, process_id, "audit.txt"),
is_relative_path=True)
fsys = storage.get_filesystem(audit_url)
logger.debug("Audit file url %s" % audit_url)
if fsys.exists(audit_url):
fsys.delete(audit_url)
self.finished_nodes.append(process)
logger.debug('finished_processes=%s' % self.finished_nodes)
for iterator, p in enumerate(self.all_processes):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.all_processes[iterator]['status'] = 'completed'
for iterator, p in enumerate(self.executed_procs):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.executed_procs[iterator]['status'] = 'completed'
for iterator, p in enumerate(self.current_processes):
if int(p['id']) == int(process_id) and p['status'] == 'running':
self.current_processes[iterator]['status'] = 'completed'
else:
logger.warn("We have already "
+ "processed output of %s on node %s" % (process_id, ip_address))
else:
print "job %s at %s not completed" % (process_id, ip_address)
failed_processes = [x for x in self.current_processes if x['status'] == 'failed']
logger.debug('failed_processes=%s' % failed_processes)
logger.debug('failed_processes=%d' % len(failed_processes))
messages.info(run_settings, "%d: waiting %d processes (%d completed, %d failed) " % (
self.id + 1, len(self.current_processes), len(self.finished_nodes),
len(failed_processes)))
def get_platform_settings(self, run_settings, namespace_prefix):
bdp_username = run_settings['%s/bdp_userprofile' %
django_settings.SCHEMA_PREFIX]['username']
platform_url = run_settings[namespace_prefix]['platform_url']
return get_platform_settings(platform_url, bdp_username)
def get_platform_settings(self, run_settings, namespace_prefix):
bdp_username = run_settings['http://rmit.edu.au/schemas/bdp_userprofile']['username']
platform_url = run_settings[namespace_prefix]['platform_url']
return get_platform_settings(platform_url, bdp_username)
def get_platform_settings(self, run_settings, namespace_prefix):
bdp_username = run_settings['%s/bdp_userprofile' % django_settings.SCHEMA_PREFIX]['username']
platform_url = run_settings[namespace_prefix]['platform_url']
return get_platform_settings(platform_url, bdp_username)