def _synchronize_ready(self, connection):
#TODO set timeout and retry count
if self.peer is None:
raise NoPeerError
others = self.core.launch_deps.values()
self.logger.info('waiting for other peers %s: ', str(others))
msg = cmsg.fill_and_pack(types.PEERS_READY_QUERY,
sender=self.peer_id,
deps=others)
ready = False
while not ready:
reply = self.__query(connection, msg, types.PEERS_READY_QUERY)
# print 'got!', reply, cmsg.unpack_msg(reply.type, reply.message)
if reply is None:
#TODO sth bad happened, raise exception?
continue
if reply.type == types.READY_STATUS:
rmsg = cmsg.unpack_msg(reply.type, reply.message)
ready = rmsg.peers_ready
if not ready:
time.sleep(2)
self.logger.info("Dependencies are ready, I can start working")
def _synchronize_ready(self, connection):
# TODO set timeout and retry count
if self.peer is None:
raise NoPeerError
others = list(self.core.launch_deps.values())
self.logger.info('waiting for other peers %s: ', str(others))
msg = cmsg.fill_and_pack(types.PEERS_READY_QUERY, sender=self.peer_id,
deps=others)
ready = False
while not ready:
reply = self.__query(connection, msg, types.PEERS_READY_QUERY)
# print 'got!', reply, cmsg.unpack_msg(reply.type, reply.message)
if reply is None:
# TODO sth bad happened, raise exception?
continue
if reply.type == types.READY_STATUS:
rmsg = cmsg.unpack_msg(reply.type, reply.message)
ready = rmsg.peers_ready
if not ready:
time.sleep(2)
self.logger.info("Dependencies are ready, I can start working")
def handle_config_message(self, mxmsg):
if mxmsg.type in cmsg.MX_CFG_MESSAGES:
message = cmsg.unpack_msg(mxmsg.type, mxmsg.message)
msg, mtype = self._call_handler(mxmsg.type, message)
if msg is None:
self.peer.no_response()
else:
msg = cmsg.pack_msg(msg)
self.peer.send_message(message=msg, type=mtype, to=int(mxmsg.from_), flush=True)
def _request_ext_params(self, connection, retries=400):
#TODO set timeout and retry count
self.logger.info("requesting external parameters")
if self.peer is None:
raise NoPeerError
def _unset_param_count():
return reduce(lambda x, y: x + y,
[len(self.core.unset_params_for_source(src)) \
for src in self.core.used_config_sources()], 0)
ready, details = self.core.config_ready()
while not ready and retries:
for src in self.core.used_config_sources():
params = self.core.unset_params_for_source(src).keys()
msg = cmsg.fill_msg(types.GET_CONFIG_PARAMS,
sender=self.peer_id,
param_names=params,
receiver=self.core.config_sources[src])
#print "requesting: {0}".format(msg)
reply = self.__query(connection, cmsg.pack_msg(msg),
types.GET_CONFIG_PARAMS)
if reply == None:
# raise something?
continue
if reply.type == types.CONFIG_ERROR:
self.logger.warning("peer {0} has not yet started".format(
msg.receiver))
elif reply.type == types.CONFIG_PARAMS:
reply_msg = cmsg.unpack_msg(reply.type, reply.message)
params = cmsg.params2dict(reply_msg)
for par, val in params.iteritems():
self.core.set_param_from_source(
reply_msg.sender, par, val)
else:
self.logger.error("WTF? {0}".format(reply.message))
# print '.',#"{0} external params still unset".format(_unset_param_count())
time.sleep(0.4)
ready, details = self.core.config_ready()
retries -= 1
if ready:
self.logger.info("External parameters initialised %s",
str(self.core.config_ready()))
return ready, details
def _request_ext_params(self, connection, retries=400):
# TODO set timeout and retry count
self.logger.info("requesting external parameters")
if self.peer is None:
raise NoPeerError
def _unset_param_count():
return reduce(lambda x, y: x + y,
[len(self.core.unset_params_for_source(src))
for src in self.core.used_config_sources()], 0)
ready, details = self.core.config_ready()
while not ready and retries:
for src in self.core.used_config_sources():
params = list(self.core.unset_params_for_source(src).keys())
msg = cmsg.fill_msg(types.GET_CONFIG_PARAMS,
sender=self.peer_id,
param_names=params,
receiver=self.core.config_sources[src])
# print "requesting: {0}".format(msg)
reply = self.__query(connection, cmsg.pack_msg(msg),
types.GET_CONFIG_PARAMS)
if reply is None:
# raise something?
continue
if reply.type == types.CONFIG_ERROR:
self.logger.warning("peer {0} has not yet started".format(msg.receiver))
elif reply.type == types.CONFIG_PARAMS:
reply_msg = cmsg.unpack_msg(reply.type, reply.message)
params = cmsg.params2dict(reply_msg)
for par, val in params.items():
self.core.set_param_from_source(reply_msg.sender, par, val)
else:
self.logger.error("WTF? {0}".format(reply.message))
# print '.',#"{0} external params still unset".format(_unset_param_count())
time.sleep(0.4)
ready, details = self.core.config_ready()
retries -= 1
if ready:
self.logger.info("External parameters initialised %s",
str(self.core.config_ready()))
return ready, details
def handle_config_message(self, mxmsg):
if mxmsg.type in cmsg.MX_CFG_MESSAGES:
message = cmsg.unpack_msg(mxmsg.type, mxmsg.message)
msg, mtype = self._call_handler(mxmsg.type, message)
if msg is None:
self.peer.no_response()
else:
msg = cmsg.pack_msg(msg)
self.peer.send_message(message=msg,
type=mtype,
to=int(mxmsg.from_),
flush=True)
def run(self):
conn = connect_client(type=peers.CONFIGURER, addresses=settings.MULTIPLEXER_ADDRESSES)
time.sleep(4)
for i in range(5):
msg = cmsg.fill_msg(types.UPDATE_PARAMS, sender="p_a")
params = dict(p=str(i))
cmsg.dict2params(params, msg)
try:
conn.send_message(message=msg, type=types.UPDATE_PARAMS)
resp, sth = conn.receive_message()
msg = cmsg.unpack_msg(resp.type, resp.message)
print "got ", msg
except Exception as e:
print "error: ", e.args, " ", type(e)
print i
def handle_message(self, mxmsg):
message = cmsg.unpack_msg(mxmsg.type, mxmsg.message)
msg, mtype, launcher_msg = self._call_handler(mxmsg.type, message)
if msg is None:
self.no_response()
else:
msg = cmsg.pack_msg(msg)
if self.__to_all:
self.send_message(message=msg, to=0, type=mtype, flush=True)
self.__to_all = False
else:
self.send_message(message=msg, to=int(mxmsg.from_), type=mtype, flush=True)
if launcher_msg is not None and self.launcher_sock is not None:
self.logger.info('SENDING msg ' + launcher_msg[:100] + '[...]')
send_msg(self.launcher_sock, launcher_msg)
time.sleep(0.1) # TODO - temporary kind-of bug fix...
def handle_message(self, mxmsg):
message = cmsg.unpack_msg(mxmsg.type, mxmsg.message)
msg, mtype, launcher_msg = self._call_handler(mxmsg.type, message)
if msg is None:
self.no_response()
else:
msg = cmsg.pack_msg(msg)
if self.__to_all:
self.send_message(message=msg, to=0, type=mtype, flush=True)
self.__to_all = False
else:
self.send_message(message=msg, to=int(mxmsg.from_), type=mtype, flush=True)
if launcher_msg is not None and self.launcher_sock is not None:
self.logger.info('SENDING msg ' + launcher_msg[:100] + '[...]')
send_msg(self.launcher_sock, launcher_msg)
time.sleep(0.1) # TODO - temporary kind-of bug fix...
def restart_scenario(conn, new_scenario, comment="Wait...", leave_on=[], overwrites=[]):
"""
new_scenario: scenario_path relative to obci_root
overwrites: {'peer_id' : ['-p', 'param_name', 'param_value',
'-p', 'etc', 'etc',
'-f' 'config_ini_file_path_relative_to_obci_root']}
leave_on: ['list', 'of', 'peer_ids', 'which', 'are', 'in', 'both', 'scenarios',
'and', 'we', 'do','not', 'want', 'them', 'to', 'restart']
"""
if new_scenario.startswith('/') or new_scenario.startswith('~'):
new_scenario = os.path.expanduser(new_scenario)
else:
new_scenario = os.path.join(obci_root(), new_scenario)
conf_msg = cmsg.fill_msg(types.GET_CONFIG_PARAMS,
sender='',
param_names=['experiment_uuid'],
receiver='config_server')
try:
reply = conn.query(message=conf_msg,
type=types.GET_CONFIG_PARAMS)
except OperationFailed:
print "OperationFailed (in restart_scenario) Could not connect to config server"
reply = None
except OperationTimedOut:
print "Operation timed out! (in restart_scenario) (could not connect to config server)"
reply = None
if reply == None:
return
if reply.type == types.CONFIG_ERROR:
print "(in restart_scenario) could not acquire current experiment uuid"
return
elif reply.type == types.CONFIG_PARAMS:
reply_msg = cmsg.unpack_msg(reply.type, reply.message)
params = cmsg.params2dict(reply_msg)
if not 'experiment_uuid' in params:
print "no experiment_uuid!!!"
else:
uid = params['experiment_uuid']
if not uid:
print 'experiment uuid unset!!!'
else:
ovr_list = ['--ovr']
for peer in overwrites:
scan = overwrites[peer]
params = overwrites[peer]
cut = 0
while scan:
if '-f' in scan:
ind = scan.index('-f')
params[cut+ind+1] = os.path.join(obci_root(), params[cut+ind+1])
cut = cut+ind+1
scan = params[cut:]
else: break
# for i in range(len(params)):
# params[i] = params[i].replace(" ", "\ ")
# print params[i]
ovr_list += ['--peer', peer]
ovr_list += params
# leave_on_str = '' if not leave_on else ' --leave_on ' + ' '.join(leave_on)
# overwr_str = '' if not overwrites else ' '.join(ovr_list)
# command = "sleep 0.5 && obci morph " + str(uid) + " " + new_scenario + \
# " " + leave_on_str + overwr_str + " &"
subpr_call = ['obci', 'morph', str(uid), new_scenario, '--leave_on'] + leave_on + ovr_list
print "---------------\n", subpr_call, "\n--------------"
subprocess.call(subpr_call)
def restart_scenario(conn, new_scenario, comment="Wait...", leave_on=[], overwrites=[]):
"""
new_scenario: scenario_path relative to obci_root
overwrites: {'peer_id' : ['-p', 'param_name', 'param_value',
'-p', 'etc', 'etc',
'-f' 'config_ini_file_path_relative_to_obci_root']}
leave_on: ['list', 'of', 'peer_ids', 'which', 'are', 'in', 'both', 'scenarios',
'and', 'we', 'do','not', 'want', 'them', 'to', 'restart']
"""
if new_scenario.startswith('/') or new_scenario.startswith('~'):
new_scenario = os.path.expanduser(new_scenario)
else:
new_scenario = os.path.join(obci_root(), new_scenario)
conf_msg = cmsg.fill_msg(types.GET_CONFIG_PARAMS,
sender='',
param_names=['experiment_uuid'],
receiver='config_server')
try:
reply = conn.query(message=conf_msg,
type=types.GET_CONFIG_PARAMS)
except OperationFailed:
print "OperationFailed (in restart_scenario) Could not connect to config server"
reply = None
except OperationTimedOut:
print "Operation timed out! (in restart_scenario) (could not connect to config server)"
reply = None
if reply == None:
return
if reply.type == types.CONFIG_ERROR:
print "(in restart_scenario) could not acquire current experiment uuid"
return
elif reply.type == types.CONFIG_PARAMS:
reply_msg = cmsg.unpack_msg(reply.type, reply.message)
params = cmsg.params2dict(reply_msg)
if not 'experiment_uuid' in params:
print "no experiment_uuid!!!"
else:
uid = params['experiment_uuid']
if not uid:
print 'experiment uuid unset!!!'
else:
ovr_list = ['--ovr']
for peer in overwrites:
scan = overwrites[peer]
params = overwrites[peer]
cut = 0
while scan:
if '-f' in scan:
ind = scan.index('-f')
params[cut+ind+1] = os.path.join(params[cut+ind+1])
cut = cut+ind+1
scan = params[cut:]
else: break
# for i in range(len(params)):
# params[i] = params[i].replace(" ", "\ ")
# print params[i]
ovr_list += ['--peer', peer]
ovr_list += params
# leave_on_str = '' if not leave_on else ' --leave_on ' + ' '.join(leave_on)
# overwr_str = '' if not overwrites else ' '.join(ovr_list)
# command = "sleep 0.5 && obci morph " + str(uid) + " " + new_scenario + \
# " " + leave_on_str + overwr_str + " &"
obci_call = ["obci"]
try:
subprocess.call(obci_call)
except: # (OSError, WindowsError) can't call launcher like that
obci_call = ["python", os.path.join(obci_root(), "control", "launcher", "obci_script.py")]
subpr_call = obci_call + ['morph', str(uid), new_scenario, '--leave_on'] + leave_on + ovr_list
print "---------------\n", subpr_call, "\n--------------"
subprocess.call(subpr_call)