print "Experiment results will not be archived, you can change this by editing the configuration file at \"" + c["config_file"] + "\""
c.update()
retry = True
done = True
while retry:
try:
print bcolors.OKBLUE + 'Connecting to server...' + bcolors.ENDC
serverconn = ServerConnection()
if not serverconn.connect(do_login = False):
raise Exception("Could not connect.")
serverconn.initialize_client()
done = True
retry = False
except Exception as e:
done = False
print bcolors.FAIL + "Error initializing: " + str(e) + bcolors.ENDC
print bcolors.OKBLUE + "Want to retry? " + bcolors.ENDC
ans = raw_input()
if ans.lower() == "yes" or ans.lower() == "y" or ans.lower() == "Y":
print bcolors.OKBLUE + 'Retrying...' + bcolors.ENDC
retry = True
else:
retry = False
serverconn.disconnect()
if not done:
print bcolors.FAIL + "Client not initialized!" + bcolors.ENDC
exit(1)
def centinel_run(args):
if not os.path.exists(conf.c['logs_dir']):
log("i", "Creating logs directory in %s." % (conf.c['logs_dir']))
os.makedirs(conf.c['logs_dir'])
logging.basicConfig(filename=os.path.join(conf.c['logs_dir'], strftime("%Y-%m-%d-%H:%M:%S") + ".log"), level=logging.DEBUG)
args.pop(0)
possible_arguments = ["-r", "--run", "-i", "--input", "-o", "--output"]
ind = -1
if "-r" in args:
ind = args.index("-r")
elif "--run" in args:
ind = args.index("--run")
if ind <> -1:
if ind + 1 >= len(args) or (ind + 1 < len(args) and args[ind + 1] in possible_arguments):
print "Error: no experiment file specified!"
exit (1)
else:
source_address = args[ind + 1]
ind = 0
if "-i" in args:
ind = args.index("-i")
elif "--input" in args:
ind = args.index("--input")
if ind:
if ind + 1 >= len(args) or (ind + 1 < len(args) and args[ind + 1] in possible_arguments):
print "Error: no input file specified!"
exit (1)
else:
input_file = args[ind + 1]
else:
input_file = os.path.splitext(source_address)[0] + ".txt"
print "No input specified, using \"%s\"." %(input_file)
ind = 0
if "-o" in args:
ind = args.index("-o")
elif "--ouput" in args:
ind = args.index("--output")
if ind:
if ind + 1 >= len(args) or (ind + 1 < len(args) and args[ind + 1] in possible_arguments):
print "Error: no output file specified!"
exit (1)
else:
output_file = args[ind + 1]
else:
output_file = os.path.splitext(source_address)[0] + ".json"
print "No output specified, using \"%s\"." %(output_file)
try:
#exp = open(args[args.index("--run") + 1], "r")
execute_python_experiment_from_source(source_address, input_file, output_file)
return 0
except Exception as e:
print "Error running the experiment: " + str(e)
exit (1)
print open("centinel_client_ascii_art", "r").read()
global serverconn
serverconn = ServerConnection()
log("i", "Client daemon is running...")
if not serverconn.connect():
log("e", 'Server not connected.')
while 1:
try:
if not serverconn.logged_in:
raise Exception("Not logged in.")
if not last_checked_for_updates or ((datetime.now() - last_checked_for_updates).seconds > int(conf.c['update_check_delay'])):
update_check()
if not experiments_last_synced or ((datetime.now() - experiments_last_synced).seconds > int(conf.c['experiment_sync_delay'])):
sync_exp()
if not results_last_synced or ((datetime.now() - results_last_synced).seconds > int(conf.c['result_sync_delay'])):
sync_res()
if not logs_last_sent or ((datetime.now() - logs_last_sent).seconds > int(conf.c['log_send_delay'])):
send_logs()
server_response = serverconn.beat()
if not server_response:
serverconn.disconnect()
raise Exception("No server response received.")
elif server_response <> 'beat':
log("i", "Executing commands... (" + server_response + ")")
for command in server_response.split(";"):
try:
command = command.strip()
if command == '':
continue
elif command == "sync_results" or command == "sync_res":
sync_res()
elif command == "send_logs":
send_logs()
elif command == "sync_experiments" or command == "sync_exp":
sync_exp()
elif command.split()[0] == "run_exp" or command.split()[0] == "run":
run_exp(command.split()[1:])
elif command == "die":
die()
else:
log("e", "Command %s not recognized." %(command))
except Exception as e:
log("e", "Command %s failed to execute: " %(command) + str(e))
time.sleep(5) # Sleep for heartbeat duration.
except (KeyboardInterrupt, SystemExit):
log("w", "Shutdown requested, shutting centinel down...")
serverconn.disconnect()
# do some shutdown stuff, then close
return 0
except UpdateException:
log("w", "Centinel update package received, shutting down to apply updates...")
return 2
except Exception as e:
log("e", "An exception occured: " + str(e))
log("i", "Trying to recover...")
fixed = False
try:
while not fixed:
try:
serverconn.disconnect()
fixed = serverconn.connect() and serverconn.logged_in
if fixed:
break
except Exception as e:
log("e", "An exception occured when trying to recover: " + str(e))
log("i", "Rerying...")
time.sleep(5) # Sleep before retrying
fixed = False
except (KeyboardInterrupt, SystemExit):
log("w", "Shutdown requested, shutting centinel down...")
# do some shutdown stuff, then close
serverconn.disconnect()
return 0
log("s", "We're back in business!")
serverconn.disconnect()