def main():
# CHANGE THE Constants.TO_RUN VARIABLE TO CHANGE THE ALGORITHM BEING RAN. OPTIONS:
# BFS
# DIJKSTRA
# ASTAR
algorithmToRun = Constants.TO_RUN
pygame.init()
SCREEN = pygame.display.set_mode(Constants.SCREEN)
SCREEN.fill(Constants.WHITE)
grid = Grid(Constants.SCREEN_WIDTH_AND_HEIGHT, 10)
draw = Draw(SCREEN, grid.get_grid(), grid.get_room_per_node())
draw.draw_grid()
pygame.display.flip()
run = Run(grid, draw)
if algorithmToRun == Constants.BFS:
run.run_bfs()
elif algorithmToRun == Constants.DIJKSTRA:
run.run_dijkstra()
elif algorithmToRun == Constants.ASTAR:
run.run_astar()
else:
run.run_bfs()
def checkTrophies():
try :
userId = request.args.get('userId')
runId = request.args.get('runId')
thisRunner = AuthenticatedUser(userId)
if thisRunner == None :
return jsonify(error="User not found")
thisRun = Run(thisRunner.sessionToken, runId)
return thisRun.getTrophies()
except :
return jsonify(error="No goals completed")
def __init__(self, Agent, run_config):
Run.__init__(self, Agent, run_config)
def gammaExperiment(Run):
print("Running Experiment for Gamma")
for episode in range(run_config['experiment_episodes']):
for gamma in run_config['gammas']:
print("Training for Gamma: {}".format(gamma))
Train(HPSearch.Agent, HPSearch.run_config)
HPSearch.gamma_rewards.append(
HPSearch.Agent.training_episode_rewards)
def update_Run(self):
#Selection/Creation Phase
if self.run==None:
self.run = Run(self.actor);
#Initialization Phase
if not(self.run.bStarted) and self.actor.getRun()>0.3:
self.run.startAct(self.nFrame);
#Updating Phase
if self.actor.bMissing: #If the subject dissapeared from screen
self.run.update_Missing_Actor(self.nFrame);
elif self.run.bStarted: #elif has started
self.run.updateAct(self.nFrame);
def __init__(self, master=None):
Frame.__init__(self, master)
self.pack()
self.v = IntVar()
self.v.set(1)
self.m = 3
self.run = Run(self.m, self.m)
self.run.init_aim()
self.p = self.run.random_state()
self.imagel = []
self.image_list2 = []
self.path = []
self.create_widget()
def recalc():
locationindex = locationbox.current()
satelliteindex = satellitebox.current()
satellitename = satellitebox.get()
minelevation1 = int(minelevation.get())
stationlon1 = float(lonvector[locationindex])
stationlat1 = float(latvector[locationindex])
stationname = str(locvector[locationindex])
stationrate = int(ratevector[locationindex])
hstation1 = float(hvector[locationindex])
mode1 = mode.get()
loopdays = int(days.get())
loophours = int(hours.get())
loopminutes = int(minutes.get())
minback1 = int(minback.get())
PosStation = [
stationlon1, stationlat1, hstation1, stationname, stationrate
]
priority1 = priority.get()
Run(mode1, line0, line1, line2, PosStation, satelliteindex, indexvector,
priorityvector, priority1, loopdays, loophours, loopminutes, minback1,
info, minelevation1)
#plot
return
def __init__(self):
# Define names of lock and last_used_setup files
self.lock_file = "run/lock"
self.lus_file = "setup/last_used_setup"
# Redefine print to send output to log file
sys.stdout = Logger()
# Create lock file
if (self.create_lock_file() == "error"): exit(1)
# Define what setup to use at startup (can be changed interactively)
initial_setup = self.get_initial_setup()
print "=== Starting PADME Run Control server with %s setup"%initial_setup
#self.write_log("=== Starting PADME Run Control server with %s setup"%initial_setup)
# Create run
self.run = Run()
if (self.run.change_setup(initial_setup) == "error"):
print "ERROR - Error while changing run setup to %s"%initial_setup
#self.write_log("ERROR - Error while changing run setup to %s"%initial_setup)
exit(1)
# Start in idle state
self.current_state = "idle"
# Create handler for PadmeDB
self.db = PadmeDB()
# Create useful regular expressions
self.re_get_board_config = re.compile("^get_board_config (\d+)$")
self.re_get_board_log_file = re.compile("^get_board_log_file (\d+)$")
self.re_change_setup = re.compile("^change_setup (\w+)$")
# Create a TCP/IP socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Bind the socket to the port
server_address = ('localhost', 10000)
print "Starting server socket on %s port %s"%server_address
#self.write_log("Starting server socket on %s port %s"%server_address)
try:
self.sock.bind(server_address) # Listen for incoming connections
except:
print "ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0])
#self.write_log("ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0]))
exit(1)
self.sock.listen(1)
# Define SIGINT handler
signal.signal(signal.SIGINT,self.sigint_handler)
# Setup main interface
self.main_loop()
# Clean up before exiting
self.sock.close()
def __init__(self, paths, props):
self.cmipPaths = []
self.paths = {}
for k in paths.keys():
if k == 'cmip':
self.cmipPaths = paths[k]
else:
self.paths[k] = paths[k]
self.cmipPaths.append({'hs': paths['input_pdb_path']})
self.properties = {}
self.properties['step'] = props['step']
for k in props.keys():
if k == 'cmip':
self.properties['cmipkwds'] = props[k]
else:
self.properties[k] = props[k]
gr = self._prepGrid()
if 'pbfocus' in self.properties['cmipkwds'] and\
self.properties['cmipkwds']['pbfocus'] == 1:
gr0 = _prepGrid(1)
inpP = InputParams(self.properties['step'], gr, gr0)
else:
inpP = InputParams(self.properties['step'], gr)
for prm in self.properties['cmipkwds'].keys():
inpP.addKeyword({prm.upper(): self.properties['cmipkwds'][prm]})
self.run = Run(inpP, 'wrapper')
if 'titration' in self.properties['cmipkwds']:
self.run.exefile = self.properties['CMIP_root'] + "/src/titration"
else:
self.run.exefile = self.properties['CMIP_root'] + "/src/cmip"
for fn in self.cmipPaths:
self.run.addFile(fn)
if 'vdw' not in self.run.files:
self.run.addFile(
{'vdw': self.properties['CMIP_root'] + '/dat/vdwprm'})
def Parameters():
print("")
print("CAPPTA S.A.")
print("-----------")
options = [
"1 - todas as vendas", "2 - vendas canceladas", "3 - vendas stone"
]
print("Diga algo..")
print(options)
while True:
request = input("Eu: ")
response = Run(request)
print("Bot: " + response)
def createHeatMap(runFile, i):
run = Run(runFile)
run.loadRun(runFile)
fileOP = re.split("/", runFile)
file = fileOP[len(fileOP)-1]
fileName = file.split(".")
print runFile
print fileName[0]
run.createHeatMapRaw("HeatMap_"+fileName[0])
def read_gpx_files(self, runs_directory):
filenames = os.listdir(runs_directory)
for filename in filenames:
gpx_data = gpxpy.parse(open(runs_directory + '/' + filename))
date = gpx_data.name[0:10]
time = gpx_data.name[11:20]
distance = gpx_data.length_3d()
duration = gpx_data.get_duration()
coordinates = [[point.latitude, point.longitude, point.elevation]
for track in gpx_data.tracks
for segment in track.segments
for point in segment.points]
self.runs.append(Run(date, time, distance, duration, coordinates))
class CMIPWrapper():
"""Wrapper class for the 2.7 version of CMIP
Adapted from CMIP standard python wrapper
"""
def __init__(self, paths, props):
self.cmipPaths = []
self.paths = {}
for k in paths.keys():
if k == 'cmip':
self.cmipPaths = paths[k]
else:
self.paths[k] = paths[k]
self.cmipPaths.append({'hs': paths['input_pdb_path']})
self.properties = {}
self.properties['step'] = props['step']
for k in props.keys():
if k == 'cmip':
self.properties['cmipkwds'] = props[k]
else:
self.properties[k] = props[k]
gr = self._prepGrid()
if 'pbfocus' in self.properties['cmipkwds'] and\
self.properties['cmipkwds']['pbfocus'] == 1:
gr0 = _prepGrid(1)
inpP = InputParams(self.properties['step'], gr, gr0)
else:
inpP = InputParams(self.properties['step'], gr)
for prm in self.properties['cmipkwds'].keys():
inpP.addKeyword({prm.upper(): self.properties['cmipkwds'][prm]})
self.run = Run(inpP, 'wrapper')
if 'titration' in self.properties['cmipkwds']:
self.run.exefile = self.properties['CMIP_root'] + "/src/titration"
else:
self.run.exefile = self.properties['CMIP_root'] + "/src/cmip"
for fn in self.cmipPaths:
self.run.addFile(fn)
if 'vdw' not in self.run.files:
self.run.addFile(
{'vdw': self.properties['CMIP_root'] + '/dat/vdwprm'})
def launch(self, deleteTmpDir=True):
# using cmd_wrapper
self.out_log, self.err_log = fu.get_logs(path=self.properties['path'],
step=self.properties['step'])
command = cmd_wrapper.CmdWrapper(self.run.setup(), self.out_log,
self.err_log)
command.launch()
# using original CMIP execute
# self.result = self.run.execute
self.result = Result(self.run.tmpdir, self.run.files)
if deleteTmpDir:
self.run.rmTmpDir()
return self.result
def _prepGrid(self, outgrid=0):
if outgrid:
lb = '0'
else:
lb = ''
gr = Grid(outgrid)
if 'readgrid' + lb in self.properties['cmipkwds']:
gr.setreadGrid(self.properties['cmipkwds']['readgrid' + lb])
del self.properties['cmipkwds']['readgrid' + lb]
if 'perfill' + lb in self.properties['cmipkwds']:
gr.setperfill(self.properties['cmipkwds']['perfill' + lb])
del self.properties['cmipkwds']['perfill' + lb]
if 'cen' + lb in self.properties['cmipkwds']:
gr.cen = self.properties['cmipkwds']['cen' + lb]
del self.properties['cmipkwds']['cen' + lb]
if 'dim' + lb in self.properties['cmipkwds']:
gr.dim = self.properties['cmipkwds']['dim' + lb]
del self.properties['cmipkwds']['dim' + lb]
if 'int' + lb in self.properties['cmipkwds']:
gr.int = self.properties['cmipkwds']['int' + lb]
del self.properties['cmipkwds']['int' + lb]
gr.update()
return gr
train_data, test_data = OnehotDatasetSplit(
config['paths']['dataset_path'], config['hyper_param']['test_split_pr'],
config['hyper_param']['batch_size'], config['hyper_param']['max_data'])
# Create a model.
model = getattr(Models, model_type)(test_data.dataset.data.shape[2:])
# Define a optimizer
optimizer = getattr(Optimizer, optimizer_type)(config, model)
# Get a loss function
loss_fn = getattr(LossFunction, loss_type)
# Class creation
Plot = Plot(config, model, optimizer)
Run = Run(config, Plot)
# Loads information from pre trained model
if config['pre_trained']['use_saved_model']:
# Load the stored information
stored_model_info = torch.load(config['pre_trained']['path_saved_model'] +
'/model.pth')
model.load_state_dict(stored_model_info['model_state_dict'])
optimizer.load_state_dict(stored_model_info['optimizer_state_dict'])
if config['utils']['train_model']:
Run.train_model(model, optimizer, loss_fn, train_data, test_data)
#Run.test_model(model, optimizer, loss_fn, train_data, test_data)
class ActivityMngr(object):
def __init__(self, actor, actorMngr):
self.actor = actor;
self.aActors = actorMngr.aActors;
self.aActive_Actors = actorMngr.aActive_Actors;
self.nFrame = -1;
self.approach = dict();
self.meet = dict();
self.loiter = None;
self.run = None;
self.chase1 = None;
self.chase2 = dict();
self.waitAndMeet1 = None;
self.waitAndMeet2 = dict();
self.aActs = [self.approach, self.meet, self.loiter,
self.run, self.chase1, self.chase2,
self.waitAndMeet1, self.waitAndMeet2];
def updateActivities(self, nFrame):
# if self.actor.nTrackId!=126 or nFrame<4900 or nFrame>5090: return;
# if self.actor.nTrackId!=19 or nFrame<4886: return;
# if self.actor.nTrackId!=53: return;
# if self.actor.nTrackId!=99: return;
# self.printCurrentActors(nFrame);
# if self.actor.nTrackId!=99 or nFrame<200 or nFrame>800: return;
if self.actor.nTrackId==24:
nAvgVel = self.actor.getAvgVel_Win(nWin=10);
if nAvgVel<3:
print nFrame, nAvgVel;
self.nFrame = nFrame;
# # self.update_Approach();
# # self.update_Loiter();
# # self.update_WaitAndMeet1();
# # self.update_WaitAndMeet2();
# self.update_Meet();
# self.update_Run();
# self.update_Chase1();
# self.update_Chase2();
def check_Active(self): #Called only when this Actor is Missing
for act in self.aActs:
if type(act)==dict:
for nTrackId in act:
if act[nTrackId].bStarted: return True;
elif act!=None and act.bStarted: return True;
def select_Approach_Actors(self): #By Distance Closing within Approach Zone & Person's Field of View
if self.actor.bMissing: return [];
aOtherActors = [];
for otherActor in self.aActive_Actors.values():
# if otherActor.nTrackId!=81: continue;
if otherActor==self.actor or otherActor.bMissing: continue;
if self.actor.aPastPos==[] or otherActor.aPastPos==[]: continue;
if not(otherActor.activityMngr.loiter.bStarted): continue; #TEMPORAL CODE [Later make an ApproachToLoiter activity]
nPastDist = self.actor.points_to_Dist(self.actor.aPastPos, otherActor.aPastPos);
nNewDist = self.actor.points_to_Dist(self.actor.aPos, otherActor.aPos);
nDistOtherActor = self.actor.getDist(otherActor);
bClosingDist = nNewDist<nPastDist;
bApproachZone = nDistOtherActor<100 and nDistOtherActor>30;
bNotCreated = not(self.approach.has_key(otherActor.nTrackId));
if bClosingDist and bApproachZone and bNotCreated and self.actor.checkFOV(otherActor.aPos):
aOtherActors.append(otherActor);
# print self.nFrame, "Person", self.actor.nTrackId, "Approaching to:", otherActor.nTrackId, nDistOtherActor;#"Pos:", otherActor.aPos; #, aActorTracksId;
# print self.nFrame, self.actor.nTrackId, otherActor.nTrackId, self.actor.nHeading, pylab.rad2deg(self.actor.nHeading), self.actor.aPastPos, self.actor.aPos, otherActor.aPos;
return aOtherActors;
def update_Approach(self):
#Selection/Creation Phase
aOtherActors = self.select_Approach_Actors();
for otherActor in aOtherActors:
nOtherTrackId = otherActor.nTrackId;
self.approach[nOtherTrackId] = Approach(self.actor, otherActor);
for nOtherTrackId in self.approach:
#Initialization Phase
otherActor = self.aActors[nOtherTrackId];
if not(self.approach[nOtherTrackId].bStarted) and self.actor.nAvgVel>0:
if not(otherActor.activityMngr.loiter.bStarted): continue; #TEMPORAL CODE [Later make an ApproachToLoiter activity]
nPastDist = self.actor.points_to_Dist(self.actor.aPastPos, otherActor.aPastPos);
nNewDist = self.actor.points_to_Dist(self.actor.aPos, otherActor.aPos);
nDistOtherActor = self.actor.getDist(otherActor);
bApproachZone = nDistOtherActor<100 and nDistOtherActor>30;
bClosingDist = nNewDist<nPastDist;
if bApproachZone and bClosingDist:
# print self.nFrame, "Person", self.actor.nTrackId, "Approaching to:", otherActor.nTrackId, nDistOtherActor;#"Pos:", otherActor.aPos; #, aActorTracksId;
self.approach[nOtherTrackId].startAct(self.nFrame);
#Updating Phase
bMissing_OtherActor = otherActor.bMissing;
bNotChase2 = not(self.chase2.has_key(otherActor)) or not(self.chase2[otherActor].bActive);
if bMissing_OtherActor and bNotChase2: #If the other subject dissapeared, then Kill this Approach (copied from Meet activity)
self.approach[nOtherTrackId].nProb = 0;
self.approach[nOtherTrackId].checkBelief_Threshold(self.nFrame);
# print "Killing actor", nTrackId;
continue;
if self.actor.bMissing: #If the subject dissapeared from screen
self.approach[nOtherTrackId].update_Missing_Actor(self.nFrame);
elif self.approach[nOtherTrackId].bStarted: #elif has started
self.approach[nOtherTrackId].updateAct(self.nFrame);
def printCurrentActors(self, nFrame):
aActorTracksId = [];
for actor in self.aActive_Actors.values():
if actor.bMissing: continue;
aActorTracksId.append(actor.nTrackId);
aActorTracksId.sort();
print nFrame, aActorTracksId;
def update_Loiter(self):
#Selection/Creation Phase
if self.loiter==None:
self.loiter = Loiter(self.actor);
#Initialization Phase
if not(self.loiter.bStarted) and self.actor.getAvgVel_Win(nWin=3)==0:
self.loiter.startAct(self.nFrame);
#Updating Phase
if self.actor.bMissing: #If the subject dissapeared from screen
self.loiter.update_Missing_Actor(self.nFrame);
elif self.loiter.bStarted: #elif has started
self.loiter.updateAct(self.nFrame);
def update_Meet(self):
#Selection/Creation Phase
aOtherActors = self.select_Meet_Actors();
for otherActor in aOtherActors:
nOtherTrackId = otherActor.nTrackId;
self.meet[nOtherTrackId] = Meet(self.actor, otherActor);
for nOtherTrackId in self.meet:
#Initialization Phase
otherActor = self.aActors[nOtherTrackId];
if not(self.meet[nOtherTrackId].bStarted):
bBoth_LowSpeed = self.actor.nAvgVel < 30 and otherActor.nAvgVel < 30;
bCloseDist = self.actor.getDist(otherActor)<30;
bLoiter = self.loiter.bActive;
if bCloseDist and bLoiter and bBoth_LowSpeed:
# print self.nFrame, "Person", self.actor.nTrackId, "Meeting with:", otherActor.nTrackId, otherApproach[nTrackId].nInitRange, otherApproach[nTrackId].nProb;
self.meet[nOtherTrackId].startAct(self.nFrame);
#Updating Phase
bMissing_OtherActor = otherActor.bMissing;
if bMissing_OtherActor: #If the other subject dissapeared, then Kill this Meet
self.meet[nOtherTrackId].nProb = 0;
self.meet[nOtherTrackId].checkBelief_Threshold(self.nFrame);
continue;
if self.actor.bMissing: #If the subject dissapeared from screen
self.meet[nOtherTrackId].update_Missing_Actor(self.nFrame);
elif self.meet[nOtherTrackId].bStarted: #elif has started
self.meet[nOtherTrackId].updateAct(self.nFrame);
def select_Meet_Actors(self):
if self.actor.bMissing: return [];
aOtherActors = [];
for otherActor in self.aActive_Actors.values():
if otherActor==self.actor or otherActor.bMissing: continue;
bBoth_LowSpeed = self.actor.nAvgVel < 30 and otherActor.nAvgVel < 30;
bCloseDist = self.actor.getDist(otherActor)<30;
bLoiter = self.loiter.bActive;
bNotCreated = not(self.meet.has_key(otherActor.nTrackId));
if bCloseDist and bLoiter and bBoth_LowSpeed and bNotCreated:
aOtherActors.append(otherActor);
# print self.nFrame, "Person", self.actor.nTrackId, "Meeting with:", otherActor.nTrackId, otherApproach[nTrackId].nInitRange, otherApproach[nTrackId].nProb; #, "Pos:", otherActor.aPos;
return aOtherActors;
def update_Run(self):
#Selection/Creation Phase
if self.run==None:
self.run = Run(self.actor);
#Initialization Phase
if not(self.run.bStarted) and self.actor.getRun()>0.3:
self.run.startAct(self.nFrame);
#Updating Phase
if self.actor.bMissing: #If the subject dissapeared from screen
self.run.update_Missing_Actor(self.nFrame);
elif self.run.bStarted: #elif has started
self.run.updateAct(self.nFrame);
def update_Chase1(self):
#Selection/Creation Phase
if self.chase1==None:
self.chase1 = Chase1(self.actor);
#Initialization Phase
if not(self.chase1.bStarted) and self.run.bActive:
self.chase1.startAct(self.nFrame);
#Updating Phase
if self.actor.bMissing: #If the subject dissapeared from screen
self.chase1.update_Missing_Actor(self.nFrame);
elif self.chase1.bStarted: #elif has started
self.chase1.updateAct(self.nFrame);
def update_Chase2(self):
#Selection/Creation Phase
aOtherActors = self.select_Chase2_Actors();
for otherActor in aOtherActors:
nTrackId = otherActor.nTrackId;
self.chase2[nTrackId] = Chase2(self.actor, otherActor);
for nOtherTrackId in self.chase2:
#Initialization Phase
otherActor = self.aActors[nOtherTrackId];
if not(self.chase2[nOtherTrackId].bStarted):
otherChase1 = otherActor.activityMngr.chase1;
bOtherChase1 = otherChase1!=None and otherChase1.bActive;
if self.run.bActive and bOtherChase1:
# print self.nFrame, "Person", self.actor.nTrackId, "Meeting with:", otherActor.nTrackId, otherApproach[nTrackId].nInitRange, otherApproach[nTrackId].nProb;
self.chase2[nOtherTrackId].startAct(self.nFrame);
#Updating Phase
if self.actor.bMissing: #If the subject dissapeared from screen
self.chase2[nOtherTrackId].update_Missing_Actor(self.nFrame);
elif self.chase2[nOtherTrackId].bStarted: #elif has started
self.chase2[nOtherTrackId].updateAct(self.nFrame);
def select_Chase2_Actors(self): #By having Run(thisPerson), Chase1(*,otherPerson) & within Person's Field of View (if still in screen)
if self.actor.bMissing: return [];
aOtherActors = [];
for otherActor in self.aActive_Actors.values():
if otherActor==self.actor: continue;
otherChase1 = otherActor.activityMngr.chase1;
bOtherChase1 = otherChase1!=None and otherChase1.bActive;
bNotCreated = not(self.chase2.has_key(otherActor.nTrackId));
bOtherMissing = otherActor.bMissing;
if self.run.bActive and bNotCreated and bOtherChase1 and (bOtherMissing or self.actor.checkFOV(otherActor.aPos)):
aOtherActors.append(otherActor);
# print self.nFrame, "Person", self.actor.nTrackId, "Approaching to:", otherActor.nTrackId, nDistOtherActor;#"Pos:", otherActor.aPos; #, aActorTracksId;
# print self.nFrame, self.actor.nTrackId, otherActor.nTrackId, self.actor.nHeading, pylab.rad2deg(self.actor.nHeading), self.actor.aPastPos, self.actor.aPos, otherActor.aPos;
return aOtherActors;
def update_WaitAndMeet1(self):
#Selection/Creation Phase
if self.waitAndMeet1==None:
self.waitAndMeet1 = WaitAndMeet1(self.actor);
#Initialization Phase
if not(self.waitAndMeet1.bStarted) and self.loiter.bActive:
self.waitAndMeet1.startAct(self.nFrame);
#Updating Phase
if self.actor.bMissing: #If the subject dissapeared from screen
self.waitAndMeet1.update_Missing_Actor(self.nFrame);
elif self.waitAndMeet1.bStarted: #elif has started
self.waitAndMeet1.updateAct(self.nFrame);
def update_WaitAndMeet2(self):
#Selection/Creation Phase
aOtherActors = self.select_WaitAndMeet2_Actors();
for otherActor in aOtherActors:
nOtherTrackId = otherActor.nTrackId;
self.waitAndMeet2[nOtherTrackId] = WaitAndMeet2(self.actor, otherActor);
for nOtherTrackId in self.waitAndMeet2:
#Initialization Phase
otherActor = self.aActors[nOtherTrackId];
if not(self.waitAndMeet2[nOtherTrackId].bStarted):
nTrackId = self.actor.nTrackId;
bWaitAndMeet1 = self.waitAndMeet1!=None and self.waitAndMeet1.bActive;
otherApproach = otherActor.activityMngr.approach;
bOtherApproach = otherApproach.has_key(nTrackId) and otherApproach[nTrackId].bStarted;
if not(bWaitAndMeet1 and bOtherApproach): continue;
self.update_Meet();
bMeet = self.meet.has_key(nOtherTrackId) and self.meet[nOtherTrackId].bStarted;
if bWaitAndMeet1 and bOtherApproach and bMeet:
# print self.nFrame, "Person", self.actor.nTrackId, "Meeting with:", otherActor.nTrackId, otherApproach[nTrackId].nInitRange, otherApproach[nTrackId].nProb;
self.waitAndMeet2[nOtherTrackId].startAct(self.nFrame);
#Updating Phase
bMissing_OtherActor = otherActor.bMissing;
if bMissing_OtherActor: #If the other subject dissapeared, then Kill this Meet
self.waitAndMeet2[nOtherTrackId].nProb = 0;
self.waitAndMeet2[nOtherTrackId].checkBelief_Threshold(self.nFrame);
continue;
if self.actor.bMissing: #If the subject dissapeared from screen
self.waitAndMeet2[nOtherTrackId].update_Missing_Actor(self.nFrame);
elif self.waitAndMeet2[nOtherTrackId].bStarted: #elif has started
self.waitAndMeet2[nOtherTrackId].updateAct(self.nFrame);
def select_WaitAndMeet2_Actors(self):
if self.actor.bMissing: return [];
aOtherActors = [];
for otherActor in self.aActive_Actors.values():
if otherActor==self.actor or otherActor.bMissing: continue;
nTrackId = self.actor.nTrackId;
nOtherTrackId = otherActor.nTrackId;
bWaitAndMeet1 = self.waitAndMeet1!=None and self.waitAndMeet1.bActive;
otherApproach = otherActor.activityMngr.approach;
bOtherApproach = otherApproach.has_key(nTrackId) and otherApproach[nTrackId].bStarted;
if not(bWaitAndMeet1 and bOtherApproach): continue;
self.update_Meet();
bMeet = self.meet.has_key(nOtherTrackId) and self.meet[nOtherTrackId].bStarted;
bNotCreated = not(self.waitAndMeet2.has_key(otherActor.nTrackId));
if bWaitAndMeet1 and bOtherApproach and bMeet and bNotCreated:
aOtherActors.append(otherActor);
print self.nFrame, "Person", self.actor.nTrackId, "Meeting with:", otherActor.nTrackId, otherApproach[nTrackId].nInitRange, otherApproach[nTrackId].nProb; #, "Pos:", otherActor.aPos;
return aOtherActors;
inF = open(inputFile, "r")
batch = 1
for line in inF.readlines():
lineSplit = line.split(",")
runFile = lineSplit[0].rstrip()
normalFile = lineSplit[1].rstrip()
weightFile = lineSplit[2].rstrip()
fileOP = re.split("/", runFile)
file = fileOP[len(fileOP) - 1]
runF = file.split(".")[0]
sampleFilter = ""
if (len(lineSplit) > 3):
sampleFilter = lineSplit[3].rstrip()
# Do this repeatedly
run = Run(runFile)
run.loadRun(runFile)
run.setSampleFiler(sampleFilter)
print "##################################################"
print "Processing " + line.rstrip()
run.processNormal(normalFile)
run.exportNormalized(outputDir + "/Normalized_" + runF + ".csv")
print "Normalized!"
run.processWeights(weightFile)
run.exportLipidWeightNormalization(outputDir + "/WeightNormalized_" +
runF + ".csv")
print "Weight Normalization Done!"
run.calcLipidAbundance()
run.exportLipidAbundance(outputDir + "/Abundance_" + runF + ".csv")
print "Lipid Relative Abundance Done!"
del self.wait_queue[i]
self._run_next()
time.sleep(0.5)
pq = ProcessQueue(10)
for delay in delays:
for mom in momenta:
for lr in learning_rates:
for bs in batch_sizes:
run = Run(epochs=epochs,
learning_rate=lr,
batch_size=bs,
delay=delay,
seed=seed,
momentum=mom,
optimizer='sgd')
cmd = run.to_shell()
print(cmd)
pq.queue(cmd)
seed += 1
pq.wait()
def one_run(self):
"""Generate parameters and run the model with these newly defined parameters"""
reference = \
Fuel_Price(local_prices, price_gas, price_coal, local_production, baseline).parameters
run_model = Run(self.scenario, reference)
return run_model.lcoe
def defaultTests(cfg):
"""
run a standard set of interesting simulations
cfg -- default configuration values
"""
disk = Disk(size=cfg.disk_size, fits=cfg.disk_fit,
nre=cfg.disk_nre,
desc="Disk: %s" % (cfg.disk_type))
raid0 = RAID0(disk, volumes=2,
nre_model=cfg.nre_model,
recovery=cfg.raid_recover,
delay=cfg.raid_replace,
objsize=cfg.obj_size)
raid1 = RAID1(disk, volumes=2,
nre_model=cfg.nre_model,
recovery=cfg.raid_recover,
delay=cfg.raid_replace,
objsize=cfg.obj_size)
raid5 = RAID5(disk, volumes=4,
nre_model=cfg.nre_model,
recovery=cfg.raid_recover,
delay=cfg.raid_replace,
objsize=cfg.obj_size)
raid6 = RAID6(disk, volumes=8,
nre_model=cfg.nre_model,
recovery=cfg.raid_recover,
delay=cfg.raid_replace,
objsize=cfg.obj_size)
tests = [disk, raid0, raid5, raid1, raid6]
# single site RADOS
for cp in (1, 2, 3):
rados = RADOS(disk, pg=cfg.rados_decluster,
copies=cp,
speed=cfg.rados_recover,
fullness=cfg.rados_fullness,
objsize=cfg.obj_size,
stripe=cfg.stripe_length,
nre_model=cfg.nre_model,
delay=cfg.rados_markout)
tests.append(rados)
# multi-site RADOS
tests.append(None)
site = Site(fits=cfg.majeure, rplc=cfg.site_recover)
tests.append(site)
for sites in (1, 2, 3, 4):
for cp in (1, 2, 3):
rados = RADOS(disk, pg=cfg.rados_decluster,
copies=cp,
speed=cfg.rados_recover,
fullness=cfg.rados_fullness,
objsize=cfg.obj_size,
stripe=cfg.stripe_length,
nre_model=cfg.nre_model,
delay=cfg.rados_markout)
multi = MultiSite(rados, site,
speed=cfg.remote_recover,
latency=cfg.remote_latency,
sites=sites)
tests.append(multi)
# and run them all
Run(tests, period=cfg.period, verbosity=cfg.verbose)
EntryCapital = AccountRisk / StopLossRisk
ExitPrice = Today['open'] - (StopLossRisk * Today['open'] * Reward)
if EntryCapital >= 0:
try:
Account.EnterPosition('Short', EntryCapital, EntryPrice,
ExitPrice, StopLoss)
except ValueError:
pass
except ValueError:
pass
import pandas as pd
from Run import Run
data = pd.read_csv('TSLA.csv')
r = Run(data)
r.Start(1000, Logic, Lookback=1, Start='10/13/2013', End='1/10/2014')
r.Results()
r.AdvancedResults()
r.Chart(ShowTrades=True)
#for trade in r.Account.OpenedTrades:
# print(trade)
# print()
#for trade in r.Account.ClosedTrades:
# print(trade)
# print()
class RunControlServer:
def __init__(self):
# Define names of lock and last_used_setup files
self.lock_file = "run/lock"
self.lus_file = "setup/last_used_setup"
# Redefine print to send output to log file
sys.stdout = Logger()
# Create lock file
if (self.create_lock_file() == "error"): exit(1)
# Define what setup to use at startup (can be changed interactively)
initial_setup = self.get_initial_setup()
print "=== Starting PADME Run Control server with %s setup"%initial_setup
#self.write_log("=== Starting PADME Run Control server with %s setup"%initial_setup)
# Create run
self.run = Run()
if (self.run.change_setup(initial_setup) == "error"):
print "ERROR - Error while changing run setup to %s"%initial_setup
#self.write_log("ERROR - Error while changing run setup to %s"%initial_setup)
exit(1)
# Start in idle state
self.current_state = "idle"
# Create handler for PadmeDB
self.db = PadmeDB()
# Create useful regular expressions
self.re_get_board_config = re.compile("^get_board_config (\d+)$")
self.re_get_board_log_file = re.compile("^get_board_log_file (\d+)$")
self.re_change_setup = re.compile("^change_setup (\w+)$")
# Create a TCP/IP socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Bind the socket to the port
server_address = ('localhost', 10000)
print "Starting server socket on %s port %s"%server_address
#self.write_log("Starting server socket on %s port %s"%server_address)
try:
self.sock.bind(server_address) # Listen for incoming connections
except:
print "ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0])
#self.write_log("ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0]))
exit(1)
self.sock.listen(1)
# Define SIGINT handler
signal.signal(signal.SIGINT,self.sigint_handler)
# Setup main interface
self.main_loop()
# Clean up before exiting
self.sock.close()
def create_lock_file(self):
# Check if lock file exists
if (os.path.exists(self.lock_file)):
if (os.path.isfile(self.lock_file)):
pid = 0
lf = open(self.lock_file,"r")
for ll in lf: pid = ll
lf.close()
print "ERROR - Lock file %s found for pid %s"%(self.lock_file,pid)
#self.write_log("ERROR - Lock file %s found for pid %s"%(self.lock_file,pid))
else:
print "ERROR - Lock file %s found but it is not a file"%self.lock_file
#self.write_log("ERROR - Lock file %s found but it is not a file"%self.lock_file)
return "error"
# Create our own lock file
pid = os.getpid()
lf = open(self.lock_file,"w")
lf.write("%d"%pid)
lf.close()
return "ok"
def get_initial_setup(self):
setup = "test"
lus = ""
if (os.path.exists(self.lus_file)):
if (os.path.isfile(self.lus_file)):
lusf = open(self.lus_file,"r")
lus = lusf.read().strip("\n")
lusf.close()
if (lus == ""):
print "WARNING - File with last used setup %s exists but it is empty - Using default setup %s"%(self.lus_file,setup)
#self.write_log("WARNING - File with last used setup %s exists but it is empty - Using default setup %s"%(self.lus_file,setup))
else:
setup = lus
else:
print "WARNING - File with last used setup %s exists but it is not a file - Using default setup %s"%(self.lus_file,setup)
#self.write_log("WARNING - File with last used setup %s exists but it is not a file - Using default setup %s"%(self.lus_file,setup))
else:
print "WARNING - Could not find file with last used setup %s - Using default setup %s"%(self.lus_file,setup)
#self.write_log("WARNING - Could not find file with last used setup %s - Using default setup %s"%(self.lus_file,setup))
return setup
def sigint_handler(self,signal,frame):
print "RunControlSever received SIGINT: exiting"
# If a run is initialized/running, abort it as cleanly as possible
if ( self.current_state == "initialized" or self.current_state == "running" ):
self.run.run_comment_end = "Run aborted because of SIGINT"
print "Aborting run on SIGINT"
#self.write_log("Aborting run on SIGINT")
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,4) # Status 4: run aborted
self.db.set_run_time_stop(self.run.run_number,self.now_str())
self.db.set_run_comment_end(self.run.run_number,self.run.run_end_comment)
open(self.run.quit_file,'w').close()
for adc in (self.run.adcboard_list):
if adc.stop_daq():
print "ADC board %02d - Terminated correctly"%adc.board_id
#self.write_log("ADC board %02d - Terminated correctly"%adc.board_id)
else:
print "ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id
#self.write_log("ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id)
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,6) # Status 6: run ended with errors
# Clean up run directory
for adc in (self.run.adcboard_list):
if (os.path.exists(adc.initok_file)): os.remove(adc.initok_file)
if (os.path.exists(adc.initfail_file)): os.remove(adc.initfail_file)
if(os.path.exists(self.run.start_file)): os.remove(self.run.start_file)
if(os.path.exists(self.run.quit_file)): os.remove(self.run.quit_file)
if os.path.exists(self.lock_file): os.remove(self.lock_file)
# Now we can exit
exit(0)
def main_loop(self):
while True:
# Wait for a connection
print "Waiting for a connection"
#self.write_log('waiting for a connection')
(self.connection,client_address) = self.sock.accept()
print "Connection from %s"%str(client_address)
#self.write_log('connection from '+str(client_address))
while True:
# Handle connection according to curren status of RunControl
if self.current_state == "idle":
new_state = self.state_idle()
elif self.current_state == "initialized":
new_state = self.state_initialized()
elif self.current_state == "running":
new_state = self.state_running()
elif self.current_state == "initfail":
new_state = self.state_initfail()
else:
print "ERROR: unknown state %s - ABORTING"%self.current_state
#self.write_log("ERROR: unknown state %s - ABORTING"%self.current_state)
new_state = "exit"
# See if status changed
if new_state == "idle" or new_state == "initialized" or new_state == "running" or new_state == "initfail":
self.current_state = new_state
elif new_state == "client_close":
self.connection.close()
break
elif new_state == "exit":
print "=== RunControlSever received exit command: exiting"
self.connection.close()
if os.path.exists(self.lock_file): os.remove(self.lock_file)
exit(0)
else:
print "=== RunControlServer = ERROR: unknown new state %s - ABORTING"%new_state
#self.write_log("ERROR: unknown new state %s - ABORTING"%new_state)
self.connection.close()
if os.path.exists(self.lock_file): os.remove(self.lock_file)
exit(1)
def write_log(self,msg):
print self.now_str()+" "+msg
def now_str(self):
return time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
def state_idle(self):
# Receive and process commands for "idle" state
while True:
cmd = self.get_command()
print "Received command %s"%cmd
#self.write_log('Received command '+cmd)
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_setup_list"):
self.send_answer(self.get_setup_list())
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_run_number"):
self.send_answer(str(self.db.get_last_run_in_db()))
elif (cmd == "new_run"):
res = self.new_run()
if (res == "client_close"):
return "client_close"
elif (res == "error"):
#self.write_log('ERROR while initializing new run')
print "ERROR while initializing new run"
elif (res == "initialized"):
return "initialized"
elif (res == "initfail"):
return "initfail"
else:
#self.write_log("ERROR: new_run returned unknown answer "+res+" (?)")
print "ERROR: new_run returned unknown answer %s (?)"%res
elif (cmd == "exit"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\tShow this help
get_state\tShow current state of RunControl
get_setup\tShow current setup name
get_setup_list\tShow list of available setups
get_board_list\tShow list of boards in use with current setup
get_board_config <b>\tShow current configuration of board <b>
get_run_number\tReturn last run number in DB
change_setup <setup>\tChange run setup to <setup>
new_run\t\tInitialize system for a new run
exit\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config.match(cmd)
if (m):
self.send_answer(self.get_board_config(int(m.group(1))))
found_re = True
m = self.re_change_setup.match(cmd)
if (m):
self.send_answer(self.change_setup(m.group(1)))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown"%cmd
def state_initialized(self):
# Receive and process commands for "initialized" state
while True:
cmd = self.get_command()
#self.write_log('received command '+cmd)
print "Received command %s"%cmd
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_run_number"):
self.send_answer(str(self.run.run_number))
elif (cmd == "abort_run"):
return self.abort_run()
elif (cmd == "start_run"):
return self.start_run()
elif (cmd == "exit"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\tShow this help
get_state\tShow current state of RunControl
get_setup\tShow current setup name
get_board_list\tShow list of boards in use with current setup
get_board_config <b>\tShow current configuration of board <b>
get_board_log_file <b>\tGet name of log file for board <b>
get_run_number\tReturn current run number
start_run\t\tStart run
abort_run\t\tAbort run
exit\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config.match(cmd)
if (m):
self.send_answer(self.get_board_config(int(m.group(1))))
found_re = True
m = self.re_get_board_log_file.match(cmd)
if (m):
self.send_answer(self.get_board_log_file(int(m.group(1))))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown"%cmd
def state_running(self):
# Receive and process commands for "running" state
while True:
cmd = self.get_command()
#self.write_log('received command '+cmd)
print "Received command %s"%cmd
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_run_number"):
self.send_answer(str(self.run.run_number))
elif (cmd == "stop_run"):
return self.stop_run()
elif (cmd == "exit"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\tShow this help
get_state\tShow current state of RunControl
get_setup\tShow current setup name
get_board_list\tShow list of boards in use with current setup
get_board_config <b>\tShow current configuration of board <b>
get_board_log_file <b>\tGet name of log file for board <b>
get_run_number\tReturn current run number
stop_run\t\tStop the run
exit\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config.match(cmd)
if (m):
self.send_answer(self.get_board_config(int(m.group(1))))
found_re = True
m = self.re_get_board_log_file.match(cmd)
if (m):
self.send_answer(self.get_board_log_file(int(m.group(1))))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown"%cmd
return "idle"
def state_initfail(self):
return "idle"
def get_command(self):
# First get length of string
l = ""
for i in range(5): # Max 99999 characters
ch = self.connection.recv(1)
if ch:
l += ch
else:
#self.write_log('no more data from client')
print "Client closed connection"
return "client_close"
ll = int(l)
# Then read the right amount of characters from the socket
cmd = ""
for i in range(ll):
ch = self.connection.recv(1)
if ch:
cmd += ch
else:
#self.write_log('no more data from client')
print "Client closed connection"
return "client_close"
return cmd
def send_answer(self,answer):
if len(answer)<100000:
#self.write_log("Sending answer "+answer)
print "Sending answer %s"%answer
self.connection.sendall("%5.5d"%len(answer)+answer)
else:
#self.write_log('answer too long: cannot send')
print "Answer is too long: cannot send"
def get_board_config(self,brdid):
if brdid in self.run.boardid_list:
return self.run.adcboard_list[self.run.boardid_list.index(brdid)].format_config()
else:
return "ERROR: board id %d does not exist"%brdid
def get_board_log_file(self,brdid):
if brdid in self.run.boardid_list:
return self.run.adcboard_list[self.run.boardid_list.index(brdid)].log_file
else:
return "ERROR: board id %d does not exist"%brdid
def read_setup_list(self):
# Get list of available setups
setup_main_dir = "setup"
setups = []
for top,dirs,files in os.walk(setup_main_dir):
if (top == setup_main_dir):
for setup_dir in dirs: setups.append(setup_dir)
setups.sort()
return setups
def get_setup_list(self):
return str(self.read_setup_list())
def change_setup(self,setup):
# Check if requested setup is known
if not (setup in self.read_setup_list()):
#self.write_log("change_setup - ERROR: request to set unknown setup "+setup)
print "change_setup - ERROR: request to set unknown setup %s"%setup
return "error"
# Change (or reload) setup
if (setup==self.run.setup):
#self.write_log("change_setup - reloading setup "+setup)
print "change_setup - reloading setup %s"%setup
else:
#self.write_log("change_setup - changing setup from "+self.run.setup+" to "+setup)
print "change_setup - changing setup from %s to %s"%(self.run.setup,setup)
self.run.change_setup(setup)
return setup
def new_run(self):
# Retrieve run number - next=next run from DB, dummy=dummy run (i.e. run nr=0)
# Return run number used (0 for dummy run) or "error" for invalid answer
newrun_number = 0
self.send_answer("run_number")
ans = self.get_command()
if (ans=="next"):
newrun_number = self.db.get_last_run_in_db()+1
elif (ans=="dummy"):
newrun_number = 0
elif (ans == "error"):
#self.write_log("run_number - client returned error")
print "run_number - client returned error"
return "error"
elif (ans=="client_close"):
return "client_close"
else:
#self.write_log("run_number - invalid option %s received"%ans)
print "run_number - invalid option %s received"%ans
self.send_answer("error")
return "error"
self.send_answer(str(newrun_number))
# Retrieve run type (TEST,DAQ,COSMIC)
# Return run type used or "error" for invalid answer
newrun_type = ""
self.send_answer("run_type")
ans = self.get_command()
if (ans=="TEST" or ans=="DAQ" or ans=="COSMIC"):
newrun_type = ans
elif (ans == "error"):
self.write_log("run_type - client returned error")
return "error"
elif (ans=="client_close"):
return "client_close"
else:
# self.write_log("run_type - invalid option %s received"%ans)
print "run_type - invalid option %s received"%ans
self.send_answer("error")
return "error"
self.send_answer(newrun_type)
newrun_user = ""
self.send_answer("shift_crew")
ans = self.get_command()
if (ans=="client_close"): return "client_close"
newrun_user = ans
newrun_comment = ""
self.send_answer("run_comment")
ans = self.get_command()
if (ans=="client_close"): return "client_close"
newrun_comment = ans
#self.write_log("Run number: "+str(newrun_number))
#self.write_log("Run type: "+newrun_type)
#self.write_log("Run crew: "+newrun_user)
#self.write_log("Run comment: "+newrun_comment)
print "Run number: %d"%newrun_number
print "Run type: %s"%newrun_type
print "Run crew: %s"%newrun_user
print "Run comment: %s"%newrun_comment
# Set run configuration according to user's request
self.run.change_run(newrun_number)
self.run.run_type = newrun_type
self.run.run_user = newrun_user
self.run.run_comment = newrun_comment
# Check if requested run number was not used before
# Saves the day if more than one RunControl program is running at the same time (DON'T DO THAT!!!)
if (self.run.run_number):
run_is_in_db = self.db.is_run_in_db(self.run.run_number)
if (run_is_in_db):
#self.write_log("ERROR - Run "+str(self.run.run_number)+" is already in the DB: cannot use it again")
#self.write_log("Please check if someone else is using this RunControl before retrying")
print "ERROR - Run %d is already in the DB: cannot use it again"%self.run.run_number
print "Please check if someone else is using this RunControl before retrying"
self.send_answer("error_init")
return "error"
# Create run structure in the DB
#self.write_log("Initializing Run "+str(self.run.run_number))
print "Initializing Run %d"%self.run.run_number
self.run.create_run()
if (self.run.run_number): self.db.set_run_time_init(self.run.run_number,self.now_str())
# Create directory to host log files
#self.write_log("Creating log directory "+self.run.log_dir)
print "Creating log directory %s"%self.run.log_dir
self.run.create_log_dir()
# Write run and boards configuration files
#self.write_log("Writing configuration file "+self.run.config_file)
print "Writing configuration file %s"%self.run.config_file
self.run.write_config()
for adc in (self.run.adcboard_list):
#self.write_log("Writing configuration file "+adc.config_file)
print "Writing configuration file %s"%adc.config_file
adc.write_config()
# Start DAQ for all boards
self.send_answer("start_init")
for adc in (self.run.adcboard_list):
p_id = adc.start_daq()
if p_id:
#self.write_log("ADC board %02d - Started DAQ with process id %d"%(adc.board_id,p_id))
print "ADC board %02d - Started DAQ with process id %d"%(adc.board_id,p_id)
self.send_answer("adc "+str(adc.board_id)+" init")
adc.status = "init"
else:
#self.write_log("ADC board %02d - ERROR: could not start DAQ"%adc.board_id)
print "ADC board %02d - ERROR: could not start DAQ"%adc.board_id
self.send_answer("adc "+str(adc.board_id)+" fail")
adc.status = "fail"
# Wait for all boards to finish initialization
n_try = 0
while(1):
all_boards_init = 1
all_boards_ready = 1
for adc in (self.run.adcboard_list):
# Check if any board changed status
if (adc.status == "init"):
if (os.path.exists(adc.initok_file)):
# Initialization ended OK
#self.write_log("ADC board %02d - Initialized and ready for DAQ"%adc.board_id)
print "ADC board %02d - Initialized and ready for DAQ"%adc.board_id
self.send_answer("adc "+str(adc.board_id)+" ready")
adc.status = "ready"
elif (os.path.exists(adc.initfail_file)):
# Problem during initialization
#self.write_log("ADC board %02d - *** Initialization failed ***"%adc.board_id)
print "ADC board %02d - *** Initialization failed ***"%adc.board_id
self.send_answer("adc "+str(adc.board_id)+" fail")
adc.status = "fail"
else:
# This board is still initializing
all_boards_init = 0
# Check if any board is in fail status
if (adc.status == "fail"): all_boards_ready = 0
if (all_boards_init == 0):
# Some boards are still initializing: keep waiting
n_try += 1
if (n_try>=10):
#self.write_log("*** ERROR *** One or more boards did not initialize within 10sec. Cannot start run")
print "*** ERROR *** One or more boards did not initialize within 10sec. Cannot start run"
if (self.run.run_number): self.db.set_run_status(self.run.run_number,5) # Status 5: run with problems at initialization
self.send_answer("init_timeout")
return "error"
time.sleep(1)
elif (all_boards_ready):
#self.write_log("All boards completed initialization: DAQ run can be started")
print "All boards completed initialization: DAQ run can be started"
if (self.run.run_number): self.db.set_run_status(self.run.run_number,1) # Status 1: run correctly initialized
self.send_answer("init_ready")
return "initialized"
else:
#self.write_log("*** ERROR *** One or more boards failed the initialization. Cannot start run")
print "*** ERROR *** One or more boards failed the initialization. Cannot start run"
if (self.run.run_number): self.db.set_run_status(self.run.run_number,5) # Status 5: run with problems at initialization
self.send_answer("init_fail")
return "initfail"
def start_run(self):
#self.write_log("Starting run")
print "Starting run"
if (self.run.run_number):
self.db.set_run_time_start(self.run.run_number,self.now_str())
self.db.set_run_status(self.run.run_number,2) # Status 2: run started
# Create "start the run" tag file
open(self.run.start_file,'w').close()
self.send_answer("run_started")
# RunControl is now in "running" mode
return "running"
def stop_run(self):
self.send_answer("run_comment_end")
ans = self.get_command()
if (ans=="client_close"): return "client_close"
#self.write_log("End of Run comment: "+ans)
print "End of Run comment: %s"%ans
self.run.run_comment_end = ans
#self.write_log("Stopping run")
print "Stopping run"
if (self.run.run_number): self.db.set_run_status(self.run.run_number,3) # Status 3: run stopped normally
return self.terminate_run()
def abort_run(self):
self.run.run_comment_end = "Run aborted"
#self.write_log("Aborting run")
print "Aborting run"
if (self.run.run_number): self.db.set_run_status(self.run.run_number,4) # Status 4: run aborted
return self.terminate_run()
def terminate_run(self):
if (self.run.run_number):
self.db.set_run_time_stop(self.run.run_number,self.now_str())
self.db.set_run_comment_end(self.run.run_number,self.run.run_end_comment)
# Create "stop the run" tag file
open(self.run.quit_file,'w').close()
# Run stop_daq procedure for each ADC board
terminate_ok = True
for adc in (self.run.adcboard_list):
if adc.stop_daq():
self.send_answer("adc %d terminate_ok"%adc.board_id)
#self.write_log("ADC board %02d - Terminated correctly"%adc.board_id)
print "ADC board %02d - Terminated correctly"%adc.board_id
else:
terminate_ok = False
self.send_answer("adc %d terminate_error"%adc.board_id)
#self.write_log("ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id)
print "ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id
if (self.run.run_number): self.db.set_run_status(self.run.run_number,6) # Status 6: run ended with errors
# Clean up run directory
for adc in (self.run.adcboard_list):
if (os.path.exists(adc.initok_file)): os.remove(adc.initok_file)
if (os.path.exists(adc.initfail_file)): os.remove(adc.initfail_file)
if(os.path.exists(self.run.start_file)): os.remove(self.run.start_file)
if(os.path.exists(self.run.quit_file)): os.remove(self.run.quit_file)
if terminate_ok:
self.send_answer("terminate_ok")
else:
self.send_answer("terminate_error")
# At the end of this procedure RunControl is back to "idle" mode
return "idle"
def now_str(self): return time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
class RunControlServer:
def __init__(self):
# Define names of lock and last_used_setup files
self.lock_file = "run/lock"
self.lus_file = "setup/last_used_setup"
# Redefine print to send output to log file
sys.stdout = Logger()
# Create lock file
if (self.create_lock_file() == "error"): exit(1)
# Define what setup to use at startup (can be changed interactively)
initial_setup = self.get_initial_setup()
print "=== Starting PADME Run Control server with %s setup" % initial_setup
#self.write_log("=== Starting PADME Run Control server with %s setup"%initial_setup)
# Create run
self.run = Run()
if (self.run.change_setup(initial_setup) == "error"):
print "ERROR - Error while changing run setup to %s" % initial_setup
#self.write_log("ERROR - Error while changing run setup to %s"%initial_setup)
if os.path.exists(self.lock_file): os.remove(self.lock_file)
exit(1)
# Start in idle state
self.current_state = "idle"
# Create handler for PadmeDB
self.db = PadmeDB()
# Create useful regular expressions
self.re_get_board_config = re.compile("^get_board_config (\d+)$")
self.re_get_board_log_file = re.compile("^get_board_log_file (\d+)$")
self.re_change_setup = re.compile("^change_setup (\w+)$")
# Create a TCP/IP socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Bind the socket to the port
server_address = ('localhost', 10000)
print "Starting server socket on %s port %s" % server_address
#self.write_log("Starting server socket on %s port %s"%server_address)
try:
self.sock.bind(server_address) # Listen for incoming connections
except:
print "ERROR - Could not bind to socket: %s" % str(
sys.exc_info()[0])
#self.write_log("ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0]))
if os.path.exists(self.lock_file): os.remove(self.lock_file)
exit(1)
self.sock.listen(1)
# Define SIGINT handler
signal.signal(signal.SIGINT, self.sigint_handler)
# Setup main interface
self.main_loop()
# Clean up before exiting
self.sock.close()
def create_lock_file(self):
# Check if lock file exists
if (os.path.exists(self.lock_file)):
if (os.path.isfile(self.lock_file)):
pid = 0
with open(self.lock_file, "r") as lf:
for ll in lf:
pid = ll
print "Lock file %s found for pid %s - checking status" % (
self.lock_file, pid)
#self.write_log("ERROR - Lock file %s found for pid %s"%(self.lock_file,pid))
ppinfo = os.popen("ps -p %s" % pid)
pinfo = ppinfo.readlines()
ppinfo.close()
if len(pinfo) == 2:
if pinfo[1].find("<defunct>") > -1:
print "There is zombie process with this pid. The RunControlServer is probably dead. Proceeding cautiously..."
else:
print "ERROR - there is already a RunControlServer running with pid %s" % pid
return "error"
else:
print "No RunControlServer process found. As you were..."
else:
print "ERROR - Lock file %s found but it is not a file" % self.lock_file
#self.write_log("ERROR - Lock file %s found but it is not a file"%self.lock_file)
return "error"
# Create our own lock file
pid = os.getpid()
with open(self.lock_file, "w") as lf:
lf.write("%d" % pid)
return "ok"
def get_initial_setup(self):
setup = "test"
lus = ""
if (os.path.exists(self.lus_file)):
if (os.path.isfile(self.lus_file)):
lusf = open(self.lus_file, "r")
lus = lusf.read().strip("\n")
lusf.close()
if (lus == ""):
print "WARNING - File with last used setup %s exists but it is empty - Using default setup %s" % (
self.lus_file, setup)
#self.write_log("WARNING - File with last used setup %s exists but it is empty - Using default setup %s"%(self.lus_file,setup))
else:
setup = lus
else:
print "WARNING - File with last used setup %s exists but it is not a file - Using default setup %s" % (
self.lus_file, setup)
#self.write_log("WARNING - File with last used setup %s exists but it is not a file - Using default setup %s"%(self.lus_file,setup))
else:
print "WARNING - Could not find file with last used setup %s - Using default setup %s" % (
self.lus_file, setup)
#self.write_log("WARNING - Could not find file with last used setup %s - Using default setup %s"%(self.lus_file,setup))
return setup
def sigint_handler(self, signal, frame):
print "RunControlSever received SIGINT: exiting"
# If a run is initialized/running, abort it as cleanly as possible
if (self.current_state == "initialized"
or self.current_state == "running"):
self.run.run_comment_end = "Run aborted because of SIGINT"
print "Aborting run on SIGINT"
#self.write_log("Aborting run on SIGINT")
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,
4) # Status 4: run aborted
self.db.set_run_time_stop(self.run.run_number, self.now_str())
self.db.set_run_comment_end(self.run.run_number,
self.run.run_comment_end)
open(self.run.quit_file, 'w').close()
for adc in (self.run.adcboard_list):
if adc.stop_daq():
print "ADC board %02d - Terminated correctly" % adc.board_id
#self.write_log("ADC board %02d - Terminated correctly"%adc.board_id)
else:
print "ADC board %02d - WARNING: problems while terminating DAQ" % adc.board_id
#self.write_log("ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id)
if (self.run.run_number):
self.db.set_run_status(
self.run.run_number,
6) # Status 6: run ended with errors
# Clean up run directory
for adc in (self.run.adcboard_list):
if (os.path.exists(adc.initok_file)):
os.remove(adc.initok_file)
if (os.path.exists(adc.initfail_file)):
os.remove(adc.initfail_file)
if (os.path.exists(self.run.start_file)):
os.remove(self.run.start_file)
if (os.path.exists(self.run.quit_file)):
os.remove(self.run.quit_file)
if os.path.exists(self.lock_file): os.remove(self.lock_file)
# Now we can exit
exit(0)
def main_loop(self):
while True:
# Wait for a connection
print "Waiting for a connection"
#self.write_log('waiting for a connection')
(self.connection, client_address) = self.sock.accept()
print "Connection from %s" % str(client_address)
#self.write_log('connection from '+str(client_address))
while True:
# Handle connection according to curren status of RunControl
if self.current_state == "idle":
new_state = self.state_idle()
elif self.current_state == "initialized":
new_state = self.state_initialized()
elif self.current_state == "running":
new_state = self.state_running()
elif self.current_state == "initfail":
new_state = self.state_initfail()
else:
print "ERROR: unknown state %s - ABORTING" % self.current_state
#self.write_log("ERROR: unknown state %s - ABORTING"%self.current_state)
new_state = "exit"
# See if status changed
if new_state == "idle" or new_state == "initialized" or new_state == "running" or new_state == "initfail":
self.current_state = new_state
elif new_state == "client_close":
self.connection.close()
break
elif new_state == "exit":
print "=== RunControlSever received exit command: exiting"
self.connection.close()
if os.path.exists(self.lock_file):
os.remove(self.lock_file)
exit(0)
else:
print "=== RunControlServer = ERROR: unknown new state %s - ABORTING" % new_state
#self.write_log("ERROR: unknown new state %s - ABORTING"%new_state)
self.connection.close()
if os.path.exists(self.lock_file):
os.remove(self.lock_file)
exit(1)
def write_log(self, msg):
print self.now_str() + " " + msg
def now_str(self):
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
def state_idle(self):
# Receive and process commands for "idle" state
while True:
cmd = self.get_command()
print "Received command %s" % cmd
#self.write_log('Received command '+cmd)
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_setup_list"):
self.send_answer(self.get_setup_list())
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_run_number"):
self.send_answer(str(self.db.get_last_run_in_db()))
elif (cmd == "new_run"):
res = self.new_run()
if (res == "client_close"):
return "client_close"
elif (res == "error"):
#self.write_log('ERROR while initializing new run')
print "ERROR while initializing new run"
elif (res == "initialized"):
return "initialized"
elif (res == "initfail"):
return "initfail"
else:
#self.write_log("ERROR: new_run returned unknown answer "+res+" (?)")
print "ERROR: new_run returned unknown answer %s (?)" % res
elif (cmd == "exit"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\tShow this help
get_state\tShow current state of RunControl
get_setup\tShow current setup name
get_setup_list\tShow list of available setups
get_board_list\tShow list of boards in use with current setup
get_board_config <b>\tShow current configuration of board <b>
get_run_number\tReturn last run number in DB
change_setup <setup>\tChange run setup to <setup>
new_run\t\tInitialize system for a new run
exit\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config.match(cmd)
if (m):
self.send_answer(self.get_board_config(int(m.group(1))))
found_re = True
m = self.re_change_setup.match(cmd)
if (m):
self.send_answer(self.change_setup(m.group(1)))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown" % cmd
def state_initialized(self):
# Receive and process commands for "initialized" state
while True:
cmd = self.get_command()
#self.write_log('received command '+cmd)
print "Received command %s" % cmd
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_run_number"):
self.send_answer(str(self.run.run_number))
elif (cmd == "abort_run"):
return self.abort_run()
elif (cmd == "start_run"):
return self.start_run()
elif (cmd == "exit"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\tShow this help
get_state\tShow current state of RunControl
get_setup\tShow current setup name
get_board_list\tShow list of boards in use with current setup
get_board_config <b>\tShow current configuration of board <b>
get_board_log_file <b>\tGet name of log file for board <b>
get_run_number\tReturn current run number
start_run\t\tStart run
abort_run\t\tAbort run
exit\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config.match(cmd)
if (m):
self.send_answer(self.get_board_config(int(m.group(1))))
found_re = True
m = self.re_get_board_log_file.match(cmd)
if (m):
self.send_answer(self.get_board_log_file(int(m.group(1))))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown" % cmd
def state_running(self):
# Receive and process commands for "running" state
while True:
cmd = self.get_command()
#self.write_log('received command '+cmd)
print "Received command %s" % cmd
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_run_number"):
self.send_answer(str(self.run.run_number))
elif (cmd == "stop_run"):
return self.stop_run()
elif (cmd == "exit"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\tShow this help
get_state\tShow current state of RunControl
get_setup\tShow current setup name
get_board_list\tShow list of boards in use with current setup
get_board_config <b>\tShow current configuration of board <b>
get_board_log_file <b>\tGet name of log file for board <b>
get_run_number\tReturn current run number
stop_run\t\tStop the run
exit\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config.match(cmd)
if (m):
self.send_answer(self.get_board_config(int(m.group(1))))
found_re = True
m = self.re_get_board_log_file.match(cmd)
if (m):
self.send_answer(self.get_board_log_file(int(m.group(1))))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown" % cmd
return "idle"
def state_initfail(self):
return "idle"
def get_command(self):
# First get length of string
l = ""
for i in range(5): # Max 99999 characters
ch = self.connection.recv(1)
if ch:
l += ch
else:
#self.write_log('no more data from client')
print "Client closed connection"
return "client_close"
ll = int(l)
# Then read the right amount of characters from the socket
cmd = ""
for i in range(ll):
ch = self.connection.recv(1)
if ch:
cmd += ch
else:
#self.write_log('no more data from client')
print "Client closed connection"
return "client_close"
return cmd
def send_answer(self, answer):
if len(answer) < 100000:
#self.write_log("Sending answer "+answer)
print "Sending answer %s" % answer
self.connection.sendall("%5.5d" % len(answer) + answer)
else:
#self.write_log('answer too long: cannot send')
print "Answer is too long: cannot send"
def get_board_config(self, brdid):
if brdid in self.run.boardid_list:
return self.run.adcboard_list[self.run.boardid_list.index(
brdid)].format_config()
else:
return "ERROR: board id %d does not exist" % brdid
def get_board_log_file(self, brdid):
if brdid in self.run.boardid_list:
return self.run.adcboard_list[self.run.boardid_list.index(
brdid)].log_file
else:
return "ERROR: board id %d does not exist" % brdid
def read_setup_list(self):
# Get list of available setups
setup_main_dir = "setup"
setups = []
for top, dirs, files in os.walk(setup_main_dir):
if (top == setup_main_dir):
for setup_dir in dirs:
setups.append(setup_dir)
setups.sort()
return setups
def get_setup_list(self):
return str(self.read_setup_list())
def change_setup(self, setup):
# Check if requested setup is known
if not (setup in self.read_setup_list()):
#self.write_log("change_setup - ERROR: request to set unknown setup "+setup)
print "change_setup - ERROR: request to set unknown setup %s" % setup
return "error"
# Change (or reload) setup
if (setup == self.run.setup):
#self.write_log("change_setup - reloading setup "+setup)
print "change_setup - reloading setup %s" % setup
else:
#self.write_log("change_setup - changing setup from "+self.run.setup+" to "+setup)
print "change_setup - changing setup from %s to %s" % (
self.run.setup, setup)
self.run.change_setup(setup)
return setup
def new_run(self):
# Retrieve run number - next=next run from DB, dummy=dummy run (i.e. run nr=0)
# Return run number used (0 for dummy run) or "error" for invalid answer
newrun_number = 0
self.send_answer("run_number")
ans = self.get_command()
if (ans == "next"):
newrun_number = self.db.get_last_run_in_db() + 1
elif (ans == "dummy"):
newrun_number = 0
elif (ans == "error"):
#self.write_log("run_number - client returned error")
print "run_number - client returned error"
return "error"
elif (ans == "client_close"):
return "client_close"
else:
#self.write_log("run_number - invalid option %s received"%ans)
print "run_number - invalid option %s received" % ans
self.send_answer("error")
return "error"
self.send_answer(str(newrun_number))
# Retrieve run type (TEST,DAQ,COSMIC)
# Return run type used or "error" for invalid answer
newrun_type = ""
self.send_answer("run_type")
ans = self.get_command()
if (ans == "TEST" or ans == "DAQ" or ans == "COSMIC"):
newrun_type = ans
elif (ans == "error"):
self.write_log("run_type - client returned error")
return "error"
elif (ans == "client_close"):
return "client_close"
else:
# self.write_log("run_type - invalid option %s received"%ans)
print "run_type - invalid option %s received" % ans
self.send_answer("error")
return "error"
self.send_answer(newrun_type)
newrun_user = ""
self.send_answer("shift_crew")
ans = self.get_command()
if (ans == "client_close"): return "client_close"
newrun_user = ans
newrun_comment = ""
self.send_answer("run_comment")
ans = self.get_command()
if (ans == "client_close"): return "client_close"
newrun_comment = ans
#self.write_log("Run number: "+str(newrun_number))
#self.write_log("Run type: "+newrun_type)
#self.write_log("Run crew: "+newrun_user)
#self.write_log("Run comment: "+newrun_comment)
print "Run number: %d" % newrun_number
print "Run type: %s" % newrun_type
print "Run crew: %s" % newrun_user
print "Run comment: %s" % newrun_comment
# Set run configuration according to user's request
self.run.change_run(newrun_number)
self.run.run_type = newrun_type
self.run.run_user = newrun_user
self.run.run_comment = newrun_comment
# Check if requested run number was not used before
# Saves the day if more than one RunControl program is running at the same time (DON'T DO THAT!!!)
if (self.run.run_number):
run_is_in_db = self.db.is_run_in_db(self.run.run_number)
if (run_is_in_db):
#self.write_log("ERROR - Run "+str(self.run.run_number)+" is already in the DB: cannot use it again")
#self.write_log("Please check if someone else is using this RunControl before retrying")
print "ERROR - Run %d is already in the DB: cannot use it again" % self.run.run_number
print "Please check if someone else is using this RunControl before retrying"
self.send_answer("error_init")
return "error"
# Create run structure in the DB
#self.write_log("Initializing Run "+str(self.run.run_number))
print "Initializing Run %d" % self.run.run_number
self.run.create_run()
if (self.run.run_number):
self.db.set_run_time_init(self.run.run_number, self.now_str())
# Create directory to host log files
#self.write_log("Creating log directory "+self.run.log_dir)
print "Creating log directory %s" % self.run.log_dir
self.run.create_log_dir()
# Write run and boards configuration files
#self.write_log("Writing configuration file "+self.run.config_file)
print "Writing configuration file %s" % self.run.config_file
self.run.write_config()
for adc in (self.run.adcboard_list):
#self.write_log("Writing configuration file "+adc.config_file)
print "Writing configuration file %s" % adc.config_file
adc.write_config()
# Start DAQ for all boards
self.send_answer("start_init")
for adc in (self.run.adcboard_list):
p_id = adc.start_daq()
if p_id:
#self.write_log("ADC board %02d - Started DAQ with process id %d"%(adc.board_id,p_id))
print "ADC board %02d - Started DAQ with process id %d" % (
adc.board_id, p_id)
self.send_answer("adc " + str(adc.board_id) + " init")
adc.status = "init"
else:
#self.write_log("ADC board %02d - ERROR: could not start DAQ"%adc.board_id)
print "ADC board %02d - ERROR: could not start DAQ" % adc.board_id
self.send_answer("adc " + str(adc.board_id) + " fail")
adc.status = "fail"
# Wait for all boards to finish initialization
n_try = 0
while (1):
all_boards_init = 1
all_boards_ready = 1
for adc in (self.run.adcboard_list):
# Check if any board changed status
if (adc.status == "init"):
if (os.path.exists(adc.initok_file)):
# Initialization ended OK
#self.write_log("ADC board %02d - Initialized and ready for DAQ"%adc.board_id)
print "ADC board %02d - Initialized and ready for DAQ" % adc.board_id
self.send_answer("adc " + str(adc.board_id) + " ready")
adc.status = "ready"
elif (os.path.exists(adc.initfail_file)):
# Problem during initialization
#self.write_log("ADC board %02d - *** Initialization failed ***"%adc.board_id)
print "ADC board %02d - *** Initialization failed ***" % adc.board_id
self.send_answer("adc " + str(adc.board_id) + " fail")
adc.status = "fail"
else:
# This board is still initializing
all_boards_init = 0
# Check if any board is in fail status
if (adc.status == "fail"): all_boards_ready = 0
if (all_boards_init == 0):
# Some boards are still initializing: keep waiting
n_try += 1
if (n_try >= 10):
#self.write_log("*** ERROR *** One or more boards did not initialize within 10sec. Cannot start run")
print "*** ERROR *** One or more boards did not initialize within 10sec. Cannot start run"
if (self.run.run_number):
self.db.set_run_status(
self.run.run_number,
5) # Status 5: run with problems at initialization
self.send_answer("init_timeout")
return "error"
time.sleep(1)
elif (all_boards_ready):
#self.write_log("All boards completed initialization: DAQ run can be started")
print "All boards completed initialization: DAQ run can be started"
if (self.run.run_number):
self.db.set_run_status(
self.run.run_number,
1) # Status 1: run correctly initialized
self.send_answer("init_ready")
return "initialized"
else:
#self.write_log("*** ERROR *** One or more boards failed the initialization. Cannot start run")
print "*** ERROR *** One or more boards failed the initialization. Cannot start run"
if (self.run.run_number):
self.db.set_run_status(
self.run.run_number,
5) # Status 5: run with problems at initialization
self.send_answer("init_fail")
return "initfail"
def start_run(self):
#self.write_log("Starting run")
print "Starting run"
if (self.run.run_number):
self.db.set_run_time_start(self.run.run_number, self.now_str())
self.db.set_run_status(self.run.run_number,
2) # Status 2: run started
# Create "start the run" tag file
open(self.run.start_file, 'w').close()
self.send_answer("run_started")
# RunControl is now in "running" mode
return "running"
def stop_run(self):
self.send_answer("run_comment_end")
ans = self.get_command()
if (ans == "client_close"): return "client_close"
#self.write_log("End of Run comment: "+ans)
print "End of Run comment: %s" % ans
self.run.run_comment_end = ans
#self.write_log("Stopping run")
print "Stopping run"
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,
3) # Status 3: run stopped normally
return self.terminate_run()
def abort_run(self):
self.run.run_comment_end = "Run aborted"
#self.write_log("Aborting run")
print "Aborting run"
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,
4) # Status 4: run aborted
return self.terminate_run()
def terminate_run(self):
if (self.run.run_number):
self.db.set_run_time_stop(self.run.run_number, self.now_str())
self.db.set_run_comment_end(self.run.run_number,
self.run.run_comment_end)
# Create "stop the run" tag file
open(self.run.quit_file, 'w').close()
# Run stop_daq procedure for each ADC board
terminate_ok = True
for adc in (self.run.adcboard_list):
if adc.stop_daq():
self.send_answer("adc %d terminate_ok" % adc.board_id)
#self.write_log("ADC board %02d - Terminated correctly"%adc.board_id)
print "ADC board %02d - Terminated correctly" % adc.board_id
else:
terminate_ok = False
self.send_answer("adc %d terminate_error" % adc.board_id)
#self.write_log("ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id)
print "ADC board %02d - WARNING: problems while terminating DAQ" % adc.board_id
if (self.run.run_number):
self.db.set_run_status(
self.run.run_number,
6) # Status 6: run ended with errors
# Clean up run directory
for adc in (self.run.adcboard_list):
if (os.path.exists(adc.initok_file)): os.remove(adc.initok_file)
if (os.path.exists(adc.initfail_file)):
os.remove(adc.initfail_file)
if (os.path.exists(self.run.start_file)):
os.remove(self.run.start_file)
if (os.path.exists(self.run.quit_file)): os.remove(self.run.quit_file)
if terminate_ok:
self.send_answer("terminate_ok")
else:
self.send_answer("terminate_error")
# At the end of this procedure RunControl is back to "idle" mode
return "idle"
def now_str(self):
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target
X /= X.std(0)
eps = 5e-3
print "Computing regularization path using the elastic net..."
models = enet_path(X, y, eps=eps, rho=0.8)
alphas_enet = np.array([model.alpha for model in models])
coefs_enet = np.array([model.coef_ for model in models])
glmnet = importr("glmnet")
predict = robjects.r["predict"]
plot = robjects.r["plot"]
run = Run()
run._setup()
dataset = run.ds_train
month_ind = 1
monthly_sales = dataset.getSalesForMonth(month_ind)
x = dataset.getFeaturesForMonth(month_ind)
x /= x.std(0)
eps = 5e-3
print "Computing regularization path using the elastic net..."
models = enet_path(x, monthly_sales, eps=eps, rho=0.8)
mx = robjects.r.matrix(x,ncol=dataset.getNumFeatures())
y = robjects.vectors.FloatVector(monthly_sales)
fit1 = glmnet.glmnet(mx,y)
def Ondestroy(self):
Run.StopTest()
self.windows.destroy()
def main(args):
measures = ['minPRR', 'meanPRR', 'maxDelay', 'meanDelay']
args.parameter = ['configname'] + args.parameter
allhosts = set()
aggregated = {}
runs = []
repetitions = set()
for f in args.input:
r = Run()
r.load(f)
allhosts.update(r.hosts.keys())
runs.append(r)
repetitions.add(r.param["seedset"])
for m in measures:
sub = aggregated
for param in args.parameter:
sub = sub.setdefault(r.param[param], {})
sub.setdefault(m, {})[r.param["seedset"]] = r.measure[m]
print(aggregated)
with open(args.output + '/aggregated.csv', 'w') as resultfile:
cols = args.parameter[:]
for m in measures:
cols += [m, m + "_error"]
writer = csv.DictWriter(resultfile, fieldnames=cols)
headers = dict((n, n) for n in cols)
writer.writerow(headers)
#for config,v in aggregated.iteritems():
stack = list(([k], v) for k, v in aggregated.items())
#for param in sorted(v.keys()):
while stack:
keys, v = stack.pop()
print(v.keys())
if not list(v)[0] in measures: # not yet deep enough
stack.extend(list((keys + [k], v) for k, v in v.items()))
else:
result = dict(zip(args.parameter, keys))
for k in measures:
(m, h) = mean_confidence_interval(list(v[k].values()))
if np.isnan(m): # TODO remove
m = 0
if np.isnan(h): # TODO remove
h = 0
#h = 0 # TODO remove
result[k] = m
result[k + "_error"] = h
writer.writerow(result)
print(result)
with open(args.output + '/per_host.csv', 'w') as resultfile:
sets = set()
perhost = {}
for r in runs:
for host in allhosts:
setname = "-".join(
str(r.param[p]) for p in args.parameter + ['seedset'])
sets.add(setname + "-received")
sets.add(setname + "-lost")
if host in r.hosts:
received = r.hosts[host]['sinkRcvdPk:count']
lost = r.hosts[host]['sentPk:count'] - r.hosts[host][
'sinkRcvdPk:count']
else:
received = 0
lost = 100
perhost.setdefault(host, {})[setname + "-received"] = received
perhost[host][setname + "-lost"] = lost
cols = ['address']
cols += sorted(sets)
#print cols
writer = csv.DictWriter(resultfile, fieldnames=cols)
headers = dict((n, n) for n in cols)
writer.writerow(headers)
for host, v in perhost.items():
data = {'address': host + 1}
data.update(v)
writer.writerow(data)
if distance < 150:
return True
else:
return False
running = True
while running:
screen.fill((200, 100, 0))
for event in pygame.event.get():
if event.type == pygame.QUIT:
pygame.quit()
Run()
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_LEFT:
playerX_change = -5
if event.key == pygame.K_RIGHT:
playerX_change = 5
if event.key == pygame.K_SPACE:
if bullet_state == "ready":
bulletX = playerX
fire_bullet(bulletX, bulletY)
if event.type == pygame.KEYUP:
if event.key == pygame.K_LEFT or event.key == pygame.K_RIGHT:
playerX_change = 0
consumers = 1000
restaurants = 50
count = 0
'''sup_title = 'Varying transaction number, average transaction distribution after '\
+ str(timesteps) + ' timesteps out of ' + str(runs) + ' runs'
'''
"""goal: np array with the average distribution of transactions in the last
timestep for each transaction share step"""
trans_dis = np.zeros([platforms, steps + 1])
for ts in trans_sh:
print('count', count, 'ts', ts)
r = Run(runs, timesteps, platforms, consumers, ts, restaurants)
r.run()
for pf in range(platforms):
trans_dis[-(pf+1)][count] = r.trans_av[pf][timesteps-1]
count += 1
plot_hist = trans_dis[:,:]
fig = plt.figure()
plt.stackplot(trans_sh * consumers, plot_hist)
#fig.suptitle(sup_title, fontsize = 18)
plt.ylabel('Transactions per platform, stacked', fontsize = 18)
plt.xlabel('Total number of transactions per timestep', fontsize = 18)
plt.xlim(0, 1)
plt.xticks(np.arange(0,1100,100))
plt.plot(trans_sh * consumers, trans_sh * consumers)
def __init__(self,mode):
# Get position of DAQ main directory from PADME_DAQ_DIR environment variable
# Default to current dir if not set
self.daq_dir = os.getenv('PADME_DAQ_DIR',".")
# Get port to use for RC connection from PADME_RC_PORT or use port 10000 as default
self.runcontrol_port = int(os.getenv('PADME_RC_PORT',"10000"))
# Define id file for passwordless ssh command execution
self.ssh_id_file = "%s/.ssh/id_rsa_daq"%os.getenv('HOME',"~")
# Define names of lock and last_used_setup files
self.lock_file = self.daq_dir+"/run/lock"
self.lus_file = self.daq_dir+"/setup/last_used_setup"
# Redefine print to send output to log file
sys.stdout = Logger()
sys.stderr = sys.stdout
if mode == "i": sys.stdout.interactive = True
# Create lock file
if (self.create_lock_file() == "error"): exit(1)
# Define what setup to use at startup (can be changed interactively)
initial_setup = self.get_initial_setup()
print "=== Starting PADME Run Control server with %s setup"%initial_setup
# Create run
self.run = Run()
if (self.run.change_setup(initial_setup) == "error"):
print "ERROR - Error while changing run setup to %s"%initial_setup
if os.path.exists(self.lock_file): os.remove(self.lock_file)
exit(1)
# Start in idle state
self.current_state = "idle"
# Create handler for PadmeDB
self.db = PadmeDB()
# Get list of possible run types from DB
self.run_type_list = self.db.get_run_types()
if self.run_type_list == []:
print "WARNING - No run types found in DB. Using default."
self.run_type_list = ['TEST','FAKE']
print "--- Known run types ---"
print self.run_type_list
# Create useful regular expressions to parse user commands
self.re_get_board_config_daq = re.compile("^get_board_config_daq (\d+)$")
self.re_get_board_config_zsup = re.compile("^get_board_config_zsup (\d+)$")
self.re_get_board_log_file_daq = re.compile("^get_board_log_file_daq (\d+)$")
self.re_get_board_log_file_zsup = re.compile("^get_board_log_file_zsup (\d+)$")
self.re_change_setup = re.compile("^change_setup (\w+)$")
# Create a TCP/IP socket
if not self.create_socket():
print "=== RunControlServer - ERROR while creating socket. Exiting"
self.final_cleanup()
exit(1)
self.sock.listen(1)
# Define SIGINT handler
signal.signal(signal.SIGINT,self.sigint_handler)
# Setup main interface
ret_main = self.main_loop()
# Reset SIGINT handler to default
signal.signal(signal.SIGINT,signal.SIG_DFL)
# Handle server exit procedure
ret_exit = self.server_exit()
# Clean up before exiting
self.final_cleanup()
if ( ret_main == "error" or ret_exit == "terminate_error" ): exit(1)
exit(0)
def oneTest(cfg, which):
"""
run a single simulation (call-back from the GUI)
cfg -- configuration values to use
which -- type of simulation to be run
"""
# everybody needs a disk simulation
disk = Disk(size=cfg.disk_size,
fits=cfg.disk_fit, fits2=cfg.disk_fit2,
nre=cfg.disk_nre,
desc="Disk: %s" % (cfg.disk_type))
if which == "disk":
Run([disk], period=cfg.period, verbosity=cfg.verbose)
return
if which == "raid":
if cfg.raid_type == "RAID-0":
raid = RAID0(disk, volumes=cfg.raid_vols,
nre_model=cfg.nre_model,
recovery=cfg.raid_recover,
delay=cfg.raid_replace,
objsize=cfg.obj_size)
elif cfg.raid_type == "RAID-1":
raid = RAID1(disk, volumes=cfg.raid_vols,
nre_model=cfg.nre_model,
recovery=cfg.raid_recover,
delay=cfg.raid_replace,
objsize=cfg.obj_size)
elif cfg.raid_type == "RAID-5":
raid = RAID5(disk, volumes=cfg.raid_vols,
nre_model=cfg.nre_model,
recovery=cfg.raid_recover,
delay=cfg.raid_replace,
objsize=cfg.obj_size)
elif cfg.raid_type == "RAID-6":
raid = RAID6(disk, volumes=cfg.raid_vols,
nre_model=cfg.nre_model,
recovery=cfg.raid_recover,
delay=cfg.raid_replace,
objsize=cfg.obj_size)
Run([raid], period=cfg.period, verbosity=cfg.verbose)
return
rados = RADOS(disk, pg=cfg.rados_decluster,
copies=cfg.rados_copies,
speed=cfg.rados_recover,
fullness=cfg.rados_fullness,
objsize=cfg.obj_size,
stripe=cfg.stripe_length,
nre_model=cfg.nre_model,
delay=cfg.rados_markout)
if which == "rados":
Run([rados], period=cfg.period, verbosity=cfg.verbose)
return
if which == "multi":
site = Site(fits=cfg.majeure, rplc=cfg.site_recover)
multi = MultiSite(rados, site,
speed=cfg.remote_recover,
latency=cfg.remote_latency,
sites=cfg.remote_sites)
Run([multi], period=cfg.period, verbosity=cfg.verbose)
return
class Application(Frame):
def __init__(self, master=None):
Frame.__init__(self, master)
self.pack()
self.v = IntVar()
self.v.set(1)
self.m = 3
self.run = Run(self.m, self.m)
self.run.init_aim()
self.p = self.run.random_state()
self.imagel = []
self.image_list2 = []
self.path = []
self.create_widget()
# 创建布局函数
def create_widget(self):
self.image = Image.open('lemon.jpg')
self.fm1 = Frame(self)
self.fm1.pack(side=LEFT, fill='y')
self.show_pictures()
self.fm2 = Frame(self)
self.fm2.pack(side=RIGHT, fill='y', pady='100')
size = [('3 * 3', 1), ('4 * 4', 2), ('5 * 5', 3)]
for s, num in size:
Radiobutton(self.fm2,
text=s,
value=num,
variable=self.v,
command=self.rb_ckecked).pack()
self.btn_mess = Button(self.fm2,
text='打乱',
width=15,
height=2,
command=self.mess_callback)
self.btn_mess.pack(padx=25, pady=10)
self.btn_start = Button(self.fm2,
text='开始排序',
width=15,
height=2,
command=self.start_callback)
self.btn_start.pack(padx=25, pady=10)
self.text_statement = Label(self.fm2,
text='本程序第一个方格为0，若算法中循环超过3000次仍计算不出，则终止算法',
wraplength=140)
self.text_statement.pack(padx=15, pady=10)
self.str = StringVar()
self.str.set('静止')
self.text_condition = Label(self.fm2, textvariable=self.str)
self.text_condition.pack(padx=25, pady=10)
self.str2 = StringVar()
self.str2.set('步数为：')
self.text_info = Label(self.fm2, textvariable=self.str2)
self.text_info.pack(padx=25, pady=10)
def rb_ckecked(self):
print(self.v.get())
# “打乱”按钮的监听响应函数
def mess_callback(self):
k = self.v.get()
if k == 1:
self.m = 3
hard = 150
elif k == 2:
self.m = 4
hard = 50
else:
self.m = 5
hard = 40
self.run.set_mn(self.m, self.m, hard)
self.run.init_aim()
self.p = self.run.random_state()
for widget in self.fm1.winfo_children():
widget.destroy()
self.show_pictures()
self.str.set('静止')
print(self.m)
# “开始排序”按钮的监听响应函数
def start_callback(self):
print('开始')
# self.text_condition.configure(text='开始')
self.str.set('开始')
th = MyThread(self.p, self.run)
th.setDaemon(True)
th.start()
th.join()
self.path = th.get_result()
if self.path:
self.printp()
self.str.set('静止')
step = th.get_Infor()
self.str2.set('步数为：' + str(step))
for i in self.path:
print(i)
else:
self.str.set('计算不出来')
# 裁剪图片函数
@staticmethod
def cut_image(image, m):
width, height = image.size
item_width = int(width / m)
box_list = []
for i in range(0, m):
for j in range(0, m):
box = (j * item_width, i * item_width, (j + 1) * item_width,
(i + 1) * item_width)
box_list.append(box)
image_list = [image.crop(box) for box in box_list]
return image_list
# 显示图片函数
def show_pictures(self):
image_list = self.cut_image(self.image, self.m)
self.image_list2.clear()
self.imagel.clear()
for im2 in image_list:
self.image_list2.append(ImageTk.PhotoImage(im2))
for i in range(0, self.m):
for j in range(0, self.m):
self.imagel.append(IntVar())
self.imagel[i * self.m + j] = Label(
self.fm1,
width=600 / self.m,
height=600 / self.m,
image=self.image_list2[self.p.state[i * self.m + j]],
borderwidth=3)
self.imagel[i * self.m + j].grid(row=i, column=j)
# 根据计算出的路径实现图片的移动函数
def printp(self):
# pic = ImageGrab.grab((self.winfo_y() + 70, self.winfo_x() + 90, self.winfo_y() + self.winfo_width() + 70,
# 230 + self.winfo_x() + self.winfo_height()))
# pic.save(str(len(self.path)) + '.jpg')
if not self.path:
return
pa = self.path[0]
count = 0
for i in self.imagel:
i.config(imag=self.image_list2[pa[count]])
count += 1
self.path.remove(pa)
self.fm1.after(200, self.printp)
def call_run_obj(self, ml_model, *args):
import numpy as np
slurm_job = args[0]
if slurm_job:
job_id = slurm_job.split("-")[1]
else:
job_id = slurm_job
timehorizon = args[4]
datasetname = args[5]
test_size = args[6]
prior_file = args[7]
prior_type = args[8]
datatype = args[9]
data_type_lo = args[10]
bias_score_splitnodeFeature = args[11]
thres_coeff_var = args[12]
num_ets_lo = args[13]
time_step = args[14]
tuning_genebygene_randomized = args[15]
n_iter_search = args[16]
tfa_bool = args[17]
num_threads = args[18]
save_models = args[19]
flag_print = args[20]
parse_4dyng3 = args[21]
auto_meth = args[22]
rnd_seed = args[23]
if self.input_dir_name:
script_dir = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
path_dataset = script_dir + "/Datasets/" + self.input_dir_name + "/"
if not (os.path.isdir(path_dataset)):
print("Dataset " + path_dataset + " does NOT exist")
exit(1)
else:
datasetname = self.input_dir_name
if self.test_set_split_ratio:
if not (self.is_number(self.test_set_split_ratio)):
print("test_set_split_ratio is NOT a number")
exit(1)
else:
test_size = self.test_set_split_ratio
if self.prior_file_name:
prior_file = self.prior_file_name
tree_method = "RF-mod"
if self.prior_file_name == "":
prior_file = "no"
prior_type = "no"
tree_method = "RF"
if sys.path.__contains__(site.USER_SITE):
print(
"Error ***************************************** Error "
)
print("To run OutPredict WITHOUT priors please run:")
print("python -s pipeline_new_organism.py")
print(
"and do NOT set the params prior_file_name and priors")
print("*****************************************")
print("To run OutPredict WITH priors please run:")
print("python pipeline_new_organism.py")
print(
"After properly setting both the params prior_file_name and priors"
)
exit(1)
else:
if self.priors:
tree_method = "RF-mod"
if self.priors != "gold_standard" and self.priors != "steady_state" and self.priors != "" and self.priors != "no":
print("*****************************************")
print(
"Error: priors param value has to be either gold_standard or steady_state"
)
exit(1)
else:
if self.priors == "gold_standard":
prior_type = "binary_all"
if not (os.path.exists(path_dataset +
self.prior_file_name)):
print(
FileNotFoundError("gold standard file " +
path_dataset +
self.prior_file_name +
" does NOT exist"))
exit(1)
elif self.priors == "steady_state":
prior_type = "real_all"
elif self.priors == "" or self.priors == "no":
prior_type = "no"
prior_file = "no"
tree_method = "RF"
if sys.path.__contains__(site.USER_SITE):
print(
"Error ***************************************** Error "
)
print(
"To run OutPredict WITHOUT priors please run:"
)
print("python -s pipeline_new_organism.py")
print(
"and do NOT set the params prior_file_name and priors"
)
print(
"*****************************************"
)
print(
"To run OutPredict WITH priors please run:"
)
print("python pipeline_new_organism.py")
print(
"After properly setting both the params prior_file_name and priors"
)
exit(1)
else:
if sys.path.__contains__(site.USER_SITE):
print(
"Error **************************************** Error "
)
print("To run OutPredict WITHOUT priors please run:")
print("python -s pipeline_new_organism.py")
print(
"and do NOT set the params prior_file_name and priors"
)
print("*****************************************")
print("To run OutPredict WITH priors please run:")
print("python pipeline_new_organism.py")
print(
"After properly setting both the params prior_file_name and priors"
)
exit(1)
else:
prior_type = "no"
prior_file = "no"
tree_method = "RF"
if sys.path.__contains__(site.USER_SITE):
print("Error ***************************************** Error ")
print("To run OutPredict WITHOUT priors please run:")
print("python -s pipeline_new_organism.py")
print("and do NOT set the params prior_file_name and priors")
print("*****************************************")
print("To run OutPredict WITH priors please run:")
print("python pipeline_new_organism.py")
print(
"After properly setting both the params prior_file_name and priors"
)
exit(1)
if self.training_data_type:
if self.training_data_type != "TS-SS" and self.training_data_type != "TS" and self.training_data_type != "SS":
print("*****************************************")
print(
"Error: training_data_type param value has to be TS-SS, TS, or SS"
)
exit(1)
else:
datatype = self.training_data_type
if self.leave_out_data_type:
if self.leave_out_data_type != "TS-SS" and self.leave_out_data_type != "TS" and self.leave_out_data_type != "SS":
print("*****************************************")
print(
"Error: leave_out_data_type param value has to be TS-SS, TS, or SS"
)
exit(1)
else:
data_type_lo = self.leave_out_data_type
if self.genes_coeff_of_var_threshold:
if not (self.is_number(self.genes_coeff_of_var_threshold)):
print("test_set_split_ratio is NOT a number")
exit(1)
else:
thres_coeff_var = self.genes_coeff_of_var_threshold
if self.time_step_or_ode_log:
if self.time_step_or_ode_log != "time-step" and self.time_step_or_ode_log != "ode-log":
print("*****************************************")
print(
"Error: time_step_or_ode_log param value has to be either time-step, or ode-log"
)
exit(1)
else:
if self.time_step_or_ode_log == "time-step":
time_step = "True"
elif self.time_step_or_ode_log == "ode-log":
time_step = "False"
if self.num_of_cores:
if not (self.is_number(self.num_of_cores)):
print("num_of_cores is NOT a number")
exit(1)
else:
num_threads = self.num_of_cores
name_run = ""
# if ml_model == "RF":
# tree_method = args[1]
k = args[2]
ntrees = args[3]
if self.num_of_trees:
if not (self.is_number(self.num_of_trees)):
print("num_of_trees is NOT a number")
exit(1)
else:
ntrees = self.num_of_trees
# #Check here the conda environment
# conda_env = os.environ['CONDA_DEFAULT_ENV']
# if conda_env == "op3":
# tree_method = "RF"
# # sys.path.insert(0,"//anaconda/envs/op3/lib/python3.7/site-packages")
# elif conda_env == "op3_priors":
# tree_method = "RF-mod"
# else:
# print("The conda environment has not been set up correctly")
# exit(1)
if tree_method == "RF":
print(":::::::RUNNING OUTPREDICT WITHOUT PRIORS:::::::::")
if sys.path.__contains__(site.USER_SITE):
print("Error***************************************** Error ")
print("To run OutPredict WITHOUT priors please run:")
print("python -s pipeline_new_organism.py")
print("and do NOT set the params prior_file_name and priors")
print("*****************************************")
print("To run OutPredict WITH priors please run:")
print("python pipeline_new_organism.py")
print(
"After properly setting both the params prior_file_name and priors"
)
exit(1)
if tree_method == "RF-mod":
print(":::::::RUNNING OUTPREDICT WITH PRIORS:::::::::")
if not (sys.path.__contains__(site.USER_SITE)):
print("Error ***************************************** Error ")
print("To run OutPredict WITH priors please run:")
print("python pipeline_new_organism.py")
print(
"After properly setting both the params prior_file_name and priors"
)
print("*****************************************")
print("To run OutPredict WITHOUT priors please run:")
print("python -s pipeline_new_organism.py")
print("and do NOT set the params prior_file_name and priors")
exit(1)
time_step = self.str_to_bool(time_step)
tuning_genebygene_randomized = self.str_to_bool(
tuning_genebygene_randomized)
tfa_bool = self.str_to_bool(tfa_bool)
flag_print = self.str_to_bool(flag_print)
parse_4dyng3 = self.str_to_bool(parse_4dyng3)
auto_meth = self.str_to_bool(auto_meth)
if not (time_step):
tau_vect = [5, 1, 45]
else:
tau_vect = [0]
if auto_meth:
tau_vect = [0, 5, 1, 45] # tau=0 implies to run time-step
oob_score_avg_vect = np.ones(len(tau_vect))
oob_mse_avg_vect = np.ones(len(tau_vect))
aupr_vect = np.ones(len(tau_vect))
mse_test_future_timepoint_vect = np.ones(len(tau_vect))
mean_corr_test_future_timepoint_vect = np.ones(len(tau_vect))
print_out_vect = [""] * len(tau_vect)
confidences_vect = []
print_only_one_time = True
for run_i, tau in enumerate(tau_vect):
gc.collect()
run_obj = Run()
if auto_meth:
if tau == 0:
time_step = True
else:
time_step = False
if flag_print:
print(
"slurm_job, tree_method, max_feat, k, ntrees, ntrees, timehorizon, datasetname, test_size, prior_file, prior_type, datatype, data_type_lo, bias_score_splitnodeFeature, thres_coeff_var, num_ets_lo, time_step, tuning_genebygene_randomized, n_iter_search, tfa_bool, num_threads, save_models, flag_print, auto_meth"
)
print(slurm_job, tree_method, "max_feat", k, "ntrees", ntrees,
timehorizon, datasetname, test_size, prior_file,
prior_type, datatype, data_type_lo,
bias_score_splitnodeFeature, thres_coeff_var, num_ets_lo,
time_step, tuning_genebygene_randomized, n_iter_search,
tfa_bool, num_threads, save_models, flag_print,
auto_meth, rnd_seed)
output_path, oob_score_avg, oob_mse_avg, aupr, mse_test_future_timepoint, mean_corr_test_future_timepoint, print_out_string_to_ret, confidences, gs, outfile = run_obj.launch_out_predict(
ml_model, tree_method, k, ntrees, tau, timehorizon,
datasetname, test_size, prior_file, prior_type, datatype,
data_type_lo, bias_score_splitnodeFeature, thres_coeff_var,
num_ets_lo, time_step, tuning_genebygene_randomized,
n_iter_search, tfa_bool, num_threads, save_models, name_run,
flag_print, parse_4dyng3, print_only_one_time, auto_meth,
rnd_seed)
if flag_print:
outfile.write(
"slurm_job, tree_method, max_feat, k, ntrees, ntrees, timehorizon, datasetname, test_size, prior_file, prior_type, datatype, data_type_lo, bias_score_splitnodeFeature, thres_coeff_var, num_ets_lo, time_step, tuning_genebygene_randomized, n_iter_search, tfa_bool, num_threads, save_models, flag_print, auto_meth"
)
outfile.write(
str(slurm_job) + str(tree_method) + str("max_feat") +
str(k) + str("ntrees") + str(ntrees) + str(timehorizon) +
str(datasetname) + str(test_size) + str(prior_file) +
str(prior_type) + str(datatype) + str(data_type_lo) +
str(bias_score_splitnodeFeature) + str(thres_coeff_var) +
str(num_ets_lo) + str(time_step) +
str(tuning_genebygene_randomized) + str(n_iter_search) +
str(tfa_bool) + str(num_threads) + str(save_models) +
str(flag_print) + str(auto_meth) + str(rnd_seed))
print("End of modeling with tau value: ", tau,
". The oob_score_avg is: ", oob_score_avg)
print("End of modeling with tau value: ", tau,
". The oob_mse_avg is: ", oob_mse_avg)
oob_score_avg_vect[run_i] = oob_score_avg
oob_mse_avg_vect[run_i] = oob_mse_avg
aupr_vect[run_i] = aupr
mse_test_future_timepoint_vect[run_i] = mse_test_future_timepoint
mean_corr_test_future_timepoint_vect[
run_i] = mean_corr_test_future_timepoint
print_out_vect[run_i] = print_out_string_to_ret
confidences_vect.append(confidences)
if auto_meth:
print_only_one_time = False
if flag_print:
print("\n \n \n \n \n")
for run_i, tau in enumerate(tau_vect):
if flag_print:
print("Modeling for tau value: ", tau)
print("The oob_score_avg is: ", oob_score_avg_vect[run_i])
print("The oob_mse_avg is: ", oob_mse_avg_vect[run_i])
print("The aupr is: ", aupr_vect[run_i])
print("The mse_test_set is: ",
mse_test_future_timepoint_vect[run_i])
print("The mean_corr_test_set is: ",
mean_corr_test_future_timepoint_vect[run_i])
if auto_meth:
index_best = np.where(
oob_mse_avg_vect == np.min(oob_mse_avg_vect))[0][0]
print(
"OutPredict has found the best model according to the out-of-bag score... \n"
)
print(
"The best model is compared to the Penultimate Value Naive approach. \n"
)
confidences = confidences_vect[index_best]
net_inf = NetworkInference(flag_print, rnd_seed)
aupr, random_aupr = net_inf.summarize_results(
output_path, "PR_CurveBestModel", confidences, gs, True)
if flag_print:
print("Area under Precision-Recall based on goldstandard: ",
aupr)
outfile.write(
"Area under Precision-Recall based on goldstandard: " +
str(aupr) + "\n")
outfile.write("Random AUPR: " + str(random_aupr) + "\n")
confidences.to_csv(output_path + "/Matrix_TF_gene_best_model.tsv",
sep="\t")
# Print ranked list of edges
List = [('TF', 'Target', 'Importance')]
for source in confidences.columns.values:
for target in confidences.index.values:
List.append((source, target, confidences[source][target]))
with open(output_path + "/Ranked_list_TF_gene_best_model.csv",
"w") as f:
writer = csv.writer(f)
writer.writerows(List)
print(print_out_vect[index_best])
outfile.write(print_out_vect[index_best])
outfile.write("Run_name: " + str(self.num_of_trees) +
self.input_dir_name +
str(self.test_set_split_ratio) +
self.training_data_type + self.leave_out_data_type +
str(self.genes_coeff_of_var_threshold) +
str(self.num_of_cores))
outfile.close()
'''
# log color test
log.debug("this is a debugging message")
log.info("this is an informational message")
log.warning("this is a warning message")
log.error("this is an error message")
log.critical("this is a critical message")
'''
# 4312 = am put, 0 = test, 4411_2 = Bermuda mit 2 Aktien, 4411_5 = Bermuda mit 5 Aktien
# Hier alles in Anführungszeichen -> R0, R00, R12 = Robbins mit 12 ZV, R13, R20 = Robbins mit 20 ZV, R30, R40, Russ1 = Meistgenutztes russisches Setting, Russ11
# Es gibt 0, W, S als pre und s, l, f als suffixe. 0, W, S geben Varianten vom Robbins Modell an und s, l gibt an ob der Durchlauf schnell oder lang sein soll. f wurde nicht genutzt
log.warning("Start")
# Hier wird der Run aufgerufen. Man kann auch mehrere Runs in Folge ausführen
Run("R00", log) # fix bug that 4411_5 scheinbar merkwürdige val/test paths hat
"""
import os
time.sleep(5)
os.chdir("../current run2")
ConfigInitializer("FR12f", log)
time.sleep(5)
# os.chdir("C:/Users/Olus/Desktop/Masterarbeit Mathe/new computer/current run3")
os.chdir("../current run3")
ConfigInitializer("SR20", log)
time.sleep(5)
os.chdir("../current run4")
ConfigInitializer("FR12f", log)
"""
def __init__(self):
# Define names of lock and last_used_setup files
self.lock_file = "run/lock"
self.lus_file = "setup/last_used_setup"
# Redefine print to send output to log file
sys.stdout = Logger()
# Create lock file
if (self.create_lock_file() == "error"): exit(1)
# Define what setup to use at startup (can be changed interactively)
initial_setup = self.get_initial_setup()
print "=== Starting PADME Run Control server with %s setup" % initial_setup
#self.write_log("=== Starting PADME Run Control server with %s setup"%initial_setup)
# Create run
self.run = Run()
if (self.run.change_setup(initial_setup) == "error"):
print "ERROR - Error while changing run setup to %s" % initial_setup
#self.write_log("ERROR - Error while changing run setup to %s"%initial_setup)
if os.path.exists(self.lock_file): os.remove(self.lock_file)
exit(1)
# Start in idle state
self.current_state = "idle"
# Create handler for PadmeDB
self.db = PadmeDB()
# Create useful regular expressions
self.re_get_board_config = re.compile("^get_board_config (\d+)$")
self.re_get_board_log_file = re.compile("^get_board_log_file (\d+)$")
self.re_change_setup = re.compile("^change_setup (\w+)$")
# Create a TCP/IP socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Bind the socket to the port
server_address = ('localhost', 10000)
print "Starting server socket on %s port %s" % server_address
#self.write_log("Starting server socket on %s port %s"%server_address)
try:
self.sock.bind(server_address) # Listen for incoming connections
except:
print "ERROR - Could not bind to socket: %s" % str(
sys.exc_info()[0])
#self.write_log("ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0]))
if os.path.exists(self.lock_file): os.remove(self.lock_file)
exit(1)
self.sock.listen(1)
# Define SIGINT handler
signal.signal(signal.SIGINT, self.sigint_handler)
# Setup main interface
self.main_loop()
# Clean up before exiting
self.sock.close()
class RunControlServer:
def __init__(self,mode):
# Get position of DAQ main directory from PADME_DAQ_DIR environment variable
# Default to current dir if not set
self.daq_dir = os.getenv('PADME_DAQ_DIR',".")
# Get port to use for RC connection from PADME_RC_PORT or use port 10000 as default
self.runcontrol_port = int(os.getenv('PADME_RC_PORT',"10000"))
# Define id file for passwordless ssh command execution
self.ssh_id_file = "%s/.ssh/id_rsa_daq"%os.getenv('HOME',"~")
# Define names of lock and last_used_setup files
self.lock_file = self.daq_dir+"/run/lock"
self.lus_file = self.daq_dir+"/setup/last_used_setup"
# Redefine print to send output to log file
sys.stdout = Logger()
sys.stderr = sys.stdout
if mode == "i": sys.stdout.interactive = True
# Create lock file
if (self.create_lock_file() == "error"): exit(1)
# Define what setup to use at startup (can be changed interactively)
initial_setup = self.get_initial_setup()
print "=== Starting PADME Run Control server with %s setup"%initial_setup
# Create run
self.run = Run()
if (self.run.change_setup(initial_setup) == "error"):
print "ERROR - Error while changing run setup to %s"%initial_setup
if os.path.exists(self.lock_file): os.remove(self.lock_file)
exit(1)
# Start in idle state
self.current_state = "idle"
# Create handler for PadmeDB
self.db = PadmeDB()
# Get list of possible run types from DB
self.run_type_list = self.db.get_run_types()
if self.run_type_list == []:
print "WARNING - No run types found in DB. Using default."
self.run_type_list = ['TEST','FAKE']
print "--- Known run types ---"
print self.run_type_list
# Create useful regular expressions to parse user commands
self.re_get_board_config_daq = re.compile("^get_board_config_daq (\d+)$")
self.re_get_board_config_zsup = re.compile("^get_board_config_zsup (\d+)$")
self.re_get_board_log_file_daq = re.compile("^get_board_log_file_daq (\d+)$")
self.re_get_board_log_file_zsup = re.compile("^get_board_log_file_zsup (\d+)$")
self.re_change_setup = re.compile("^change_setup (\w+)$")
# Create a TCP/IP socket
if not self.create_socket():
print "=== RunControlServer - ERROR while creating socket. Exiting"
self.final_cleanup()
exit(1)
self.sock.listen(1)
# Define SIGINT handler
signal.signal(signal.SIGINT,self.sigint_handler)
# Setup main interface
ret_main = self.main_loop()
# Reset SIGINT handler to default
signal.signal(signal.SIGINT,signal.SIG_DFL)
# Handle server exit procedure
ret_exit = self.server_exit()
# Clean up before exiting
self.final_cleanup()
if ( ret_main == "error" or ret_exit == "terminate_error" ): exit(1)
exit(0)
def create_socket(self):
# Create a TCP/IP socket
self.sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
# Allow immediate reuse of socket after server restart (i.e. disable socket TIME_WAIT)
self.sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
# Bind the socket to the port
server_address = ('localhost',self.runcontrol_port)
print "Starting server socket on %s port %s"%server_address
try:
self.sock.bind(server_address) # Listen for incoming connections
except:
print "ERROR - Could not bind to socket: %s"%str(sys.exc_info()[0])
return False
return True
def final_cleanup(self):
# Save setup currently in use before exiting
self.save_final_setup(self.run.setup)
# Final clean up procedure before exiting
if self.sock:
self.sock.shutdown(socket.SHUT_RDWR)
self.sock.close()
if os.path.exists(self.lock_file): os.remove(self.lock_file)
def create_lock_file(self):
# Check if lock file exists
if (os.path.exists(self.lock_file)):
if (os.path.isfile(self.lock_file)):
pid = 0
with open(self.lock_file,"r") as lf:
for ll in lf: pid = ll
print "Lock file %s found for pid %s - checking status"%(self.lock_file,pid)
ppinfo = os.popen("ps -p %s"%pid)
pinfo = ppinfo.readlines()
ppinfo.close()
if len(pinfo)==2:
if pinfo[1].find("<defunct>")>-1:
print "There is zombie process with this pid. The RunControlServer is probably dead. Proceeding cautiously..."
else:
print "ERROR - there is already a RunControlServer running with pid %s"%pid
return "error"
else:
print "No RunControlServer process found. As you were..."
else:
print "ERROR - Lock file %s found but it is not a file"%self.lock_file
return "error"
# Create our own lock file
pid = os.getpid()
with open(self.lock_file,"w") as lf: lf.write("%d\n"%pid)
return "ok"
def get_initial_setup(self):
setup = "test"
lus = ""
if (os.path.exists(self.lus_file)):
if (os.path.isfile(self.lus_file)):
lusf = open(self.lus_file,"r")
lus = lusf.read().strip("\n")
lusf.close()
if (lus == ""):
print "WARNING - File with last used setup %s exists but it is empty - Using default setup %s"%(self.lus_file,setup)
else:
setup = lus
else:
print "WARNING - File with last used setup %s exists but it is not a file - Using default setup %s"%(self.lus_file,setup)
else:
print "WARNING - Could not find file with last used setup %s - Using default setup %s"%(self.lus_file,setup)
return setup
def save_final_setup(self,setup):
print "Saving current setup %s to %s\n"%(setup,self.lus_file)
with open(self.lus_file,"w") as lf: lf.write("%s\n"%setup)
def sigint_handler(self,signal,frame):
print "=== RunControlSever received SIGINT: exiting"
# If a run is initialized/running, abort it as cleanly as possible
ret_exit = "terminate_ok"
if ( self.current_state == "initialized" or self.current_state == "running" ):
self.run.final_status = self.db.DB_RUN_STATUS_ABORTED
self.run.run_comment_end = "Run aborted due to RunControlServer SIGINT"
#if (self.run.run_number):
# self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_ABORTED)
ret_exit = self.terminate_run()
# Clean up before exiting
self.final_cleanup()
if ( ret_exit == "terminate_error" ): exit(1)
exit(0)
def server_exit(self):
# Procedure to gracefully handle server shutdown
print "=== RunControlSever is shutting down"
# If a run is initialized/running, abort it as cleanly as possible
if ( self.current_state == "initialized" or self.current_state == "running" ):
self.run.final_status = self.db.DB_RUN_STATUS_ABORTED
self.run.run_comment_end = "Run aborted due to RunControlServer shutdown"
#if (self.run.run_number):
# self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_ABORTED)
return self.terminate_run()
# Otherwise nothing to do
return "ok"
def main_loop(self):
while True:
# Wait for a connection
print "Waiting for a connection"
#self.write_log('waiting for a connection')
(self.connection,client_address) = self.sock.accept()
print "Connection from %s"%str(client_address)
#self.write_log('connection from '+str(client_address))
while True:
# Handle connection according to curren status of RunControl
if self.current_state == "idle":
new_state = self.state_idle()
elif self.current_state == "initialized":
new_state = self.state_initialized()
elif self.current_state == "running":
new_state = self.state_running()
elif self.current_state == "initfail":
new_state = self.state_initfail()
else:
print "ERROR: unknown state %s - ABORTING"%self.current_state
#self.write_log("ERROR: unknown state %s - ABORTING"%self.current_state)
new_state = "exit"
# See if status changed
if new_state == "idle" or new_state == "initialized" or new_state == "running" or new_state == "initfail":
self.current_state = new_state
else:
self.connection.close()
if new_state == "exit":
print "=== RunControlSever received shutdown command: exiting"
return "exit"
elif new_state != "client_close":
print "=== RunControlServer = ERROR: unknown new state %s - ABORTING"%new_state
return "error"
# Exit from client handling loop and wait for a new client
break
def state_idle(self):
# Receive and process commands for "idle" state
while True:
cmd = self.get_command()
#print "Received command %s"%cmd
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_setup_list"):
self.send_answer(self.get_setup_list())
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_trig_config"):
self.send_answer(self.get_trig_config())
elif (cmd == "get_run_number"):
self.send_answer(str(self.db.get_last_run_in_db()))
elif (cmd == "new_run"):
res = self.new_run()
if (res == "client_close"):
return "client_close"
elif (res == "error"):
print "ERROR while initializing new run"
elif (res == "initialized"):
return "initialized"
elif (res == "initfail"):
return "initfail"
else:
print "ERROR: new_run returned unknown answer %s (?)"%res
elif (cmd == "shutdown"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\t\t\tShow this help
get_state\t\t\tShow current state of RunControl
get_setup\t\t\tShow current setup name
get_setup_list\t\t\tShow list of available setups
get_board_list\t\t\tShow list of boards in use with current setup
get_board_config_daq <b>\tShow current configuration of board DAQ process <b>
get_board_config_zsup <b>\tShow current configuration of board ZSUP process <b>
get_trig_config\t\t\tShow current configuration of trigger process
get_run_number\t\t\tReturn last run number in DB
change_setup <setup>\t\tChange run setup to <setup>
new_run\t\t\t\tInitialize system for a new run
shutdown\t\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config_daq.match(cmd)
if (m):
self.send_answer(self.get_board_config_daq(int(m.group(1))))
found_re = True
m = self.re_get_board_config_zsup.match(cmd)
if (m):
self.send_answer(self.get_board_config_zsup(int(m.group(1))))
found_re = True
m = self.re_change_setup.match(cmd)
if (m):
self.send_answer(self.change_setup(m.group(1)))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
print "Command %s is unknown"%cmd
def state_initialized(self):
# Receive and process commands for "initialized" state
while True:
cmd = self.get_command()
#print "Received command %s"%cmd
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_trig_config"):
self.send_answer(self.get_trig_config())
elif (cmd == "get_trig_log"):
self.send_answer(self.get_trig_log())
elif (cmd == "get_run_number"):
self.send_answer(str(self.run.run_number))
elif (cmd == "abort_run"):
return self.abort_run()
elif (cmd == "start_run"):
return self.start_run()
elif (cmd == "shutdown"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\t\t\tShow this help
get_state\t\t\tShow current state of RunControl
get_setup\t\t\tShow current setup name
get_board_list\t\t\tShow list of boards in use with current setup
get_board_config_daq <b>\tShow current configuration of board DAQ process<b>
get_board_config_zsup <b>\tShow current configuration of board ZSUP process<b>
get_board_log_file_daq <b>\tGet name of log file for board DAQ process<b>
get_board_log_file_zsup <b>\tGet name of log file for board ZSUP process<b>
get_trig_config\t\t\tShow current configuration of trigger process
get_trig_log\t\t\tGet name of log file for trigger process
get_run_number\t\t\tReturn current run number
start_run\t\t\t\tStart run
abort_run\t\t\t\tAbort run
shutdown\t\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config_daq.match(cmd)
if (m):
self.send_answer(self.get_board_config_daq(int(m.group(1))))
found_re = True
m = self.re_get_board_config_zsup.match(cmd)
if (m):
self.send_answer(self.get_board_config_zsup(int(m.group(1))))
found_re = True
m = self.re_get_board_log_file_daq.match(cmd)
if (m):
self.send_answer(self.get_board_log_file_daq(int(m.group(1))))
found_re = True
m = self.re_get_board_log_file_zsup.match(cmd)
if (m):
self.send_answer(self.get_board_log_file_zsup(int(m.group(1))))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown"%cmd
def state_running(self):
# Receive and process commands for "running" state
while True:
cmd = self.get_command()
#print "Received command %s"%cmd
if (cmd == "client_close"):
return "client_close"
elif (cmd == "get_state"):
self.send_answer(self.current_state)
elif (cmd == "get_setup"):
self.send_answer(self.run.setup)
elif (cmd == "get_board_list"):
self.send_answer(str(self.run.boardid_list))
elif (cmd == "get_trig_config"):
self.send_answer(self.get_trig_config())
elif (cmd == "get_trig_log"):
self.send_answer(self.get_trig_log())
elif (cmd == "get_run_number"):
self.send_answer(str(self.run.run_number))
elif (cmd == "stop_run"):
return self.stop_run()
elif (cmd == "shutdown"):
self.send_answer("exiting")
return "exit"
elif (cmd == "help"):
msg = """Available commands:
help\t\t\t\tShow this help
get_state\t\t\tShow current state of RunControl
get_setup\t\t\tShow current setup name
get_board_list\t\t\tShow list of boards in use with current setup
get_board_config_daq <b>\tShow current configuration of board DAQ process<b>
get_board_config_zsup <b>\tShow current configuration of board ZSUP process<b>
get_board_log_file_daq <b>\tGet name of log file for board DAQ process<b>
get_board_log_file_zsup <b>\tGet name of log file for board ZSUP process<b>
get_trig_config\t\t\tShow current configuration of trigger process
get_trig_log\t\t\tGet name of log file for trigger process
get_run_number\t\t\tReturn current run number
stop_run\t\t\t\tStop the run
shutdown\t\t\tTell RunControl server to exit (use with extreme care!)"""
self.send_answer(msg)
else:
# See if command can be handled by a regular expression
found_re = False
m = self.re_get_board_config_daq.match(cmd)
if (m):
self.send_answer(self.get_board_config_daq(int(m.group(1))))
found_re = True
m = self.re_get_board_config_zsup.match(cmd)
if (m):
self.send_answer(self.get_board_config_zsup(int(m.group(1))))
found_re = True
m = self.re_get_board_log_file_daq.match(cmd)
if (m):
self.send_answer(self.get_board_log_file_daq(int(m.group(1))))
found_re = True
m = self.re_get_board_log_file_zsup.match(cmd)
if (m):
self.send_answer(self.get_board_log_file_zsup(int(m.group(1))))
found_re = True
# No regular expression matched: command is unknown
if not found_re:
self.send_answer("unknown command")
#self.write_log('command '+cmd+' is unknown')
print "Command %s is unknown"%cmd
return "idle"
def state_initfail(self):
# Here we will insert some cleanup code to handle failed initialization
return "idle"
def get_command(self):
# First get length of string
l = ""
for i in range(5): # Max 99999 characters
ch = self.connection.recv(1)
if ch:
l += ch
else:
print "Client closed connection"
return "client_close"
ll = int(l)
# Then read the right amount of characters from the socket
cmd = ""
for i in range(ll):
ch = self.connection.recv(1)
if ch:
cmd += ch
else:
print "Client closed connection"
return "client_close"
print "Received command %s"%cmd
return cmd
def send_answer(self,answer):
if len(answer)<100000:
print "Sending answer %s"%answer
self.connection.sendall("%5.5d"%len(answer)+answer)
else:
print "Answer is too long: cannot send"
def get_board_config_daq(self,brdid):
if brdid in self.run.boardid_list:
return self.run.adcboard_list[self.run.boardid_list.index(brdid)].format_config_daq()
else:
return "ERROR: board id %d does not exist"%brdid
def get_board_config_zsup(self,brdid):
if brdid in self.run.boardid_list:
return self.run.adcboard_list[self.run.boardid_list.index(brdid)].format_config_zsup()
else:
return "ERROR: board id %d does not exist"%brdid
def get_board_log_file_daq(self,brdid):
if brdid in self.run.boardid_list:
return self.run.adcboard_list[self.run.boardid_list.index(brdid)].log_file_daq
else:
return "ERROR: board id %d does not exist"%brdid
def get_board_log_file_zsup(self,brdid):
if brdid in self.run.boardid_list:
return self.run.adcboard_list[self.run.boardid_list.index(brdid)].log_file_zsup
else:
return "ERROR: board id %d does not exist"%brdid
def get_trig_config(self):
return self.run.trigger.format_config()
def get_trig_log(self):
return self.run.trigger.log_file
def read_setup_list(self):
# Get list of available setups
setup_main_dir = self.daq_dir+"/setup"
setups = []
for top,dirs,files in os.walk(setup_main_dir):
if (top == setup_main_dir):
for setup_dir in dirs: setups.append(setup_dir)
setups.sort()
return setups
def get_setup_list(self):
return str(self.read_setup_list())
def change_setup(self,setup):
# Check if requested setup is known
if not (setup in self.read_setup_list()):
print "change_setup - ERROR: request to set unknown setup %s"%setup
return "error"
# Change (or reload) setup
if (setup==self.run.setup):
print "change_setup - reloading setup %s"%setup
else:
print "change_setup - changing setup from %s to %s"%(self.run.setup,setup)
if self.run.change_setup(setup) == "error":
print "RunControlServer::change_setup - ERROR while loading new setup %s"%setup
return "error"
print "change_setup - saving new setup %s to %s\n"%(setup,self.lus_file)
with open(self.lus_file,"w") as lf: lf.write("%s\n"%setup)
return setup
def new_run(self):
## Retrieve run number - next=next run from DB, dummy=dummy run (i.e. run nr=0)
## Return run number used (0 for dummy run) or "error" for invalid answer
#newrun_number = 0
#self.send_answer("run_number")
#ans = self.get_command()
#if (ans=="next"):
# newrun_number = self.db.get_last_run_in_db()+1
#elif (ans=="dummy"):
# newrun_number = 0
#elif (ans == "error"):
# print "run_number - client returned error"
# return "error"
#elif (ans=="client_close"):
# return "client_close"
#else:
# print "run_number - invalid option %s received"%ans
# self.send_answer("error")
# return "error"
#self.send_answer(str(newrun_number))
# Report current setup
self.send_answer("setup %s"%self.run.setup)
# Assign number to new run using first free number in DB
newrun_number = self.db.get_last_run_in_db()+1
# Retrieve run type. Accept only from list of known run types
# Return run type used or "error" for invalid answer
self.send_answer("run_types %s"%",".join(self.run_type_list))
newrun_type = ""
#self.send_answer("run_type")
ans = self.get_command()
if (ans == "error"):
print "run_type - client returned error"
return "error"
elif (ans=="client_close"):
return "client_close"
elif ans in self.run_type_list:
newrun_type = ans
if newrun_type == "FAKE":
print "run_type - *** FAKE run requested: setting run number to 0 ***"
newrun_number = 0
self.send_answer(newrun_type)
else:
print "run_type - invalid option %s received"%ans
self.send_answer("error")
return "error"
#newrun_type = None
#for rtype in self.run_type_list:
# if (ans == rtype): newrun_type = ans
#if (newrun_type):
# # Verify that FAKE runs are not stored in the DB
# if (newrun_type == "FAKE" and newrun_number != 0):
# print "run_type - attempt to store FAKE run in DB: this is not allowed"
# self.send_answer("error")
# return "error"
# self.send_answer(newrun_type)
#else:
# print "run_type - invalid option %s received"%ans
# self.send_answer("error")
# return "error"
# Now we can send back the assigned run number
self.send_answer("run_number %d"%newrun_number)
newrun_user = ""
self.send_answer("shift_crew")
ans = self.get_command()
if (ans=="client_close"): return "client_close"
newrun_user = ans
newrun_comment = ""
self.send_answer("run_comment")
ans = self.get_command()
if (ans=="client_close"): return "client_close"
newrun_comment = ans
print "Run number: %d"%newrun_number
print "Run type: %s"%newrun_type
print "Run crew: %s"%newrun_user
print "Run comment: %s"%newrun_comment
# Set run configuration according to user's request
self.run.run_number = newrun_number
self.run.run_type = newrun_type
self.run.run_user = newrun_user
self.run.run_comment_start = newrun_comment
if not self.run.change_run():
self.send_answer("init_error")
return "error"
# Create directory to host log files
print "Creating log directory %s"%self.run.log_dir
self.run.create_log_dir()
# Write configuration files
print "Writing configuration files for run %d"%self.run.run_number
self.run.write_config()
# Create merger input and output lists
self.run.create_merger_input_list()
self.run.create_merger_output_list()
# Start run initialization procedure
self.send_answer("init_start")
## Create level1 output rawdata directories
self.run.create_level1_output_dirs()
# Create pipes for data transfer
print "Creating named pipes for run %d"%self.run.run_number
self.run.create_fifos()
# Start Level1 processes
for lvl1 in (self.run.level1_list):
p_id = lvl1.start_level1()
if p_id:
print "Level1 %d - Started with process id %d"%(lvl1.level1_id,p_id)
self.send_answer("level1 %d ready"%lvl1.level1_id)
else:
print "Level1 %d - ERROR: could not start process"%lvl1.level1_id
self.send_answer("level1 %d fail"%lvl1.level1_id)
time.sleep(0.5)
# Create receiving end of network tunnels (if needed)
self.run.create_receivers()
# Start merger
p_id = self.run.merger.start_merger()
if p_id:
print "Merger - Started with process id %d"%p_id
self.send_answer("merger ready")
else:
print "Merger - ERROR: could not start process"
self.send_answer("merger fail")
# Create sending ends of network tunnels (if needed)
self.run.create_senders()
# Start trigger process
p_id = self.run.trigger.start_trig()
if p_id:
print "Trigger - Started with process id %d"%p_id
self.send_answer("trigger init")
else:
print "Trigger - ERROR: could not start process"
self.send_answer("trigger fail")
# Start ZSUP for all boards
for adc in (self.run.adcboard_list):
p_id = adc.start_zsup()
if p_id:
print "ADC board %02d - Started ZSUP with process id %d"%(adc.board_id,p_id)
self.send_answer("adc %d zsup_init"%adc.board_id)
else:
print "ADC board %02d - ERROR: could not start ZSUP"%adc.board_id
self.send_answer("adc %d zsup_fail"%adc.board_id)
time.sleep(0.5)
# Start DAQ for all boards
for adc in (self.run.adcboard_list):
p_id = adc.start_daq()
if p_id:
print "ADC board %02d - Started DAQ with process id %d"%(adc.board_id,p_id)
self.send_answer("adc %d daq_init"%adc.board_id)
adc.status = "init"
else:
print "ADC board %02d - ERROR: could not start DAQ"%adc.board_id
self.send_answer("adc %d daq_fail"%adc.board_id)
adc.status = "fail"
time.sleep(0.5)
# Wait for trigger to complete initialization
n_try = 1
while(True):
trig_status = self.check_trig_init_status(self.run.trigger)
if (trig_status == "ready"):
print "Trigger board - Initialized and ready for DAQ"
self.send_answer("trigger ready")
break
elif (trig_status == "fail"):
print "*** ERROR *** Trigger board failed the initialization. Cannot start run"
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_INIT_ERROR)
self.send_answer("init_fail")
return "initfail"
n_try += 1
if (n_try>=60):
print "*** ERROR *** Trigger board did not initialize within 30sec. Cannot start run"
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_INIT_ERROR)
self.send_answer("init_timeout")
return "initfail"
time.sleep(0.5)
# Wait for all boards to complete initialization
n_try = 0
while(1):
all_boards_init = True
all_boards_ready = True
for adc in (self.run.adcboard_list):
# Check if any board changed status
if (adc.status == "init"):
adc.status = self.check_init_status(adc)
if (adc.status == "ready"):
# Initialization ended OK
print "ADC board %02d - Initialized and ready for DAQ"%adc.board_id
self.send_answer("adc %d ready"%adc.board_id)
elif (adc.status == "fail"):
# Problem during initialization
print "ADC board %02d - *** Initialization failed ***"%adc.board_id
self.send_answer("adc %d fail"%adc.board_id)
else:
# This board is still initializing
all_boards_init = False
# Check if any board is in fail status
if (adc.status == "fail"): all_boards_ready = False
# Check if all boards completed initialization
if (all_boards_init):
# Check if all boards initialized correctly
if (all_boards_ready):
print "ADC boards - All boards initialized and ready for DAQ"
self.send_answer("adc all ready")
break
#self.send_answer("init_ready")
#return "initialized"
else:
print "*** ERROR *** One or more boards failed the initialization. Cannot start run"
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_INIT_ERROR)
self.send_answer("init_fail")
return "initfail"
# Some boards are still initializing: keep waiting (wait up to ~30sec)
n_try += 1
if (n_try>=60):
print "*** ERROR *** One or more boards did not initialize within 30sec. Cannot start run"
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_INIT_ERROR)
self.send_answer("init_timeout")
return "initfail"
time.sleep(0.5)
# All subsystems initialized: ready to start the run
print "RunControl - All subsystems initialized: DAQ run can be started"
if (self.run.run_number):
self.db.set_run_time_init(self.run.run_number,self.db.now_str())
self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_INITIALIZED)
self.send_answer("init_ready")
return "initialized"
def start_run(self):
print "Starting run"
## Create "start the run" tag file
#open(self.run.start_file,'w').close()
self.run.start()
## Update run status in DB
#if (self.run.run_number):
# self.db.set_run_time_start(self.run.run_number,self.db.now_str())
# self.db.set_run_status(self.run.run_number,2) # Status 2: run started
self.send_answer("run_started")
# RunControl is now in "running" mode
return "running"
def stop_run(self):
self.send_answer("run_comment_end")
ans = self.get_command()
if (ans=="client_close"): return "client_close"
print "End of Run comment: %s"%ans
self.run.run_comment_end = ans
print "Stopping run"
#if (self.run.run_number):
# self.db.set_run_status(self.run.run_number,3) # Status 3: run stopped normally
self.run.final_status = self.db.DB_RUN_STATUS_END_OK
return self.terminate_run()
def abort_run(self):
print "Aborting run"
self.run.run_comment_end = "Run aborted"
self.run.final_status = self.db.DB_RUN_STATUS_ABORTED
#if (self.run.run_number):
# self.db.set_run_status(self.run.run_number,4) # Status 4: run aborted
return self.terminate_run()
def terminate_run(self):
#if (self.run.run_number):
# self.db.set_run_time_stop(self.run.run_number,self.db.now_str())
# self.db.set_run_comment_end(self.run.run_number,self.db.now_str(),self.run.run_comment_end)
## Create "stop the run" tag file
#open(self.run.quit_file,'w').close()
self.send_answer("terminate_start")
self.run.stop()
terminate_ok = True
# Run stop_daq procedure for each ADC board
for adc in (self.run.adcboard_list):
if adc.stop_daq():
self.send_answer("adc %d daq_terminate_ok"%adc.board_id)
print "ADC board %02d - DAQ terminated correctly"%adc.board_id
else:
terminate_ok = False
self.send_answer("adc %d daq_terminate_error"%adc.board_id)
print "ADC board %02d - WARNING: problems while terminating DAQ"%adc.board_id
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_END_ERROR)
# Run stop_zsup procedure for each ADC board
for adc in (self.run.adcboard_list):
if adc.stop_zsup():
self.send_answer("adc %d zsup_terminate_ok"%adc.board_id)
print "ADC board %02d - ZSUP terminated correctly"%adc.board_id
else:
terminate_ok = False
self.send_answer("adc %d zsup_terminate_error"%adc.board_id)
print "ADC board %02d - WARNING: problems while terminating ZSUP"%adc.board_id
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_END_ERROR)
## If this is a real run, get final info from merger before stopping it
#if (self.run.run_number):
# (tot_evts,tot_size) = self.db.get_merger_final_info(self.run.merger.merger_id)
# self.db.set_run_total_events(self.run.run_number,tot_evts)
# Run stop_trig procedure
if self.run.trigger.stop_trig():
self.send_answer("trigger terminate_ok")
print "Trigger terminated correctly"
else:
terminate_ok = False
self.send_answer("trigger terminate_error")
print "WARNING: problems while terminating Trigger"
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_END_ERROR)
# Run stop_merger procedure
if self.run.merger.stop_merger():
self.send_answer("merger terminate_ok")
print "Merger terminated correctly"
else:
terminate_ok = False
self.send_answer("merger terminate_error")
print "WARNING: problems while terminating Merger"
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_END_ERROR)
# Run stop_level1 procedures
for lvl1 in self.run.level1_list:
if lvl1.stop_level1():
self.send_answer("level1 %d terminate_ok"%lvl1.level1_id)
print "Level1 %02d terminated correctly"%lvl1.level1_id
else:
terminate_ok = False
self.send_answer("level1 %d terminate_error"%lvl1.level1_id)
print "Level1 %02d - WARNING: problems while terminating"%lvl1.level1_id
if (self.run.run_number):
self.db.set_run_status(self.run.run_number,self.db.DB_RUN_STATUS_END_ERROR)
# Parse all log files looking for DBINFO lines
self.run.trigger.parse_log()
for adc in (self.run.adcboard_list):
adc.parse_log_daq()
adc.parse_log_zsup()
self.run.merger.parse_log()
for lvl1 in self.run.level1_list:
lvl1.parse_log()
# Update total number of events for run
self.db.set_run_total_events(self.run.run_number,self.db.compute_run_total_events(self.run.run_number))
# Clean up run directory
self.run.clean_up()
if terminate_ok:
self.send_answer("terminate_ok")
else:
self.send_answer("terminate_error")
# At the end of this procedure RunControl is back to "idle" mode
return "idle"
def check_init_status(self,adc):
if adc.node_id == 0:
if (os.path.exists(adc.initok_file_daq) and os.path.exists(adc.initok_file_zsup)):
return "ready"
elif (os.path.exists(adc.initfail_file_daq) or os.path.exists(adc.initfail_file_zsup)):
return "fail"
else:
return "init"
else:
if (self.file_exists(adc.node_ip,adc.initok_file_daq) and self.file_exists(adc.node_ip,adc.initok_file_zsup)):
return "ready"
elif (self.file_exists(adc.node_ip,adc.initfail_file_daq) or self.file_exists(adc.node_ip,adc.initfail_file_zsup)):
return "fail"
else:
return "init"
def check_trig_init_status(self,trig):
if trig.node_id == 0:
if (os.path.exists(trig.initok_file)):
return "ready"
elif (os.path.exists(trig.initfail_file)):
return "fail"
else:
return "init"
else:
if (self.file_exists(trig.node_ip,trig.initok_file)):
return "ready"
elif (self.file_exists(trig.node_ip,trig.initfail_file)):
return "fail"
else:
return "init"
def file_exists(self,node_ip,name):
command = "ssh -i %s %s '( test -e %s )'"%(self.ssh_id_file,node_ip,name)
rc = os.system(command)
if (rc == 0): return True
return False
