def RunWorker(connectionStringKey):
InitLogging()
while (True):
jobManager = JobManager(connectionStringKey)
jobCount = jobManager.count()
# not spawning new thread for unless queue has a job
# otherwise lot of SQS calls are made and bill is high
if jobCount > 0:
jobProcesses = []
while len(jobProcesses) < int(os.environ['MAX_JOB_THREADS']):
jobProcess = Process(target=DequeueAndStartJob,
args=(connectionStringKey, ))
jobProcess.start()
jobProcesses.append(jobProcess)
logging.info("Process limit reached")
for jobProcess in jobProcesses:
jobProcess.join(150)
if jobProcess.is_alive():
logging.warning("Process timed out. Terminating it")
jobProcess.terminate()
else:
logging.info("No job found in queue. Sleeping")
time.sleep(5)
def __init__(self):
JobManager.__init__(self)
pickledGraphs = "graphs.pickle"
self.pickledGraphs = join(self.jobManagerDir, pickledGraphs)
if not isfile(self.pickledGraphs):
self.initGraphs()
self.graphs = {}
self.loadGraphs()
def __init__(self):
self.daemon = True
vmms = None
if Config.VMMS_NAME == "tashiSSH":
from vmms.tashiSSH import TashiSSH
vmms = TashiSSH()
elif Config.VMMS_NAME == "ec2SSH":
from vmms.ec2SSH import Ec2SSH
vmms = Ec2SSH()
elif Config.VMMS_NAME == "localDocker":
from vmms.localDocker import LocalDocker
vmms = LocalDocker()
elif Config.VMMS_NAME == "distDocker":
from vmms.distDocker import DistDocker
vmms = DistDocker()
self.preallocator = Preallocator({Config.VMMS_NAME: vmms})
self.jobQueue = JobQueue(self.preallocator)
if not Config.USE_REDIS:
# creates a local Job Manager if there is no persistent
# memory between processes. Otherwise, JobManager will
# be initiated separately
JobManager(self.jobQueue).start()
logging.basicConfig(
filename=Config.LOGFILE,
format="%(levelname)s|%(asctime)s|%(name)s|%(message)s",
level=Config.LOGLEVEL,
)
self.start_time = time.time()
self.log = logging.getLogger("TangoServer")
self.log.info("Starting Tango server")
def DequeueAndStartJob(connectionStringKey):
"""
Dequeue a job from the queue and start executing it.
"""
logging.info("Dequeing a job.")
jobManager = JobManager(connectionStringKey)
job = jobManager.dequeueJob()
if job is None:
logging.info("No job found.")
return
logging.info("Job found. Starting it now." + "Job id: %s. Job Name: %s.",
job.jobId, job.jobName)
RunJob(job)
def main():
global server_socket, clients
server_socket, clients = None, []
global last_auto_resched
try:
defs.db_m = DatabaseManager()
defs.job_m = defs.db_m.get_jobManager()
if not defs.job_m:
defs.job_m = JobManager()
defs.db_m.register_jobManager(defs.job_m)
transaction.commit()
last_auto_resched = time()
server_socket = listen_on(SERVER_BIND, SERVER_PORT)
socket2addr = {}
print(WELCOME_MESSAGE)
print("Enter 'Help' or 'H' for more details.\n")
show_prompt()
while True:
r, _, _ = select([server_socket, stdin] + clients, [], [])
for c in r:
if c == server_socket:
c, addr = server_socket.accept()
socket2addr[c] = addr
clients.append(c)
elif c == stdin:
handle_request(input())
show_prompt()
else:
handle_client(c, socket2addr[c])
auto_resched()
transaction.commit()
except Exception as e:
ndprint('Error occurred: \n\t' + repr(e))
finally:
print("Start closing.")
if defs.db_m:
defs.db_m.close()
if server_socket:
server_socket.close()
for c in clients:
c.close()
print("Done closing.")
def __init__(self):
logging.basicConfig(
filename = self.LOGFILE,
format = "%(levelname)s|%(asctime)s|%(name)s|%(message)s",
level = Config.LOGLEVEL
)
vmms = None
if Config.VMMS_NAME == "localSSH":
from vmms.localSSH import LocalSSH
vmms = LocalSSH()
elif Config.VMMS_NAME == "tashiSSH":
from vmms.tashiSSH import TashiSSH
vmms = TashiSSH()
elif Config.VMMS_NAME == "ec2SSH":
from vmms.ec2SSH import Ec2SSH
vmms = Ec2SSH()
elif Config.VMMS_NAME == "localDocker":
from vmms.localDocker import LocalDocker
vmms = LocalDocker()
self.vmms = {Config.VMMS_NAME: vmms}
self.preallocator = Preallocator(self.vmms)
self.queue = JobQueue(self.preallocator)
if not Config.USE_REDIS:
# creates a local Job Manager if there is no persistent
# memory between processes. Otherwise, JobManager will
# be initiated separately
JobManager(self.queue, self.vmms, self.preallocator)
self.tango = TangoServer(self.queue, self.preallocator, self.vmms)
logging.basicConfig(
filename=self.LOGFILE,
format="%(levelname)s|%(asctime)s|%(name)s|%(message)s",
level=Config.LOGLEVEL
)
logging.getLogger('boto').setLevel(logging.INFO)
self.log = logging.getLogger("TangoREST")
self.log.info("Starting RESTful Tango server")
self.status = Status()
def __init__(self):
JobManager.__init__(self, 'CLUSTER_JOBSQUEUE_CONNECTIONSTRING')