import os import sys import json from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestermessenger import shared_file_messenger workerID = int(sys.argv[1]) eventID = sys.argv[2] status = sys.argv[3] proxy = DBProxy() workSpec = proxy.get_worker_with_id(workerID) jobSpec = proxy.get_jobs_with_worker_id(workerID, None)[0] accessPoint = workSpec.get_access_point() try: os.makedirs(accessPoint) except: pass node = {} node['eventRangeID'] = eventID node['eventStatus'] = status f = open( os.path.join(accessPoint, shared_file_messenger.jsonEventsUpdateFileName), 'w') json.dump([node], f) f.close()
import os import sys from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy from pandaharvester.harvestercore.communicator_pool import CommunicatorPool from pandaharvester.harvestermessenger import shared_file_messenger workerID = int(sys.argv[1]) proxy = DBProxy() workSpec = proxy.get_worker_with_id(workerID) jobSpec = proxy.get_jobs_with_worker_id(workerID, None)[0] accessPoint = workSpec.get_access_point() try: os.makedirs(accessPoint) except: pass node = {} node['pandaID'] = jobSpec.PandaID node['jobsetID'] = jobSpec.jobParams['jobsetID'] node['taskID'] = jobSpec.taskID a = CommunicatorPool() tmpStat, tmpVal = a.getEventRanges(node) mess = shared_file_messenger.SharedFileMessenger() mess.feed_events(workSpec, tmpVal)
class HttpHandler(BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.dbProxy = DBProxy() self.tmpLog = None BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def log_message(self, format, *args): # suppress console logging pass def get_form(self): dataStr = self.rfile.read(int(self.headers['Content-Length'])) return json.loads(dataStr) def do_postprocessing(self, message): self.end_headers() self.wfile.write(message) def do_POST(self): # logger if self.tmpLog is None: self.tmpLog = core_utils.make_logger(_logger) toSkip = False form = None methodName = None dataStr = None message = '' # parse the form data posted try: form = self.get_form() except: message = 'corrupted json' toSkip = True # check parameters if not toSkip: toSkip = True # method is not set if 'methodName' not in form: message = 'methodName is not given' self.send_response(400) elif 'workerID' not in form: message = 'workerID is not given' self.send_response(400) elif 'data' not in form: message = 'data is not given' self.send_response(400) else: toSkip = False # get worker if not toSkip: try: workerID = form['workerID'] workSpec = self.dbProxy.get_worker_with_id(workerID) if workSpec is None: message = 'workerID={0} not found in DB'.format(workerID) self.send_response(400) else: # chose file and operation for each action methodName = form['methodName'] opType = None filePath = '' if methodName == 'requestJobs': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonJobRequestFileName) opType = 'w' elif methodName == 'getJobs': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jobSpecFileName) opType = 'r' elif methodName == 'requestEventRanges': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonEventsRequestFileName) opType = 'w' elif methodName == 'getEventRanges': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonEventsFeedFileName) opType = 'r' elif methodName == 'updateJobs': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonAttrsFileName) opType = 'w' elif methodName == 'uploadJobReport': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonJobReport) opType = 'w' elif methodName == 'uploadEventOutputDump': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonOutputsFileName) opType = 'w' elif methodName == 'setPandaIDs': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.pandaIDsFile) opType = 'w' else: self.send_response(501) message = 'method not implemented' toSkip = True # take action if not toSkip: # write actions if opType == 'w': # check if file exists if os.path.exists(filePath): message = 'previous request is not yet processed' self.send_response(503) else: with open(filePath, 'w') as fileHandle: json.dump(form['data'], fileHandle) message = 'OK' self.send_response(200) else: # read actions if os.path.exists(filePath): with open(filePath) as fileHandle: message = json.load(fileHandle) self.send_response(200) self.send_header('Content-Type', 'application/json') else: message = 'previous request is not yet processed' self.send_response(503) except: self.send_response(500) message = core_utils.dump_error_message(_logger) if harvester_config.frontend.verbose: self.tmpLog.debug('method={0} json={1} msg={2}'.format(methodName, dataStr, message)) # set the response self.do_postprocessing(message) return
class HttpHandler(BaseHTTPRequestHandler): def __init__(self, *args, **kwargs): self.dbProxy = DBProxy() self.tmpLog = None BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def log_message(self, format, *args): # suppress console logging pass def get_form(self): dataStr = self.rfile.read(int(self.headers['Content-Length'])) return json.loads(dataStr) def do_postprocessing(self, message): self.end_headers() self.wfile.write(message) def do_POST(self): # logger if self.tmpLog is None: self.tmpLog = core_utils.make_logger(_logger) toSkip = False form = None methodName = None dataStr = None message = '' # parse the form data posted try: form = self.get_form() except Exception: message = 'corrupted json' toSkip = True # check parameters if not toSkip: toSkip = True # method is not set if 'methodName' not in form: message = 'methodName is not given' self.send_response(400) elif 'workerID' not in form: message = 'workerID is not given' self.send_response(400) elif 'data' not in form: message = 'data is not given' self.send_response(400) else: toSkip = False # get worker if not toSkip: try: workerID = form['workerID'] workSpec = self.dbProxy.get_worker_with_id(workerID) if workSpec is None: message = 'workerID={0} not found in DB'.format(workerID) self.send_response(400) else: # chose file and operation for each action methodName = form['methodName'] opType = None filePath = '' if methodName == 'requestJobs': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonJobRequestFileName) opType = 'w' elif methodName == 'getJobs': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jobSpecFileName) opType = 'r' elif methodName == 'requestEventRanges': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonEventsRequestFileName) opType = 'w' elif methodName == 'getEventRanges': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonEventsFeedFileName) opType = 'r' elif methodName == 'updateJobs': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonAttrsFileName) opType = 'w' elif methodName == 'uploadJobReport': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonJobReport) opType = 'w' elif methodName == 'uploadEventOutputDump': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.jsonOutputsFileName) opType = 'w' elif methodName == 'setPandaIDs': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.pandaIDsFile) opType = 'w' elif methodName == 'killWorker': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.killWorkerFile) opType = 'w' elif methodName == 'heartbeat': filePath = os.path.join(workSpec.get_access_point(), shared_file_messenger.heartbeatFile) opType = 'w' else: self.send_response(501) message = 'method not implemented' toSkip = True # take action if not toSkip: # write actions if opType == 'w': # check if file exists. Methods such as heartbeat however need to overwrite the file if os.path.exists(filePath) and methodName not in ['heartbeat']: message = 'previous request is not yet processed' self.send_response(503) else: with open(filePath, 'w') as fileHandle: json.dump(form['data'], fileHandle) message = 'OK' self.send_response(200) else: # read actions if os.path.exists(filePath): with open(filePath) as fileHandle: try: _message = json.load(fileHandle) message = json.dumps(_message) self.send_header('Content-Type', 'application/json') except JSONDecodeError: _f_qs = open(filePath).read() # _message = dict(parse_qsl(_f_qs, keep_blank_values=True)) message = _f_qs self.send_header('Content-Type', 'text/plain') self.send_response(200) else: message = 'previous request is not yet processed' self.send_response(503) except Exception: self.send_response(500) message = core_utils.dump_error_message(_logger) if harvester_config.frontend.verbose: self.tmpLog.debug('method={0} json={1} msg={2}'.format(methodName, dataStr, message)) # set the response self.do_postprocessing(message) return