class PilotStreaming: def __init__(self): self._logger = PandaLogger().getLogger('PilotStreaming') return def run(self): """ Gets and iterates over ups queues, deciding the job requirements and sending these to Harvester via the command interface :return: """ # timing time_start = time.time() self._logger.debug('Start.') # get unified pilot streaming (ups) queues ups_queues = taskBuffer.ups_get_queues() self._logger.debug('UPS queues: {0}'.format(ups_queues)) # get worker stats worker_stats = taskBuffer.ups_load_worker_stats() for ups_queue in ups_queues: # get the worker and job stats for the queue try: tmp_worker_stats = worker_stats[ups_queue] self._logger.debug('worker_stats for queue {0}: {1}'.format(ups_queue, tmp_worker_stats)) # tmp_job_stats = job_stats[ups_queue] except KeyError: # skip queue if no data available self._logger.debug('No worker stats for queue {0}'.format(ups_queue)) continue new_workers_per_harvester = taskBuffer.ups_new_worker_distribution(ups_queue, tmp_worker_stats) self._logger.info('queue: {0}, results: {1}'.format(ups_queue, new_workers_per_harvester)) # variables for the harvester command command = '{0}:{1}'.format('SET_N_WORKERS', ups_queue) status = 'new' ack_requested = False lock_interval = None com_interval = None for harvester_id in new_workers_per_harvester: params = new_workers_per_harvester[harvester_id] taskBuffer.commandToHarvester(harvester_id, command, ack_requested, status, lock_interval, com_interval, params) # timing time_stop = time.time() self._logger.debug('Done. Pilot streaming took: {0} s'.format(time_stop - time_start)) return
class PilotStreaming: def __init__(self): self._logger = PandaLogger().getLogger('PilotStreaming') return def run(self): """ Gets and iterates over ups queues, deciding the job requirements and sending these to Harvester via the command interface :return: """ # get unified pilot streaming (ups) queues ups_queues = taskBuffer.ups_get_queues() self._logger.debug('UPS queues: {0}'.format(ups_queues)) # get worker stats worker_stats = taskBuffer.ups_load_worker_stats() # get global share distribution # hs_distribution = proxyS.get_hs_distribution() # gs_tree = proxyS # print(proxyS.tree.pretty_print_hs_distribution(proxyS._DBProxy__hs_distribution for ups_queue in ups_queues: # get the worker and job stats for the queue try: tmp_worker_stats = worker_stats[ups_queue] self._logger.debug('worker_stats for queue {0}: {1}'.format(ups_queue, tmp_worker_stats)) # tmp_job_stats = job_stats[ups_queue] except KeyError: # skip queue if no data available self._logger.debug('No worker stats for queue {0}'.format(ups_queue)) continue new_workers_per_harvester = taskBuffer.ups_new_worker_distribution(ups_queue, tmp_worker_stats) self._logger.info('queue: {0}, results: {1}'.format(ups_queue, new_workers_per_harvester)) # variables for the harvester command command = '{0}:{1}'.format('SET_N_WORKERS', ups_queue) status = 'new' ack_requested = False lock_interval = None com_interval = None for harvester_id in new_workers_per_harvester: params = new_workers_per_harvester[harvester_id] # TODO: figure out if a command lock call is necessary or how that works taskBuffer.commandToHarvester(harvester_id, command, ack_requested, status, lock_interval, com_interval, params) return
jobD = urlparse.parse_qs(data) # jobD indicates it's a job in dictionary format, not a JobSpec object return jobD if __name__ == "__main__": proxyS = DBProxy() proxyS.connect(panda_config.dbhost,panda_config.dbpasswd,panda_config.dbuser,panda_config.dbname) #proxyS.getCriteriaForGlobalShares('BNL-OSG') site = 'CERN-PROD' DIRECT = 'direct' WEB = 'web' mode = WEB if mode == DIRECT: for i in xrange(3): t_before = time.time() _logger.info(proxyS.getJobs(1, site, 'managed', None, 1000, 0, 'aipanda081.cern.ch', 20, None, None, None, None, None, None, None)) t_after = time.time() total = t_after-t_before _logger.info('Took {0}s'.format(total)) elif mode == WEB: job = retrieveJob(site) _logger.info(job)
class PilotStreaming: def __init__(self): self._logger = PandaLogger().getLogger('PilotStreaming') return def run(self): """ Gets and iterates over ups queues, deciding the job requirements and sending these to Harvester via the command interface :return: """ # timing time_start = time.time() self._logger.debug('Start.') # get unified pilot streaming (ups) queues ups_queues = taskBuffer.ups_get_queues() self._logger.debug('UPS queues: {0}'.format(ups_queues)) # get worker stats worker_stats = taskBuffer.ups_load_worker_stats() for ups_queue in ups_queues: # get the worker and job stats for the queue try: tmp_worker_stats = worker_stats[ups_queue] self._logger.debug('worker_stats for queue {0}: {1}'.format( ups_queue, tmp_worker_stats)) # tmp_job_stats = job_stats[ups_queue] except KeyError: # skip queue if no data available self._logger.debug( 'No worker stats for queue {0}'.format(ups_queue)) continue new_workers_per_harvester = taskBuffer.ups_new_worker_distribution( ups_queue, tmp_worker_stats) self._logger.info('queue: {0}, results: {1}'.format( ups_queue, new_workers_per_harvester)) # variables for the harvester command command = '{0}:{1}'.format('SET_N_WORKERS', ups_queue) status = 'new' ack_requested = False lock_interval = None com_interval = None for harvester_id in new_workers_per_harvester: params = new_workers_per_harvester[harvester_id] taskBuffer.commandToHarvester(harvester_id, command, ack_requested, status, lock_interval, com_interval, params) # timing time_stop = time.time() self._logger.debug( 'Done. Pilot streaming took: {0} s'.format(time_stop - time_start)) return
if __name__ == "__main__": proxyS = DBProxy() proxyS.connect(panda_config.dbhost, panda_config.dbpasswd, panda_config.dbuser, panda_config.dbname) #proxyS.getCriteriaForGlobalShares('BNL-OSG') site = 'CERN-PROD' DIRECT = 'direct' WEB = 'web' mode = WEB if mode == DIRECT: for i in xrange(3): t_before = time.time() _logger.info( proxyS.getJobs(1, site, 'managed', None, 1000, 0, 'aipanda081.cern.ch', 20, None, None, None, None, None, None, None)) t_after = time.time() total = t_after - t_before _logger.info('Took {0}s'.format(total)) elif mode == WEB: job = retrieveJob(site) _logger.info(job)