def tickets_home(request): sounds_in_moderators_queue_count = _get_sounds_in_moderators_queue_count( request.user) new_upload_count = new_sound_tickets_count() tardy_moderator_sounds_count = len(list(_get_tardy_moderator_tickets())) tardy_user_sounds_count = len(list(_get_tardy_user_tickets())) sounds_queued_count = Sound.objects.filter( processing_ongoing_state='QU').count() sounds_pending_count = Sound.objects.filter(processing_state='PE').count() sounds_processing_count = Sound.objects.filter( processing_ongoing_state='PR').count() sounds_failed_count = Sound.objects.filter(processing_state='FA').count() # Get gearman status try: gm_admin_client = gearman.GearmanAdminClient( settings.GEARMAN_JOB_SERVERS) gearman_status = gm_admin_client.get_status() except gearman.errors.ServerUnavailable: gearman_status = list() tvars = { "new_upload_count": new_upload_count, "tardy_moderator_sounds_count": tardy_moderator_sounds_count, "tardy_user_sounds_count": tardy_user_sounds_count, "sounds_queued_count": sounds_queued_count, "sounds_pending_count": sounds_pending_count, "sounds_processing_count": sounds_processing_count, "sounds_failed_count": sounds_failed_count, "gearman_status": gearman_status, "sounds_in_moderators_queue_count": sounds_in_moderators_queue_count } return render(request, 'tickets/tickets_home.html', tvars)
def get_gearman_status(request): try: gm_admin_client = gearman.GearmanAdminClient(settings.GEARMAN_JOB_SERVERS) gearman_status = gm_admin_client.get_status() except gearman.errors.ServerUnavailable: gearman_status = list() return render(request, 'monitor/gearman_status.html', {'gearman_status': gearman_status})
def discover_tasks(server, port): d = {'data': []} gm_admin_client = gearman.GearmanAdminClient( [':'.join([server, str(port)])]) for task in gm_admin_client.get_status(): d['data'].append({'{#TASK}': task['task']}) return json.dumps(d)
def _get_client(self, host, port): if not (host, port) in self.gearman_clients: self.log.debug("Connecting to gearman at address %s:%s" % (host, port)) self.gearman_clients[(host, port)] = gearman.GearmanAdminClient( ["%s:%s" % (host, port)]) return self.gearman_clients[(host, port)]
def collect(self): """ Collector gearman stats """ def get_fds(gearman_pid_path): with open(gearman_pid_path) as fp: gearman_pid = fp.read().strip() proc_path = os.path.join('/proc', gearman_pid, 'fd') command = [self.config['bin'], proc_path] if str_to_bool(self.config['use_sudo']): command.insert(0, self.config['sudo_cmd']) process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, errors = process.communicate() if errors: raise Exception(errors) return len(output.splitlines()) def publish_server_stats(gm_admin_client): # Publish idle/running worker counts # and no. of tasks queued per task for entry in gm_admin_client.get_status(): total = entry.get('workers', 0) running = entry.get('running', 0) idle = total - running self.dimensions = { 'task': entry['task'] } # Internally, this dict is cleared on self.publish self.publish('gearman.queued', entry['queued']) self.dimensions = {'type': 'running'} self.publish('gearman.workers', running) self.dimensions = {'type': 'idle'} self.publish('gearman.workers', idle) try: if gearman is None: self.log.error("Unable to import python gearman client") return # Collect and Publish Metrics self.log.debug("Using pid file: %s & gearman endpoint : %s", self.config['gearman_pid_path'], self.config['url']) gm_admin_client = gearman.GearmanAdminClient([self.config['url']]) self.publish('gearman.ping', gm_admin_client.ping_server()) self.publish('gearman.fds', get_fds(self.config['gearman_pid_path'])) publish_server_stats(gm_admin_client) except Exception, e: self.log.error("GearmanCollector Error: %s", e)
def _get_client(self, host, port): try: import gearman except ImportError: raise Exception( "Cannot import Gearman module. Check the instructions to install" + "this module at https://app.datadoghq.com/account/settings#integrations/gearman") self.log.debug("Connecting to gearman at address %s:%s" % (host, port)) return gearman.GearmanAdminClient(["%s:%s" % (host, port)])
def get_task_info(server, port, function, task_name): gm_admin_client = gearman.GearmanAdminClient( [':'.join([server, str(port)])]) if function == 'jobs': for task in gm_admin_client.get_status(): if task['task'] == task_name: print task['queued'] elif function == 'workers': for task in gm_admin_client.get_status(): if task['task'] == task_name: print task['workers']
def index(): 'get some gearman stats' gmc = gearman.GearmanAdminClient([app.config.get('GEARMAN_SERVER','localhost:4730')]) gm_status, gm_version, gm_workers, gm_ping = 'NA', '', [], 0 try: gm_status = gmc.get_status() gm_version = gmc.get_version() gm_workers = gmc.get_workers() gm_ping = gmc.ping_server() except Exception: pass return render_template('index.html', session=session, gm_status=gm_status, gm_version=gm_version, gm_workers=gm_workers, gm_ping=gm_ping)
def _set_new_gm_server(self, block=False): while True: if self.gm_host: self.gm_hosts.append(self.gm_host) for i in range(len(self.gm_hosts)): LOG.debug('Trying set new gearman server...') try: self.gm_host = self.gm_hosts.pop(0) gm_adm_client = gearman.GearmanAdminClient([self.gm_host]) gm_adm_client.ping_server() ## GearmanAdminClient.send_maxqueue - python gearman client bug #try: # gm_adm_client.send_maxqueue('message.send', QUEUE_SIZE) #except: # LOG.warning(util.exc_info()) self.gm_client = gearman.GearmanClient([self.gm_host]) LOG.debug('Gearman server: %s' % self.gm_host) break except gearman.errors.ServerUnavailable: LOG.error('%s %s' % (self.gm_host, util.exc_info())) self.gm_hosts.append(self.gm_host) self.gm_host = None except: LOG.error(util.exc_info()) raise else: if block: time.sleep(5) continue else: LOG.error('Set new gearman server failed') raise gearman.errors.ServerUnavailable break
import gearman gm_admin_client = gearman.GearmanAdminClient(['localhost:4730']) workers = gm_admin_client.get_workers() print(workers)
def monitor_home(request): sounds_in_moderators_queue_count =\ tickets.views._get_sounds_in_moderators_queue_count(request.user) new_upload_count = tickets.views.new_sound_tickets_count() tardy_moderator_sounds_count =\ len(tickets.views._get_tardy_moderator_tickets()) tardy_user_sounds_count = len(tickets.views._get_tardy_user_tickets()) # Processing sounds_queued_count = sounds.views.Sound.objects.filter( processing_ongoing_state='QU').count() sounds_pending_count = sounds.views.Sound.objects.\ filter(processing_state='PE')\ .exclude(processing_ongoing_state='PR')\ .exclude(processing_ongoing_state='QU')\ .count() sounds_processing_count = sounds.views.Sound.objects.filter( processing_ongoing_state='PR').count() sounds_failed_count = sounds.views.Sound.objects.filter( processing_state='FA').count() sounds_ok_count = sounds.views.Sound.objects.filter( processing_state='OK').count() # Analysis sounds_analysis_pending_count = sounds.views.Sound.objects.filter( analysis_state='PE').count() sounds_analysis_queued_count = sounds.views.Sound.objects.filter( analysis_state='QU').count() sounds_analysis_ok_count = sounds.views.Sound.objects.filter( analysis_state='OK').count() sounds_analysis_failed_count = sounds.views.Sound.objects.filter( analysis_state='FA').count() sounds_analysis_skipped_count = sounds.views.Sound.objects.filter( analysis_state='SK').count() # Get gearman status try: gm_admin_client = gearman.GearmanAdminClient( settings.GEARMAN_JOB_SERVERS) gearman_status = gm_admin_client.get_status() except gearman.errors.ServerUnavailable: gearman_status = list() tvars = { "new_upload_count": new_upload_count, "tardy_moderator_sounds_count": tardy_moderator_sounds_count, "tardy_user_sounds_count": tardy_user_sounds_count, "sounds_queued_count": sounds_queued_count, "sounds_pending_count": sounds_pending_count, "sounds_processing_count": sounds_processing_count, "sounds_failed_count": sounds_failed_count, "sounds_ok_count": sounds_ok_count, "sounds_analysis_pending_count": sounds_analysis_pending_count, "sounds_analysis_queued_count": sounds_analysis_queued_count, "sounds_analysis_ok_count": sounds_analysis_ok_count, "sounds_analysis_failed_count": sounds_analysis_failed_count, "sounds_analysis_skipped_count": sounds_analysis_skipped_count, "gearman_status": gearman_status, "sounds_in_moderators_queue_count": sounds_in_moderators_queue_count } return render(request, 'monitor/monitor.html', tvars)
#!/usr/bin/env python # -*- coding: utf-8 -*- import string import web import gearman gearman_hostports = [('localhost', 4730), ('localhost', 4731)] gearman_connections = {} for hostport in gearman_hostports: gearman_connections[hostport] = gearman.GearmanAdminClient([hostport]) def _get_info_from_gearman(): server_infos = [] for hostport in gearman_hostports: server_info = {'hostport': hostport} try: gm_conn = gearman_connections[hostport] version = gm_conn.get_version() status = gm_conn.get_status() cl_wks = gm_conn.get_workers() clients = [w for w in cl_wks if len(w['tasks']) == 0] workers = [w for w in cl_wks if len(w['tasks']) > 0] server_info['version'] = version server_info['status'] = status server_info['workers'] = workers server_info['clients'] = clients server_info['failed'] = False
def run(self): db_manager = util.DBManager(self.config['connections']['mysql']) db = db_manager.get_db() self._set_new_gm_server(block=True) timestep = 5 while True: session = db.session try: gm_adm_client = gearman.GearmanAdminClient([self.gm_host]) gm_adm_client.ping_server() # fix gearman v2.0.2 memory leak bug self.gm_client = gearman.GearmanClient([self.gm_host]) self._update_submitted_jobs() if len(self.submitted_jobs) > 5000: LOG.warning('Too much of a submitted jobs. Skip iteration') time.sleep(timestep) continue where1 = and_(db.messages.type == 'out', db.messages.status == 0, db.messages.message_version == 2) where2 = and_( func.unix_timestamp(db.messages.dtlasthandleattempt) +\ db.messages.handle_attempts *\ CRATIO < func.unix_timestamp(func.now())) if self.submitted_jobs: where3 = and_( not_( db.messages.messageid.in_( self.submitted_jobs.keys()))) msgs = session.query( db.messages.messageid, db.messages.handle_attempts).filter( where1, where2, where3).order_by( asc(db.messages.id)).all()[0:QSIZE] else: msgs = session.query( db.messages.messageid, db.messages.handle_attempts).filter( where1, where2).order_by(asc( db.messages.id)).all()[0:QSIZE] for msg in msgs: # simple unique version req = self.gm_client.submit_job('message.send', msg.messageid, unique=msg.messageid[0:64], wait_until_complete=False) # sha256 unique version ''' req = self.gm_client.submit_job('message.send', msg.messageid, unique=hashlib.sha256(msg.messageid).hexdigest(), wait_until_complete=False) ''' self.gm_client.wait_until_jobs_accepted([req]) self.submitted_jobs.update({ msg.messageid: (req, int(time.time() + CRATIO * (msg.handle_attempts + 1))) }) LOG.info('Sumbit message: msg_id:%s' % msg.messageid) except db_exc.OperationalError: LOG.error(util.exc_info()) time.sleep(5) except gearman.errors.ServerUnavailable: LOG.error(util.exc_info()) self._set_new_gm_server(block=True) except: LOG.error(util.exc_info()) raise finally: session.close() session.remove() time.sleep(timestep)
def __init__(self): self.client = gearman.GearmanClient(['localhost:4730']) self.admin = gearman.GearmanAdminClient(['localhost:4730']) self.job_list = [] self.db = MysqlConn()
def __init__(self): self.client = gearman.GearmanClient(['localhost:4730']) self.admin = gearman.GearmanAdminClient(['localhost:4730']) self.job_list = [] self.conn, self.cursor = connect()
# -*- coding: utf-8 -*- import gearman ad_client = gearman.GearmanAdminClient(['www.aispring.top:4730']) lists = ad_client.get_workers() for row in lists: print row print "\n" lists = ad_client.get_status() for row in lists: print row
#!/usr/bin/python import sys import time import yaml import rethinkdb as r import gearman parameter_file = open("parameters.yml", "r") parameters = yaml.load(parameter_file) rethink = r.connect(parameters['rethinkdb_server']['host'], parameters['rethinkdb_server']['port']).repl() rethink.use(parameters['rethinkdb_server']['database']) gm_client = gearman.GearmanClient(parameters['gearman_server']['hosts']) gm_admin = gearman.GearmanAdminClient(parameters['gearman_server']['hosts']) max_tasks = parameters['gearman_server']['max_tasks'] def num_dequeue_urls(): gearman_status = gm_admin.get_status() for func in gearman_status: if func['task'] == "crawler": return int(max_tasks - func['queued']) return max_tasks def main(argv): # Main code here print "I'm dequeue worker"
def run(self): worker_found = False task_name = 'exe_job' try: # Check if there are a workers that have the ssh job registered # If not, bail out gmadmin = gearman.GearmanAdminClient(gearman_servers) stats = list(gmadmin.get_status()) for stat in stats: if stat['task'] == task_name and stat['workers'] > 0: worker_found = True break if worker_found: l.debug("Found atleast one worker with the task: " + task_name + " registered") else: l.error("Did not find any workers with the task: " + task_name + " registered") sys.exit(1) # Gearman client should now submit tasks to the gearman workers # We submit jobs based on what is specified in parallelism self._gmclient = gmclient = gearman.GearmanClient(gearman_servers) num_hosts = len(self._hosts) num_parallel = get_num_hosts(self._parallelism, num_hosts) if num_parallel == None: l.error("The parallelism key should be a positive number") sys.exit(1) start = 0 while True: for i in range(start, start + num_parallel): try: host = str( self._hosts[i]) # Gearman fails on unicode strings debug_str = "job_id: " + self._job_id + ", command: " + self._command debug_str += ", host: " + host + ", retries: " + str( self._retries) l.debug( "Submitting job with the following attributes to the gearman worker: " + debug_str) worker_args = json.dumps({ "host": host, "command": self._command, "retries": str(self._retries) }) gmjob = gmclient.submit_job(task_name, worker_args, background=False, wait_until_complete=False) self._gmjobs.append(gmjob) except IndexError: return self.poll() start = start + i + 1 except gearman.errors.ServerUnavailable: l.error("Gearman server(s): " + str(gearman_servers) + " not available!") sys.exit(1)
def __init__(self, config): self.host = config.get("host", DEFAULT_HOST) self.port = config.get("port", DEFAULT_PORT) server_url = "%s:%s" % (self.host, self.port) self.gm_admin_client = gearman.GearmanAdminClient([server_url])
def _get_client(self, host, port): self.log.debug("Connecting to gearman at address %s:%s" % (host, port)) return gearman.GearmanAdminClient(["%s:%s" % (host, port)])
def __init__(self, core, gearman_host): super(GearmanStatusPlugin, self).__init__(core) self.admin_client = gearman.GearmanAdminClient([gearman_host])
def handle(self, **options): gm_client = gearman.GearmanAdminClient(settings.GEARMAN_JOB_SERVERS) for task in gm_client.get_status(): for key, value in task.items(): self.stdout.write('%s: %s\n' % (key, str(value)))
def get_server_info(self, task_filter=None): """Read Gearman server info - status, workers and and version.""" result = '' # Read server status info. client = gearman.GearmanAdminClient([self.host]) self.server_version = client.get_version() self.tasks = client.get_status() self.workers = client.get_workers() self.ping_time = client.ping_server() self.ping_time_str = '{0:0.016f}'.format(self.ping_time) # if task_filter is set, filter list of tasks and workers by regex pattern task_filter if task_filter: # filter tasks self.tasks = [ item for item in self.tasks if task_filter in item['task'] ] # filter workers by registered task name self.workers = [ item for item in self.workers if item['tasks'] and task_filter in [t for t in item['tasks']] ] # sort tasks by task name self.tasks = sorted(self.tasks, key=lambda item: item['task']) # sort registered workers by task name self.workers = sorted(self.workers, key=lambda item: item['tasks']) # Use prettytable if available, otherwise raw output. try: from prettytable import PrettyTable except ImportError: PrettyTable = None if PrettyTable is not None: # Use PrettyTable for output. # server table = PrettyTable([ 'Gearman Server Host', 'Gearman Server Version', 'Ping Response Time' ]) table.add_row([self.host, self.server_version, self.ping_time_str]) result += '{0:s}.\n\n'.format(table) # tasks table = PrettyTable( ['Task Name', 'Total Workers', 'Running Jobs', 'Queued Jobs']) for r in self.tasks: table.add_row( [r['task'], r['workers'], r['running'], r['queued']]) result += '{0:s}.\n\n'.format(table) # workers table = PrettyTable([ 'Worker IP', 'Registered Tasks', 'Client ID', 'File Descriptor' ]) for r in self.workers: if r['tasks']: # ignore workers with no registered task table.add_row([ r['ip'], ','.join(r['tasks']), r['client_id'], r['file_descriptor'] ]) result += '{0:s}.\n\n'.format(table) else: # raw output without PrettyTable result += 'Gearman Server Host:{0:s}\n'.format(self.host) result += 'Gearman Server Version:{0:s}.\n'.format( self.server_version) result += 'Gearman Server Ping Response Time:{0:s}.\n'.format( self.ping_time_str) result += 'Tasks:\n{0:s}\n'.format(self.tasks) result += 'Workers:\n{0:s}\n'.format(self.workers) return result