def notifier(self): """ Connect to the instruction queue and notify bugyou to create a queue for the plugin and start pushing the fedmsg messags. """ queue = Queue('instruction') queue.connect() for plugin in self.plugins: try: topic = self.config.get(plugin, 'topic') except ConfigParser.NoOptionError: log.error("Config does not exists") if topic is None: log.info("Config does not exists") continue payload = { 'type': 'create', 'queue_name': plugin, 'topic': topic, } task = Task(payload) queue.enqueue(task) if plugin in self.active_plugins: Plugin = self.active_plugins[plugin] plugin_obj = Plugin() plugin_obj.initialize()
class DarkserverConsumer(fedmsg.consumers.FedmsgConsumer): topic = 'org.fedoraproject.prod.buildsys.build.state.change' config_key = 'darkserver.consumer.enabled' def __init__(self, *args, **kwargs): super(DarkserverConsumer, self).__init__(*args, **kwargs) self.config = get_redis_config() self.jobqueue = Queue('jobqueue', self.config) self.jobqueue.connect() print 'DarkserverConsumer ready for action' def consume(self, msg): """ Consumer the koji messages """ msg_body = msg['body'] completed = msg_body['msg']['new'] == 1 if completed: build_id = msg_body['msg']['build_id'] instance = msg_body['msg']['instance'] release = msg_body['msg']['release'].split('.')[-1] if release != 'el5': info = { 'build_id': build_id, 'instance': instance, 'release': release, } task = Task(info) self.jobqueue.enqueue(task) log.info("In job queue %s" % build_id)
def produce_jobs(infox): """ Queue the jobs into jobqueue :args infox: list of dictionaries contains the image url and the buildid """ jobqueue = Queue('jobqueue') jobqueue.connect() session = init_model() timestamp = datetime.datetime.now() for info in infox: jd = JobDetails(taskid=info['buildid'], status='q', created_on=timestamp, user='******', last_updated=timestamp) session.add(jd) session.commit() job_details_id = jd.id log.info('Save {jd_id} to database'.format(jd_id=job_details_id)) info.update({'job_id': jd.id}) task = Task(info) jobqueue.enqueue(task) log.info('Enqueue {jd_id} to redis'.format(jd_id=job_details_id)) publish_to_fedmsg(topic='image.queued', image_url=info['image_url'], image_name=info['name'], status='queued', buildid=info['buildid'], job_id=info['job_id'], release=info['release'])
def produce_jobs(infox): """ Queue the jobs into jobqueue :args infox: list of dictionaries contains the image url and the buildid """ jobqueue = Queue('jobqueue') jobqueue.connect() session = init_model() timestamp = datetime.datetime.now() for info in infox: jd = JobDetails( taskid=info['buildid'], status='q', created_on=timestamp, user='******', last_updated=timestamp) session.add(jd) session.commit() job_details_id = jd.id log.info('Save {jd_id} to database'.format(jd_id=job_details_id)) info.update({'job_id': jd.id}) task = Task(info) jobqueue.enqueue(task) log.info('Enqueue {jd_id} to redis'.format(jd_id=job_details_id)) publish_to_fedmsg(topic='image.queued', image_url=info['image_url'], image_name=info['name'], status='queued', buildid=info['buildid'], job_id=info['job_id'], release=info['release'])
def produce_jobs(infox): """ Queue the jobs into jobqueue :args infox: list of dictionaries contains the image url and the buildid """ jobqueue = Queue('jobqueue') jobqueue.connect() family_mapping = { 'Cloud_Base': 'b', 'Atomic': 'a', 'AtomicHost': 'a', } session = init_model() timestamp = datetime.datetime.now() for info in infox: image_name = info['path'].split('/')[-1].split(info['arch'])[0] jd = ComposeJobDetails( arch=info['arch'], compose_id=info['compose']['id'], created_on=timestamp, family=family_mapping[info['subvariant']], image_url=info['absolute_path'], last_updated=timestamp, release=info['compose']['release'], status='q', subvariant=info['subvariant'], user='******', image_format=info['format'], image_type=info['type'], image_name=image_name, ) session.add(jd) session.commit() job_details_id = jd.id log.info('Save {jd_id} to database'.format(jd_id=job_details_id)) info.update({'job_id': jd.id}) task = Task(info) jobqueue.enqueue(task) log.info('Enqueue {jd_id} to redis'.format(jd_id=job_details_id)) publish_to_fedmsg(topic='image.queued', compose_url=info['absolute_path'], compose_id=info['compose']['id'], image_name=image_name, status='queued', job_id=info['job_id'], release=info['compose']['release'], family=jd.family.value, type=info['type']) session.close()
def monitor_buildqueue(): """ This function monitors the build queue. If the build is still on then it puts it back to the queue. If the build is finished then it goes to the job queue. """ key = get_key('darkbuildqueue') config = get_redis_config() jobqueue = Queue('jobqueue', config) jobqueue.connect() buildqueue = Queue('buildqueue', config) buildqueue.connect() rdb = redis_connection() if not rdb: log(key, 'redis is missing', 'error') return None rdb.set('darkbuildqueue-status', '1') while True: if check_shutdown(): break try: time.sleep(60) length = buildqueue.length if length == 0: log(key, "Sleeping, no buildqueue job", 'info') time.sleep(60) continue task = buildqueue.dequeue() kojiurl = task.data['kojiurl'] idx = task.data['jobid'] kc = koji.ClientSession(kojiurl, {'debug': False, 'password': None,\ 'debug_xmlrpc': False, 'user': None}) res = kc.getBuild(idx) if not res: #We reached to the new build yet to start #Time to sleep log(key, "build deleted %s" % idx, 'error') continue if res['state'] == 1: #completed build now push to our redis queue jobqueue.enqueue(task) log(key, "in job queue %s" % idx, 'info') continue if res['state'] == 0: #building state buildqueue.enqueue(task) log(key, "in build queue %s" % idx, 'info') continue except Exception, error: log(key, str(error), 'error')
def produce_jobs(infox): """ Queue the jobs into jobqueue :args infox: list of dictionaries contains the image url and the buildid """ jobqueue = Queue('jobqueue') jobqueue.connect() family_mapping = { 'Cloud_Base': 'b', 'Atomic': 'a' } session = init_model() timestamp = datetime.datetime.now() for info in infox: image_name = info['path'].split('.x86_64')[0].split('/')[-1] jd = ComposeJobDetails( arch=info['arch'], compose_id=info['compose']['id'], created_on=timestamp, family=family_mapping[info['subvariant']], image_url=info['absolute_path'], last_updated=timestamp, release=info['compose']['release'], status='q', subvariant=info['subvariant'], user='******', image_format=info['format'], image_type=info['type'], image_name=image_name, ) session.add(jd) session.commit() job_details_id = jd.id log.info('Save {jd_id} to database'.format(jd_id=job_details_id)) info.update({'job_id': jd.id}) task = Task(info) jobqueue.enqueue(task) log.info('Enqueue {jd_id} to redis'.format(jd_id=job_details_id)) publish_to_fedmsg(topic='image.queued', compose_url=info['absolute_path'], compose_id=info['compose']['id'], image_name=image_name, status='queued', job_id=info['job_id'], release=info['compose']['release'], family=jd.family.value, type=info['type']) session.close()
def update(data): ''' Updates the git repo for the given user :arg user: github username :arg repo: Code repo name ''' queue = Queue('puluupdates') if not queue.connect(): return task = Task(data=data, raw=True) queue.enqueue(task)
def upload_file(): if request.method == "POST": file = request.files["file"] if file and allowed_file(file.filename): filename = secure_filename(file.filename) file.save(os.path.join(APP.config["UPLOAD_FOLDER"], filename)) # Now add the information in the queue for processing t = Task({"filename": filename}) queue = Queue("incoming_files") queue.connect() queue.enqueue(t) return "Log uploaded successfully." return """
class LogBot(irc.IRCClient): """A logging IRC bot.""" nickname = 'pyconsprints' def __init__(self, channel): self.chn = '#'+channel self.qs_queue = [] self.logger = None self.q = Queue('bug-messages') self.q.connect() self.channel_admin = ['kushal',] def connectionMade(self): irc.IRCClient.connectionMade(self) self.islogging = False self._namescallback = {} def connectionLost(self, reason): irc.IRCClient.connectionLost(self, reason) self.islogging = False def signedOn(self): """Called when bot has succesfully signed on to server.""" self.join(self.factory.channel) def privmsg(self, user, channel, msg): """This will get called when the bot receives a message.""" user = user.split('!', 1)[0] if user == BOTNAME: print '[[%s]]' % msg task = Task(msg) self.q.enqueue(task) user_cond = user in self.channel_admin if msg == '#masters' and user_cond: self.msg(self.chn, "My current masters are: %s" % ",".join(self.channel_admin)) if msg.startswith('#add:') and user_cond: try: name = msg.split()[1] print name self.channel_admin.append(name) self.msg(self.chn,'%s is a master now.' % name) except Exception, err: print err if msg.startswith('#test:') and user_cond: bugid = msg.split()[1] msg = 'dummy/issue%s' % bugid task = Task(msg) self.q.enqueue(task)
def compile(request): """ Enqueue the task to Queue """ filename = request.POST.get('filename', False) text = request.POST.get('text', False) if filename is False: return HttpResponse(json.dumps({'output':'Invalid filename'}), content_type="application/json") if text is False: return HttpResponse(json.dumps({'output':'Empty file'}), content_type="application/json") try: queue = Queue('rcc') queue.connect() task = Task({'filename':filename, 'text':text}) job = queue.enqueue(task) except: return HttpResponse(json.dumps({'output':'Error creating Job'}), content_type="application/json") while True: if job.result is None: continue break return HttpResponse(json.dumps({'output' : job.result}), content_type="application/json")
class RedisQueue(object): def __init__(self, host, name, port=6379, password=None): self.super_queue = Queue( name, { 'host': host, 'port': port, 'db': 0, 'password': password, }) self.super_queue.connect() def get(self): return self.super_queue.wait() def put(self, data): self.super_queue.enqueue(Task(data))
def compile(request): """ Enqueue the task to Queue """ filename = request.POST.get('filename', False) text = request.POST.get('text', False) if filename is False: return HttpResponse(json.dumps({'error':'Invalid filename'})) if text is False: return HttpResponse(json.dumps({'error':'Empty file'})) try: queue = Queue('rcc') queue.connect() task = Task({'filename':filename, 'text':text}) job = queue.enqueue(task) except: return HttpResponse(json.dumps({'error':'Error creating Job'})) return HttpResponse(json.dumps({'status':'Job Created'}))
#!/usr/bin/python # coding: utf-8 NUM_QUEUES = 2 import sys sys.path.append("/usr/share/copr/") from retask.task import Task from retask.queue import Queue from backend.helpers import BackendConfigReader opts = BackendConfigReader().read() redis_config = { 'host': opts['redis_host'], 'port': opts['redis_port'], 'db': opts['redis_db'], } for i in range(0, NUM_QUEUES): print("## Queue {}".format(i)) q = Queue("copr-be-{}".format(i), config=redis_config) q.connect() save_q = [] while q.length != 0: task = q.dequeue() print task.data save_q.append(task) for t in save_q: q.enqueue(t)
class Channel(object): """ Abstraction above retask (the set of "channels" between backend(s), jobgrabber and workers). We could use multiple backends and/or diffferent "atomic" medium (other implemntation than Queue) in future. But make sure nobody needs to touch the "medium" directly. """ def __init__(self, opts, log=None): self.log = log self.opts = opts # channel for Backend <--> JobGrabber communication self.jg_start = Queue("jg_control_start") # channel for JobGrabber <--> [[Builders]] communication self.build_queues = dict() while not self.jg_start.connect(): wait_log(self.log, "waiting for redis", 5) def _get_queue(self, bgroup): if not bgroup in self.build_queues: q_id = "copr-be-{0}".format(bgroup) q = Queue(q_id) if not q.connect(): # As we already connected to jg_control_message, this should # be also OK. raise Exception("can't connect to redis, should never happen!") return q return self.build_queues[bgroup] def add_build(self, bgroup, build): """ this should be used by job_grab only for now """ q = self._get_queue(bgroup) try: q.enqueue(Task(build)) except Exception as err: # I've seen isses Task() was not able to jsonify urllib exceptions if not self.log: return False self.log.error("can't enqueue build {0}, reason:\n{1}".format( build, err )) return True # Builder's API def get_build(self, bgroup): """ Return task from queue or return 0 """ q = self._get_queue(bgroup) t = q.dequeue() return t.data if t else None # JobGrab's API def backend_started(self): return self.jg_start.length def job_graber_initialized(self): while self.jg_start.dequeue(): pass def remove_all_builds(self): for bgroup in self.build_queues: q = self._get_queue(bgroup) while q.dequeue(): pass self.build_queues = dict() # Backend's API def backend_start(self): """ Notify jobgrab about service start. """ if not self.jg_start.enqueue(Task("start")): raise Exception("can't append to retask queue, should never happen!") while self.jg_start.length: wait_log(self.log, "waiting until jobgrabber initializes queue")
from retask.task import Task from retask.queue import Queue queue = Queue('example') info1 = {'user':'******', 'url':'http://kushaldas.in'} info2 = {'user':'******', 'url':'http://planet.fedoraproject.org'} task1 = Task(info1) task2 = Task(info2) queue.connect() queue.enqueue(task1) queue.enqueue(task2)
def produce_jobs(idx): key = get_key('darkproducer') log(key, "starting with %s" % str(idx), 'info') kojiurl = 'http://koji.fedoraproject.org/' kojiurl2 = kojiurl + 'kojihub' kc = koji.ClientSession(kojiurl2, {'debug': False, 'password': None,\ 'debug_xmlrpc': False, 'user': None}) config = get_redis_config() jobqueue = Queue('jobqueue', config) jobqueue.connect() buildqueue = Queue('buildqueue', config) buildqueue.connect() #lastbuild = {'id':None, 'time':None} rdb = redis_connection() if not rdb: log(key, 'redis is missing', 'error') return None rdb.set('darkproducer-id', idx) while True: if check_shutdown(): break try: rdb.set('darkproducer-status', '1') idx = int(rdb.get('darkproducer-id')) utils.msgtext = "ID: %s" % idx res = kc.getBuild(idx) url = kojiurl + 'koji/buildinfo?buildID=%s' % idx if not res: #FIXME!! #http://koji.fedoraproject.org/koji/buildinfo?buildID=367374 #missing build from db :( #if lastbuild['id'] != idx: # lastbuild['id'] = idx # lastbuild['time'] = time.time.now() #else: # diff = time.time.now() - lastbuild['time'] # if diff > 300: #We have a buildid stuck, raise alarm #We reached to the new build yet to start #Time to sleep log(key, "Sleeping with %s" % idx, 'info') time.sleep(60) continue if res['state'] == 1: # completed build now push to our redis queue info = {'url': url, 'jobid': idx} task = Task(info) jobqueue.enqueue(task) log(key, "In job queue %s" % idx, 'info') rdb.incr('darkproducer-id') continue if res['state'] == 0: #building state info = {'url': url, 'jobid': idx, 'kojiurl': kojiurl2} task = Task(info) buildqueue.enqueue(task) log(key, "In build queue %s" % idx, 'info') rdb.incr('darkproducer-id') continue else: rdb.incr('darkproducer-id') except Exception, error: log(key, str(error), 'error')
def mail_update(rpage): 'Send a message for each update.' q = Queue('wikiupdate') q.connect() q.enqueue(Task(rpage))
# coding: utf-8 NUM_QUEUES = 2 import sys sys.path.append("/usr/share/copr/") from retask.task import Task from retask.queue import Queue from backend.helpers import BackendConfigReader opts = BackendConfigReader().read() redis_config = { 'host': opts['redis_host'], 'port': opts['redis_port'], 'db': opts['redis_db'], } for i in range(0, NUM_QUEUES): print("## Queue {}".format(i)) q = Queue("copr-be-{}".format(i), config=redis_config) q.connect() save_q = [] while q.length != 0: task = q.dequeue() print task.data save_q.append(task) for t in save_q: q.enqueue(t)
from retask.task import Task from retask.queue import Queue queue = Queue('example') info1 = {'user': '******', 'url': 'http://planet.fedoraproject.org'} task1 = Task(info1) queue.connect() job = queue.enqueue(task1) job.wait() print job.result
def runTest(self): queue = Queue('testqueue') queue.connect() t = Task({'name':'kushal'}) self.assertTrue(queue.enqueue(t)[0])
def setUp(self): queue = Queue('testqueue') queue.connect() t = Task({'name':'kushal'}) queue.enqueue(t)