def republish_job(self, buildreq): if isinstance(buildreq, BuildRequest): routing_key = pybit.get_build_route_name(buildreq.get_dist(), buildreq.get_arch(), buildreq.get_suite(), buildreq.get_format()) try: msg = jsonpickle.encode(buildreq) self.message_chan.basic_publish(amqp.Message(msg), exchange=pybit.exchange_name, routing_key=routing_key, mandatory=True) except amqp.AMQPConnectionException as e: logging.debug("Couldn't connect to channel. traceback: %s" % e)
def process_job(self, dist, architectures, version, name, suite, pkg_format, transport): try: build_arches = self.process_achitectures(architectures, suite) if len(build_arches) == 0: response.status = "404 - no build architectures for this suite." return except Exception as e: raise Exception("Error parsing arch information: " + str(e)) response.status = "500 - Error parsing arch information" return try: current_package = self.process_package(name, version) if not current_package.id: return current_suite = myDb.get_suite_byname(suite)[0] current_dist = myDb.get_dist_byname(dist)[0] current_format = myDb.get_format_byname(pkg_format)[0] # for arch in supported_arches: for arch in build_arches: current_arch = myDb.get_arch_byname(arch)[0] current_packageinstance = self.process_packageinstance( current_arch, current_package, current_dist, current_format, current_suite ) if current_packageinstance.id: new_job = myDb.put_job(current_packageinstance, None) print "CREATED NEW JOB ID", new_job.id if new_job.id: self.cancel_superceded_jobs(new_job) build_req = jsonpickle.encode(BuildRequest(new_job, transport, self.settings["webserver_url"])) msg = amqp.Message(build_req) msg.properties["delivery_mode"] = 2 routing_key = pybit.get_build_route_name( new_job.packageinstance.distribution.name, new_job.packageinstance.arch.name, new_job.packageinstance.suite.name, new_job.packageinstance.format.name, ) print "SENDING BUILD REQUEST FOR JOB ID", new_job.id, new_job.packageinstance.package.name, new_job.packageinstance.package.version, new_job.packageinstance.distribution.name, new_job.packageinstance.arch.name, new_job.packageinstance.suite.name, new_job.packageinstance.format.name # print "\n____________SENDING", build_req, "____________TO____________", routing_key self.chan.basic_publish(msg, exchange=pybit.exchange_name, routing_key=routing_key) else: print "FAILED TO ADD JOB" response.status = "404 - failed to add job." else: print "PACKAGE INSTANCE ERROR" response.status = "404 - failed to add/retrieve package instance." except Exception as e: raise Exception("Error submitting job: " + str(e)) response.status = "500 - Error submitting job" return return
def __init__(self, arch, distribution, pkg_format, suites, conn_info, settings) : self.state_table = {} self.state_table["UNKNOWN"] = self.fatal_error_handler self.state_table["IDLE"] = self.idle_handler self.state_table["FATAL_ERROR"] = self.fatal_error_handler self.state_table["CHECKOUT"] = self.checkout_handler self.state_table["BUILD"] = self.build_handler self.state_table["UPLOAD"] = self.upload_handler self.state_table["CLEAN"] = self.clean_handler self.overall_success = None self.state = "UNKNOWN" self.arch = arch self.distribution = distribution self.pkg_format = pkg_format self.listen_list = dict() self.conn = None self.command_chan = None self.message_chan = None self.settings = settings self.poll_time = 60 self.distros = {} self.handlers = {} if 'poll_time' in self.settings: self.poll_time = self.settings['poll_time'] for suite in suites: route = pybit.get_build_route_name(self.distribution, self.arch, suite, self.pkg_format) queue = pybit.get_build_queue_name(self.distribution, self.arch, suite, self.pkg_format) self.listen_list[suite] = { 'route': route, 'queue': queue} self.plugin_handler () self.conn_info = conn_info if (self.distribution in self.distros): self.format_handler = self.distros[self.distribution] logging.info ("Using %s build client" % self.distribution) elif (self.pkg_format == "deb" and "Debian" in self.distros) : self.format_handler = self.distros['Debian'] logging.warning ("Using default Debian build client for %s package format" % self.pkg_format) else: logging.debug ("Empty build client") self.format_handler = None self._clean_current() self.move_state("IDLE")
def __init__(self, arch, distribution, pkg_format, suite, conn_info, settings) : self.state_table = {} self.state_table["UNKNOWN"] = self.fatal_error_handler self.state_table["IDLE"] = self.idle_handler self.state_table["FATAL_ERROR"] = self.fatal_error_handler self.state_table["CHECKOUT"] = self.checkout_handler self.state_table["BUILD"] = self.build_handler self.state_table["UPLOAD"] = self.upload_handler self.state_table["CLEAN"] = self.clean_handler self.overall_success = None self.state = "UNKNOWN" self.arch = arch self.distribution = distribution self.pkg_format = pkg_format self.suite = suite self.conn = None self.command_chan = None self.message_chan = None self.settings = settings self.routing_key = pybit.get_build_route_name(self.distribution, self.arch, self.suite, self.pkg_format) self.queue_name = pybit.get_build_queue_name(self.distribution, self.arch, self.suite, self.pkg_format) self.conn_info = conn_info #self.message_chan.basic_consume(queue=self.queue_name, no_ack=False, callback=self.message_handler, consumer_tag="build_callback") #self.command_chan.basic_consume(queue=self.client_queue_name, no_ack=True, callback=self.command_handler, consumer_tag="cmd_callback") if (pkg_format == "deb") : self.format_handler = DebianBuildClient(self.settings) else: print "Empty build client" self.format_handler = None self._clean_current() self.move_state("IDLE")
def process_job(self, dist, architectures, version, name, suite, pkg_format, transport, build_environment = None) : try: # Look at blacklist, dont build excluded package names if self.db.check_blacklist("name",name): return False # Look at blacklist, dont build packages from SVN paths which match the blacklist rule. if self.db.check_blacklist("vcs_uri",transport.uri): return False except Exception as e: print(("Exception checking blacklist " + str(e))) return False try: current_package = self.process_package(name, version) if not current_package.id : return False current_suite = self.db.get_suite_byname(suite)[0] current_dist = self.db.get_dist_byname(dist)[0] current_format = self.db.get_format_byname(pkg_format)[0] build_env_suite_arch = self.process_build_environment_architectures(current_suite,architectures,build_environment) if len(build_env_suite_arch) == 0: self.log.warn("INVALID BUILD ENV SUITE ARCH MAPPINGS FOR %s, %s, %s - CHECK YOU HAVE SOME CONFIGURED.", current_suite,architectures,build_environment) response.status = "500 - Error submitting job" return else: current_build_env = build_env_suite_arch[0].buildenv master_flag = True chan = self.get_amqp_channel() for build_env_suite_arch in build_env_suite_arch : current_arch = build_env_suite_arch.suitearch.arch if current_build_env and current_build_env.name != build_env_suite_arch.get_buildenv_name() : #FIXME #first packageinstance for each build environment should have master flag set master_flag = True current_build_env = build_env_suite_arch.buildenv current_packageinstance = self.process_packageinstance(current_build_env, current_arch, current_package, current_dist, current_format, current_suite, master_flag) if current_packageinstance.id : new_job = self.db.put_job(current_packageinstance,None) if (new_job and new_job.id): # self.log.debug("\nCREATED NEW JOB: %s\n", jsonpickle.encode(new_job)) self.cancel_superceded_jobs(new_job) # NEW STUFF FOR RESUBMITTING JOBS build_request_obj = BuildRequest(new_job,transport, "%s:%s" % (self.settings['web']['hostname'], self.settings['web']['port'])); build_req = jsonpickle.encode(build_request_obj) self.db.log_buildRequest(build_request_obj) #print "SENDING REQUEST WITH DATA", str(build_req) msg = amqp.Message(build_req) msg.properties["delivery_mode"] = 2 routing_key = pybit.get_build_route_name(new_job.packageinstance.get_distribution_name(), new_job.packageinstance.get_arch_name(), new_job.packageinstance.get_suite_name(), new_job.packageinstance.get_format_name()) build_queue = pybit.get_build_queue_name(new_job.packageinstance.get_distribution_name(), new_job.packageinstance.get_arch_name(), new_job.packageinstance.get_suite_name(), new_job.packageinstance.get_format_name()) self.add_message_queue(build_queue, routing_key, chan) chan.basic_publish(msg,exchange=pybit.exchange_name,routing_key=routing_key,mandatory=True) #self.log.debug("\n____________SENDING %s ____________TO____________ %s", build_req, routing_key) self.log.debug("SENDING BUILD REQUEST FOR JOB ID %i %s %s %s %s %s %s", new_job.id, new_job.packageinstance.get_distribution_name(), new_job.packageinstance.get_package_version(), new_job.packageinstance.get_distribution_name(), new_job.packageinstance.get_arch_name(), new_job.packageinstance.get_suite_name(), new_job.packageinstance.get_format_name()) else : self.log.warn("FAILED TO ADD JOB") response.status = "404 - failed to add job." return False master_flag = False else : self.log.warn("PACKAGE INSTANCE ERROR") response.status = "404 - failed to add/retrieve package instance." return False except Exception as e: raise Exception('Error submitting job: ' + str(e)) response.status = "500 - Error submitting job" return False return True # success
def process_job(self, dist, architectures, version, name, suite, pkg_format, transport, build_environment=None): """Overridden to return the build request ID""" current_package = self.process_package(name, version) if not current_package.id: raise Exception("Huh?") ##return False current_suite = self.db.get_suite_byname(suite)[0] current_dist = self.db.get_dist_byname(dist)[0] current_format = self.db.get_format_byname(pkg_format)[0] # FIXME The use of 'any' is evil. Let's kill this behavior. build_env_suite_arch = self.process_build_environment_architectures( current_suite, architectures, build_environment ) if len(build_env_suite_arch) == 0: raise Exception("Unable to submit job to PyBit") else: current_build_env = build_env_suite_arch[0].buildenv master_flag = True chan = self.get_amqp_channel() jobs = [] for build_env_suite_arch in build_env_suite_arch: current_arch = build_env_suite_arch.suitearch.arch if current_build_env and current_build_env.name != build_env_suite_arch.get_buildenv_name(): # FIXME # first packageinstance for each build environment should have master flag set master_flag = True current_build_env = build_env_suite_arch.buildenv current_packageinstance = self.process_packageinstance( current_build_env, current_arch, current_package, current_dist, current_format, current_suite, master_flag, ) if current_packageinstance.id is None: raise Exception("Package instance not found...") new_job = self.db.put_job(current_packageinstance, None) if new_job is None: raise Exception("Job not created...") self.cancel_superceded_jobs(new_job) build_request_obj = BuildRequest( new_job, transport, "%s:%s" % (self.settings["web"]["hostname"], self.settings["web"]["port"]) ) build_req = jsonpickle.encode(build_request_obj) self.db.log_buildRequest(build_request_obj) msg = amqp.Message(build_req) msg.properties["delivery_mode"] = 2 routing_key = pybit.get_build_route_name( new_job.packageinstance.get_distribution_name(), new_job.packageinstance.get_arch_name(), new_job.packageinstance.get_suite_name(), new_job.packageinstance.get_format_name(), ) build_queue = pybit.get_build_queue_name( new_job.packageinstance.get_distribution_name(), new_job.packageinstance.get_arch_name(), new_job.packageinstance.get_suite_name(), new_job.packageinstance.get_format_name(), ) self.add_message_queue(build_queue, routing_key, chan) chan.basic_publish(msg, exchange=pybit.exchange_name, routing_key=routing_key, mandatory=True) jobs.append(new_job.id) return jobs
def process_job(self, dist, architectures, version, name, suite, pkg_format, transport, commands = None) : try: # Look at blacklist, dont build excluded package names if self.db.check_blacklist("name",name): return False # Look at blacklist, dont build packages from SVN paths which match the blacklist rule. if self.db.check_blacklist("vcs_uri",transport.uri): return False except Exception as e: print "Exception checking blacklist " + str(e) return False try: build_arches = self.process_achitectures(architectures, suite) if (len(build_arches) == 0): response.status = "404 - no build architectures for this suite." return except Exception as e: raise Exception('Error parsing arch information: ' + str(e)) response.status = "500 - Error parsing arch information" return try: current_package = self.process_package(name, version) if not current_package.id : return current_suite = self.db.get_suite_byname(suite)[0] current_dist = self.db.get_dist_byname(dist)[0] current_format = self.db.get_format_byname(pkg_format)[0] master = True #for arch in supported_arches: chan = self.get_amqp_channel() for arch in build_arches: current_arch = self.db.get_arch_byname(arch)[0] current_packageinstance = self.process_packageinstance(current_arch, current_package, current_dist, current_format, current_suite, master) if current_packageinstance.id : new_job = self.db.put_job(current_packageinstance,None) print "CREATED NEW JOB ID", new_job.id if new_job.id : self.cancel_superceded_jobs(new_job) build_req = jsonpickle.encode(BuildRequest(new_job,transport, "%s:%s" % (self.settings['web']['hostname'], self.settings['web']['port']),commands)) msg = amqp.Message(build_req) msg.properties["delivery_mode"] = 2 routing_key = pybit.get_build_route_name(new_job.packageinstance.distribution.name, new_job.packageinstance.arch.name, new_job.packageinstance.suite.name, new_job.packageinstance.format.name) build_queue = pybit.get_build_queue_name(new_job.packageinstance.distribution.name, new_job.packageinstance.arch.name, new_job.packageinstance.suite.name, new_job.packageinstance.format.name) self.add_message_queue(build_queue, routing_key, chan) if chan.basic_publish(msg,exchange=pybit.exchange_name,routing_key=routing_key,mandatory=True) : #print "\n____________SENDING", build_req, "____________TO____________", routing_key print "SENDING BUILD REQUEST FOR JOB ID", new_job.id, new_job.packageinstance.package.name, new_job.packageinstance.package.version, new_job.packageinstance.distribution.name, new_job.packageinstance.arch.name, new_job.packageinstance.suite.name, new_job.packageinstance.format.name else : print "UNABLE TO ROUTE BUILD REQUEST TO", routing_key else : print "FAILED TO ADD JOB" response.status = "404 - failed to add job." master = False else : print "PACKAGE INSTANCE ERROR" response.status = "404 - failed to add/retrieve package instance." except Exception as e: raise Exception('Error submitting job: ' + str(e)) response.status = "500 - Error submitting job" return return