def generate_configs(args, job_name, host_id, instance_id): hosts = args.kafka_config.jobs[job_name].hosts task_id = deploy_utils.get_task_id(hosts, host_id, instance_id) kafka_cfg_dict = args.kafka_config.configuration.generated_files["kafka.cfg"] kafka_cfg_dict["broker.id"] = task_id kafka_cfg = deploy_utils.generate_properties_file(args, kafka_cfg_dict) config_files = { "kafka.cfg": kafka_cfg, } config_files.update(args.kafka_config.configuration.raw_files) return config_files
def update_active_tasks(self): # Add all active tasks self.metric_sources = [] for service_name, service in self.collector_config.services.iteritems(): # Save to database. # The active field has the default value True. service_record, created = Service.objects.get_or_create( name=service_name, defaults={"metric_url":service.metric_url}) if not created: # Mark it as active if it exists. service_record.active = True service_record.save() for cluster_name, cluster in service.clusters.iteritems(): cluster_record, created = Cluster.objects.get_or_create( service=service_record, name=cluster_name) if not created: cluster_record.active = True cluster_record.save() for job_name in service.jobs: job_record, created = Job.objects.get_or_create( cluster=cluster_record, name=job_name) if not created: job_record.active = True job_record.save() job = cluster.jobs[job_name] # We assume http port is always base_port + 1 port = job.base_port + 1 # support multiple instances hosts = job.hosts for host_id, host in hosts.iteritems(): host_name = job.hostnames[host_id] for instance_id in range(host.instance_num): task_id = deploy_utils.get_task_id(hosts, host_id, instance_id) instance_port = deploy_utils.get_base_port(port,instance_id) task_record, created = Task.objects.get_or_create( job=job_record, task_id=task_id, defaults={"host":host_name, "port":instance_port}) if not created or task_record.host != host_name or ( task_record.port != instance_port): task_record.active = True task_record.host = host_name task_record.port = instance_port task_record.save() self.metric_sources.append( MetricSource(self.collector_config, task_record))
def generate_bootstrap_script(args, host, job_name, host_id, instance_id): supervisor_client = deploy_utils.get_supervisor_client(host, "zookeeper", args.zookeeper_config.cluster.name, job_name, instance_id=instance_id) data_dir = supervisor_client.get_available_data_dirs()[0] myid_file = "%s/%s" % (data_dir, MYID_FILE) hosts = args.zookeeper_config.jobs["zookeeper"].hosts task_id = deploy_utils.get_task_id(hosts, host_id, instance_id) script_dict = { 'myid_file': myid_file, 'host_id': task_id, } return deploy_utils.create_run_script( '%s/bootstrap_zk.sh.tmpl' % deploy_utils.get_template_dir(), script_dict)
def generate_configs(args, job_name, host_id, instance_id): kafka_cfg_dict = args.kafka_config.configuration.generated_files["kafka.cfg"] hosts = args.kafka_config.jobs[job_name].hosts kafka_cfg_dict["broker.id"] = deploy_utils.get_task_id(hosts, host_id, instance_id) kafka_cfg = deploy_utils.generate_properties_file(args, kafka_cfg_dict) kafka_scribe_cfg_dict = args.kafka_config.configuration.generated_files["kafka-scribe.cfg"] kafka_job = args.kafka_config.jobs["kafka"] kafka_scribe_cfg_dict["metadata.broker.list"] = ",".join( service_config.get_job_host_port_list(kafka_job)) kafka_scribe_cfg = deploy_utils.generate_properties_file(args, kafka_scribe_cfg_dict) config_files = { "kafka.cfg": kafka_cfg, "kafka-scribe.cfg": kafka_scribe_cfg, } config_files.update(args.kafka_config.configuration.raw_files) return config_files
def generate_configs(args, job_name, host_id, instance_id): kafka_cfg_dict = args.kafka_config.configuration.generated_files[ "kafka.cfg"] hosts = args.kafka_config.jobs[job_name].hosts kafka_cfg_dict["broker.id"] = deploy_utils.get_task_id( hosts, host_id, instance_id) kafka_cfg = deploy_utils.generate_properties_file(args, kafka_cfg_dict) kafka_scribe_cfg_dict = args.kafka_config.configuration.generated_files[ "kafka-scribe.cfg"] kafka_job = args.kafka_config.jobs["kafka"] kafka_scribe_cfg_dict["metadata.broker.list"] = ",".join( service_config.get_job_host_port_list(kafka_job)) kafka_scribe_cfg = deploy_utils.generate_properties_file( args, kafka_scribe_cfg_dict) config_files = { "kafka.cfg": kafka_cfg, "kafka-scribe.cfg": kafka_scribe_cfg, } config_files.update(args.kafka_config.configuration.raw_files) return config_files