def setup_agent4(hostname=None, domain=None, pc="1", agent_conf="files/puppet-agent.conf", puppetserver=None, proxy_url=None, hosts_file=None):
"""Setup Puppet 4 agent"""
import package, util, config
if not hostname:
hostname = util.get_hostname()
if not domain:
domain = util.get_domain()
install_puppetlabs_release_package(pc, proxy_url=proxy_url)
package.install("puppet-agent")
# Use puppetserver value from setting.ini file if none is given on the
# command-line. If that fails use the default.
if not puppetserver:
try: puppetserver = config.get("puppet", "puppetserver")
except: puppetserver = None
# Add a customized puppet.conf
util.put_and_chown(agent_conf, "/etc/puppetlabs/puppet/puppet.conf")
if puppetserver: server = puppetserver
else: server = "puppet.%s" % domain
sudo("puppet config set --section agent server %s" % server)
util.set_hostname(hostname + "." + domain)
util.add_host_entry(util.get_ip(), hostname, domain)
# Optionally add hosts from a separate file. This is useful when the IP of
# the puppetmaster as seen from the Puppet agent node does not match its
# name in DNS.
util.add_host_entries(hosts_file)
util.add_to_path("/opt/puppetlabs/bin")
run_agent(noop="True", onlychanges="False")
def setup_agent5(hostname=None, domain=None, pc="1", agent_conf="files/puppet-agent5.conf", puppetserver=None,
proxy_url=None, hosts_file=None):
"""Setup Puppet 5 agent"""
import package, util, config
if not hostname:
hostname = util.get_hostname()
if not domain:
domain = util.get_domain()
install_puppetlabs_release_package(pc, proxy_url=proxy_url)
package.install("puppet-agent")
# Use puppetserver value from setting.ini file if none is given on the command-line. If that fails use the default.
if not puppetserver:
try: puppetserver = config.get("puppet", "puppetserver")
except: puppetserver = None
# Add a customized puppet.conf
util.put_and_chown(agent_conf, "/etc/puppetlabs/puppet/puppet.conf")
if puppetserver: server = puppetserver
else: server = "puppet.%s" % domain
sudo("puppet config set --section agent server %s" % server)
util.set_hostname(hostname + "." + domain)
util.add_host_entry(util.get_ip(), hostname, domain)
# Optionally add hosts from a separate file. This is useful when the IP of the puppetmaster as seen from the Puppet agent
# node does not match its name in DNS.
util.add_host_entries(hosts_file)
util.add_to_path("/opt/puppetlabs/bin")
run_agent(noop="False", onlychanges="False")
def setup_agent4(hostname=None,
domain=None,
pc="1",
agent_conf="files/puppet-agent.conf",
proxy_url=None,
hosts_file=None):
"""Setup Puppet 4 agent"""
import package, util
if not hostname:
hostname = util.get_hostname()
if not domain:
domain = util.get_domain()
install_puppetlabs_release_package(pc, proxy_url=proxy_url)
package.install("puppet-agent")
util.put_and_chown(agent_conf, "/etc/puppetlabs/puppet/puppet.conf")
util.set_hostname(hostname + "." + domain)
util.add_host_entry("127.0.1.1", hostname, domain)
# Optionally add hosts from a separate file. This is useful when the IP of
# the puppetmaster does not match its name in DNS.
util.add_host_entries(hosts_file)
util.add_to_path("/opt/puppetlabs/bin")
run_agent(noop="True", onlychanges="False")
def setup_agent4(hostname=None, domain=None, pc="1", agent_conf="files/puppet-agent.conf", proxy_url=None, hosts_file=None):
"""Setup Puppet 4 agent"""
import package, util
if not hostname:
hostname = util.get_hostname()
if not domain:
domain = util.get_domain()
install_puppetlabs_release_package(pc, proxy_url=proxy_url)
package.install("puppet-agent")
util.put_and_chown(agent_conf, "/etc/puppetlabs/puppet/puppet.conf")
util.set_hostname(hostname + "." + domain)
util.add_host_entry("127.0.1.1", hostname, domain)
# Optionally add hosts from a separate file. This is useful when the IP of
# the puppetmaster does not match its name in DNS.
util.add_host_entries(hosts_file)
util.add_to_path("/opt/puppetlabs/bin")
run_agent(noop="True", onlychanges="False")
def gff_to_jbrowse(self,
gff_file,
index_html,
data_dir_out,
jbrowse_url=None
):
opt = self.opt
if jbrowse_url is None:
jbrowse_url = opt["jbrowse_url"]
env = None
if self.opt.get("jbrowse_bin_dir",None):
env = os.environ.copy()
util.add_to_path(opt["jbrowse_bin_dir"],
prepend=True,
env=env)
if not os.path.exists(data_dir_out):
os.makedirs(data_dir_out)
gff_file = util.abspath(gff_file)
#fasta_file = util.abspath(fasta_file)
jbrowse_out_dir = os.path.join(data_dir_out,opt["jbrowse_data_subdir"])
#can use os.devnull to discard all output
jbrowse_conv_log_base = os.path.join(os.getcwd(),"jbrowse_conv_log")
with nested( open(jbrowse_conv_log_base+".out","w"),\
open(jbrowse_conv_log_base+".err","w") ) as ( stdout, stderr ):
check_call(["prepare-refseqs.pl","--gff",gff_file,"--out",jbrowse_out_dir],
env=env,
stdout=stdout,
stderr=stderr)
#@todo use biodb-to-json instead with flat file input, and accept config
#file as a parameter (provide a default one too). See volvox.json config
#in the distribution. Also add dropped_features param to load everything
#unique in field 3 of GFF and check that only dropped_features are missing
#from the config
check_call(["flatfile-to-json.pl","--gff",gff_file,"--out",jbrowse_out_dir,
"--trackLabel","Genes",
"--cssClass","feature5",
"--type","gene",
"--autocomplete","all"
"--getLabel",
"--getType"
],
env=env,
stdout=stdout,
stderr=stderr)
check_call(["flatfile-to-json.pl","--gff",gff_file,"--out",jbrowse_out_dir,
"--trackLabel","CDS",
"--cssClass","generic_parent",
"--subfeatureClasses",'{ "exon" : "exon" }',
"--type","CDS",
"--type","exon",
"--autocomplete","all"
"--getLabel",
"--getType",
"--getSubs",
"--getPhase"
],
env=env,
stdout=stdout,
stderr=stderr)
check_call(["flatfile-to-json.pl","--gff",gff_file,"--out",jbrowse_out_dir,
"--trackLabel","Peptides",
"--cssClass","est",
"--subfeatureClasses",'{ "mat_peptide" : "transcript-CDS" }',
"--type","mat_peptide",
"--autocomplete","all"
"--getLabel",
"--getType",
"--getSubs",
"--getPhase"
],
env=env,
stdout=stdout,
stderr=stderr)
check_call(["flatfile-to-json.pl","--gff",gff_file,"--out",jbrowse_out_dir,
"--trackLabel","Misc",
"--cssClass","feature3",
"--type","misc_feature",
"--autocomplete","all"
"--getLabel",
"--getType",
"--getSubs",
"--getPhase"
],
env=env,
stdout=stdout,
stderr=stderr)
check_call(["generate-names.pl","--out",jbrowse_out_dir],
env=env,
stdout=stdout,
stderr=stderr)
tracks_conf_file = os.path.join(jbrowse_out_dir,"trackList.json")
tracks_conf = util.load_config_json(tracks_conf_file)
tracks_conf["refSeqDropdown"] = True #show pull-down even for very many sequences
util.save_config_json(tracks_conf,tracks_conf_file)
#create index.html that redirects to JBrowse index.html with correct data param etc
_jbrowse_dataset_index_html = \
config.get_data_string(self.opt["jbrowse_galaxy_index_html_tpl"],
"galaxy.index.html")
jbrowse_url_params = util.to_url_params(dict(
tracks=",".join(("DNA","Genes","CDS","Peptides","Misc")),
tracklist=0
))
with open(index_html,"w") as f:
f.write(_jbrowse_dataset_index_html.\
format(jbrowse_url=jbrowse_url.rstrip("/"),
jbrowse_data_subdir=opt["jbrowse_data_subdir"],
jbrowse_url_params=jbrowse_url_params))
def setup_server4(hostname=None, domain=None, pc="1", forge_modules=["puppetlabs/stdlib", "puppetlabs/concat", "puppetlabs/firewall", "puppetlabs/apt"]):
"""Setup Puppet 4 server"""
import package, util, git, service
# Local files to copy over
basedir = "/etc/puppetlabs"
local_master_conf = "files/puppet-master.conf"
remote_master_conf = basedir+"/puppet/puppet.conf"
local_hiera_yaml = "files/hiera.yaml"
remote_hiera_yaml = basedir+"/code/hiera.yaml"
local_fileserver_conf = "files/fileserver.conf"
remote_fileserver_conf = basedir+"/puppet/fileserver.conf"
local_environments = "files/environments"
remote_codedir = basedir+"/code"
local_gitignore = "files/gitignore"
remote_gitignore = basedir+"/.gitignore"
modules_dir = basedir+"/code/environments/production/modules"
# Verify that all the local files are in place
try:
open(local_master_conf)
open(local_hiera_yaml)
except IOError:
print "ERROR: some local config files were missing!"
sys.exit(1)
# Autodetect hostname and domain from env.host, if they're not overridden
# with method parameters
if not hostname:
hostname = util.get_hostname()
if not domain:
domain = util.get_domain()
# Ensure that clock is correct before doing anything else, like creating SSL
# certificates.
util.set_clock()
# Start the install
install_puppetlabs_release_package(pc)
package.install("puppetserver")
util.put_and_chown(local_master_conf, remote_master_conf)
util.put_and_chown(local_hiera_yaml, remote_hiera_yaml)
util.put_and_chown(local_fileserver_conf, remote_fileserver_conf)
util.put_and_chown(local_gitignore, remote_gitignore)
util.add_to_path("/opt/puppetlabs/bin")
util.set_hostname(hostname + "." + domain)
# "facter fqdn" return a silly name on EC2 without this
util.add_host_entry("127.0.1.1", hostname, domain)
# Copy over template environments
util.put_and_chown(local_environments, remote_codedir)
# Add modules from Puppet Forge. These should in my experience be limited to
# those which provide new types and providers. In particular puppetlabs'
# modules which control some daemon (puppetdb, postgresql, mysql) are
# extremely complex, very prone to breakage and nasty to debug.
for module in forge_modules:
add_forge_module(module)
# Git setup
git.install()
git.init(basedir)
if not exists(modules_dir):
sudo("mkdir "+modules_dir)
git.init(modules_dir)
git.add_submodules(basedir=modules_dir)
git.add_all(basedir)
git.commit(basedir, "Initial commit")
# Link hieradata and manifests from production to testing. This keeps the
# testing environment identical to the production environment. The modules
# directory in testing is separate and may (or may not) contain modules that
# override or complement those in production.
util.symlink(remote_codedir+"/environments/production/hieradata", remote_codedir+"/environments/testing/hieradata")
util.symlink(remote_codedir+"/environments/production/manifests", remote_codedir+"/environments/testing/manifests")
# Start puppetserver to generate the CA and server certificates/keys
service.start("puppetserver")
run_agent(noop="False")
def setup_server4(hostname=None,
domain=None,
pc="1",
forge_modules=[
"puppetlabs/stdlib", "puppetlabs/concat",
"puppetlabs/firewall", "puppetlabs/apt"
]):
"""Setup Puppet 4 server"""
import package, util, git, service
# Local files to copy over
basedir = "/etc/puppetlabs"
local_master_conf = "files/puppet-master.conf"
remote_master_conf = basedir + "/puppet/puppet.conf"
local_hiera_yaml = "files/hiera.yaml"
remote_hiera_yaml = basedir + "/code/hiera.yaml"
local_fileserver_conf = "files/fileserver.conf"
remote_fileserver_conf = basedir + "/puppet/fileserver.conf"
local_environments = "files/environments"
remote_codedir = basedir + "/code"
local_gitignore = "files/gitignore"
remote_gitignore = basedir + "/.gitignore"
modules_dir = basedir + "/code/environments/production/modules"
# Verify that all the local files are in place
try:
open(local_master_conf)
open(local_hiera_yaml)
except IOError:
print "ERROR: some local config files were missing!"
sys.exit(1)
# Autodetect hostname and domain from env.host, if they're not overridden
# with method parameters
if not hostname:
hostname = util.get_hostname()
if not domain:
domain = util.get_domain()
# Ensure that clock is correct before doing anything else, like creating SSL
# certificates.
util.set_clock()
# Start the install
install_puppetlabs_release_package(pc)
package.install("puppetserver")
util.put_and_chown(local_master_conf, remote_master_conf)
util.put_and_chown(local_hiera_yaml, remote_hiera_yaml)
util.put_and_chown(local_fileserver_conf, remote_fileserver_conf)
util.put_and_chown(local_gitignore, remote_gitignore)
util.add_to_path("/opt/puppetlabs/bin")
util.set_hostname(hostname + "." + domain)
# "facter fqdn" return a silly name on EC2 without this
util.add_host_entry("127.0.1.1", hostname, domain)
# Copy over template environments
util.put_and_chown(local_environments, remote_codedir)
# Add modules from Puppet Forge. These should in my experience be limited to
# those which provide new types and providers. In particular puppetlabs'
# modules which control some daemon (puppetdb, postgresql, mysql) are
# extremely complex, very prone to breakage and nasty to debug.
for module in forge_modules:
add_forge_module(module)
# Git setup
git.install()
git.init(basedir)
if not exists(modules_dir):
sudo("mkdir " + modules_dir)
git.init(modules_dir)
git.add_submodules(basedir=modules_dir)
git.add_all(basedir)
git.commit(basedir, "Initial commit")
# Link hieradata and manifests from production to testing. This keeps the
# testing environment identical to the production environment. The modules
# directory in testing is separate and may (or may not) contain modules that
# override or complement those in production.
util.symlink(remote_codedir + "/environments/production/hieradata",
remote_codedir + "/environments/testing/hieradata")
util.symlink(remote_codedir + "/environments/production/manifests",
remote_codedir + "/environments/testing/manifests")
# Start puppetserver to generate the CA and server certificates/keys
service.start("puppetserver")
run_agent(noop="False")