def setup_repos():
# Add repos
if conf.get_config("AMI", "Type") == "Enterprise":
logger.pipe(
'echo "deb http://{0}:{1}@debian.datastax.com/enterprise stable main"'
.format(options.username, options.password),
'sudo tee /etc/apt/sources.list.d/datastax.sources.list')
else:
logger.pipe(
'echo "deb http://debian.datastax.com/community stable main"',
'sudo tee /etc/apt/sources.list.d/datastax.sources.list')
# Add repokeys
logger.exe(
'sudo apt-key add /home/ubuntu/datastax_ami/repo_keys/DataStax.key')
# Perform the install
logger.exe('sudo apt-get update')
while True:
output = logger.exe('sudo apt-get update')
if not output[1] and not 'err' in output[0].lower(
) and not 'failed' in output[0].lower():
break
time.sleep(5)
def opscenter_installation():
if instance_data['launchindex'] == 0 and options.opscenter != "no":
logger.info('Installing OpsCenter...')
logger.exe('sudo apt-get install -y opscenter libssl0.9.8')
logger.exe('sudo service opscenterd stop')
elif options.opscenter == "no":
conf.set_config("OpsCenter", "NoOpsCenter", True)
def check_and_launch_opscenter():
if config_data['launchindex'] == 0 and conf.get_config(
"OpsCenter", "DNS") and not conf.get_config(
"AMI", "CompletedFirstBoot") and not conf.get_config(
"OpsCenter", "NoOpsCenter"):
logger.exe('sudo service opscenterd restart')
conf.set_config("AMI", "CompletedFirstBoot", True)
def additional_pre_configurations():
# Get required keys for Ubuntu
logger.exe(
'sudo apt-key add /home/ubuntu/datastax_ami/repo_keys/Launchpad_VLC.C2518248EEA14886.key'
)
logger.exe(
'sudo apt-key add /home/ubuntu/datastax_ami/repo_keys/Ubuntu_Archive.40976EAF437D05B5.key'
)
def prepare_for_raid():
# Only create raid0 once. Mount all times in init.d script. A failsafe against deleting this file.
if conf.get_config("AMI", "RAIDAttempted"):
return
conf.set_config("AMI", "CurrentStatus", "Raiding started")
# Remove EC2 default /mnt from fstab
fstab = ''
file_to_open = '/etc/fstab'
logger.exe('sudo chmod 777 {0}'.format(file_to_open))
with open(file_to_open, 'r') as f:
for line in f:
if not "/mnt" in line:
fstab += line
with open(file_to_open, 'w') as f:
f.write(fstab)
logger.exe('sudo chmod 644 {0}'.format(file_to_open))
# Create a list of devices
devices = glob.glob('/dev/xvd*')
devices.remove('/dev/xvda1')
devices.sort()
logger.info('Unformatted devices: {0}'.format(devices))
# Check if there are enough drives to start a RAID set
if len(devices) > 1:
time.sleep(3) # was at 20
mnt_point = mount_raid(devices)
# Not enough drives to RAID together.
else:
mnt_point = format_xfs(devices)
if not options.raidonly:
# Change cassandra.yaml to point to the new data directories
with open(os.path.join(config_data['conf_path'], 'cassandra.yaml'),
'r') as f:
yaml = f.read()
yaml = yaml.replace('/var/lib/cassandra/data',
os.path.join(mnt_point, 'cassandra', 'data'))
yaml = yaml.replace(
'/var/lib/cassandra/saved_caches',
os.path.join(mnt_point, 'cassandra', 'saved_caches'))
yaml = yaml.replace('/var/lib/cassandra/commitlog',
os.path.join(mnt_point, 'cassandra', 'commitlog'))
with open(os.path.join(config_data['conf_path'], 'cassandra.yaml'),
'w') as f:
f.write(yaml)
# Never create raid array again
conf.set_config("AMI", "RAIDAttempted", True)
logger.info("Mounted Raid.\n")
conf.set_config("AMI", "MountDirectory", mnt_point)
conf.set_config("AMI", "CurrentStatus", "Raiding complete")
def fix_profile():
# Setup a link to the motd script that is provided in the git repository
file_to_open = '/home/ubuntu/.profile'
logger.exe('sudo chmod 777 ' + file_to_open)
with open(file_to_open, 'r+') as f:
profile = f.read()
f.seek(0)
f.write(profile.replace('jdk1.6.0_31', 'jdk1.6.0_38'))
logger.exe('sudo chmod 644 ' + file_to_open)
def run():
print '[INFO] Waiting 60 seconds to restart opscenter, setup demos, and possibly send emails...'
time.sleep(40)
logger.exe('sudo service datastax-agent restart')
time.sleep(20)
get_ec2_data()
get_email_auth()
check_and_launch_opscenter()
check_and_send_emails()
def run():
initial_configurations()
write_bin_tools()
restart_tasks()
if conf.get_config("AMI", "OpsCenterOnly"):
logger.exe('sudo service opscenterd restart')
if conf.get_config("AMI", "LeadingSeed"):
wait_for_seed()
launch_opscenter()
start_services()
def run():
# Remove script files
logger.exe('sudo rm ds2_configure.py')
logger.info(
'Deleting ds2_configure.py now. This AMI will never change any configs after this first run.'
)
additional_pre_configurations()
clear_motd()
try:
get_ec2_data()
except urllib2.HTTPError:
exit_path(
"Clusters within a VPC or backed by Spot Instances are not supported."
)
parse_ec2_userdata()
if not options.raidonly:
use_ec2_userdata()
confirm_authentication()
if options.javaversion:
install_java()
setup_repos()
clean_installation()
opscenter_installation()
get_seed_list()
checkpoint_info()
if not options.raidonly:
calculate_tokens()
construct_yaml()
construct_opscenter_conf()
construct_opscenter_cluster_conf()
construct_env()
construct_dse()
prepare_for_raid()
if not options.raidonly:
construct_core_site()
construct_mapred_site()
sync_clocks()
additional_post_configurations()
logger.info("ds2_configure.py completed!\n")
conf.set_config("AMI", "CurrentStatus", "Complete!")
def start_services():
# Wait for system setup changes to settle
time.sleep(5)
# Actually start the application
if conf.get_config("AMI", "Type") == "Community" or conf.get_config(
"AMI", "Type") == "False":
logger.info('Starting DataStax Community...')
logger.exe('sudo service cassandra restart')
elif conf.get_config("AMI", "Type") == "Enterprise":
logger.info('Starting DataStax Enterprise...')
logger.exe('sudo service dse restart')
def construct_mapred_site():
if conf.get_config("AMI", "Type") == "Enterprise":
with open('/etc/dse/hadoop/mapred-site.xml', 'r') as f:
mapred_site = f.read()
mapred_local_dir = os.path.join(
conf.get_config("AMI", "MountDirectory"), 'hadoop', 'mapredlocal')
mapred_site = mapred_site.replace('/tmp/mapredlocal', mapred_local_dir)
logger.exe('sudo mkdir -p %s' % mapred_local_dir)
logger.exe('sudo chown -R cassandra:cassandra %s' % mapred_local_dir)
with open('/etc/dse/hadoop/mapred-site.xml', 'w') as f:
f.write(mapred_site)
def get_git_reset_arg(commitish):
if not commitish:
return ''
# This will work if the commitish is a remote branch name
(commit_id, err) = logger.exe('git rev-parse origin/' + commitish)
if err:
# This will work if the commitish is a short-hash or long-hash
(commit_id, err) = logger.exe('git rev-parse ' + commitish)
if err:
# Can't figure out what commit is being referenced, return no
# commit-id and git reset will simply hard reset the workspace
# using the current commit in the git checkout baked into the AMI
return ''
return commit_id
def construct_agent():
logger.exe('sudo mkdir -p /var/lib/datastax-agent/conf')
logger.exe('sudo chown ubuntu:ubuntu /var/lib/datastax-agent/conf')
with open('/var/lib/datastax-agent/conf/address.yaml', 'w') as f:
f.write('stomp_interface: %s' % config_data['opscenterseed'])
logger.exe('cat /var/lib/datastax-agent/conf/address.yaml')
logger.exe(
'sudo chown opscenter-agent:opscenter-agent /var/lib/datastax-agent/conf'
)
logger.info('address.yaml configured.')
def setup_repos():
# Clear repo when filled, primarily for debugging purposes
logger.exe('sudo rm /etc/apt/sources.list.d/datastax.sources.list',
log=False,
expectError=True)
# Add repos
if conf.get_config("AMI", "Type") == "Enterprise":
logger.pipe(
'echo "deb http://{0}:{1}@debian.datastax.com/enterprise stable main"'
.format(options.username, options.password),
'sudo tee -a /etc/apt/sources.list.d/datastax.sources.list')
else:
logger.pipe(
'echo "deb http://debian.datastax.com/community stable main"',
'sudo tee -a /etc/apt/sources.list.d/datastax.sources.list')
# Add repokeys
logger.pipe(
'curl -s http://installer.datastax.com/downloads/ubuntuarchive.repo_key',
'sudo apt-key add -')
logger.pipe('curl -s http://opscenter.datastax.com/debian/repo_key',
'sudo apt-key add -')
logger.pipe('curl -s http://debian.datastax.com/debian/repo_key',
'sudo apt-key add -')
if options.dev:
logger.pipe(
'echo "deb {0} maverick main"'.format(options.dev.split(',')[0]),
'sudo tee -a /etc/apt/sources.list.d/datastax.sources.list')
logger.pipe('curl -s {0}'.format(options.dev.split(',')[1]),
'sudo apt-key add -')
# Perform the install
logger.exe('sudo apt-get update')
while True:
output = logger.exe('sudo apt-get update')
if not output[1] and not 'err' in output[0].lower(
) and not 'failed' in output[0].lower():
break
time.sleep(5)
def sync_clocks():
# Confirm that NTP is installed
logger.exe('sudo apt-get -y install ntp')
with open('/etc/ntp.conf', 'r') as f:
ntp_conf = f.read()
# Create a list of ntp server pools
server_list = ""
for i in range(0, 4):
server_list += "server {0}.north-america.pool.ntp.org\n".format(i)
# Overwrite the single ubuntu ntp server with the server pools
ntp_conf = ntp_conf.replace('server ntp.ubuntu.com', server_list)
with open('/etc/ntp.conf', 'w') as f:
f.write(ntp_conf)
# Restart the service
logger.exe('sudo service ntp restart')
def syncClocks():
# Confirm that NTP is installed
logger.exe('sudo apt-get -y install ntp')
with open('/etc/ntp.conf', 'r') as f:
ntpConf = f.read()
# Create a list of ntp server pools
serverList = ""
for i in range(0, 4):
serverList += "server " + str(i) + ".north-america.pool.ntp.org\n"
# Overwrite the single ubuntu ntp server with the server pools
ntpConf = ntpConf.replace('server ntp.ubuntu.com', serverList)
with open('/etc/ntp.conf', 'w') as f:
f.write(ntpConf)
# Restart the service
logger.exe('sudo service ntp restart')
def initial_configurations():
# Begin configuration this is only run once in Public Packages
if conf.get_config("AMI", "CurrentStatus") != "Complete!":
# Configure DataStax variables
try:
import ds2_configure
ds2_configure.run()
except:
conf.set_config(
"AMI", "Error",
"Exception seen in %s. Please check ~/datastax_ami/ami.log for more info."
% 'ds1_launcher.py')
logger.exception('ds1_launcher.py')
# Set ulimit hard limits
logger.pipe('echo "* soft nofile 32768"',
'sudo tee -a /etc/security/limits.conf')
logger.pipe('echo "* hard nofile 32768"',
'sudo tee -a /etc/security/limits.conf')
logger.pipe('echo "root soft nofile 32768"',
'sudo tee -a /etc/security/limits.conf')
logger.pipe('echo "root hard nofile 32768"',
'sudo tee -a /etc/security/limits.conf')
# Change permission back to being ubuntu's and cassandra's
logger.exe('sudo chown -hR ubuntu:ubuntu /home/ubuntu')
logger.exe('sudo chown -hR cassandra:cassandra /raid0/cassandra',
False)
logger.exe('sudo chown -hR cassandra:cassandra /mnt/cassandra', False)
else:
logger.info('Skipping initial configurations.')
def format_xfs(devices):
# Make sure the device is umounted, then run fdisk on the device
logger.info(
'Clear "invalid flag 0x0000 of partition table 4" by issuing a write, then running fdisk on the device...'
)
formatCommands = "echo 'd\nn\np\n1\n\n\nt\n83\nw'"
logger.exe('sudo umount {0}'.format(devices[0]), expectError=True)
logger.pipe("echo 'w'", 'sudo fdisk -c -u {0}'.format(devices[0]))
logger.pipe(formatCommands, 'sudo fdisk -c -u {0}'.format(devices[0]))
# Create a list of partitions to RAID
logger.exe('sudo fdisk -l')
partitions = glob.glob('/dev/xvd*[0-9]')
if '/dev/xvda1' in partitions:
partitions.remove('/dev/xvda1')
partitions.sort()
logger.info('Formatting the new partition:')
logger.exe('sudo mkfs.xfs -f {0}'.format(partitions[0]))
# Configure fstab and mount the new formatted device
mnt_point = '/mnt'
create_cassandra_directories(mnt_point=mnt_point, device=partitions[0])
return mnt_point
def getAddresses():
# Find internal IP address for seed list
global internalip
req = urllib2.Request('http://instance-data/latest/meta-data/local-ipv4')
internalip = urllib2.urlopen(req).read()
# Find public hostname for JMX
req = urllib2.Request('http://instance-data/latest/meta-data/public-hostname')
global publichostname
publichostname = urllib2.urlopen(req).read()
# Find launch index for token splitting
req = urllib2.Request('http://instance-data/latest/meta-data/ami-launch-index')
global launchindex
launchindex = int(urllib2.urlopen(req).read())
# Find reservation-id for cluster-id and jmxpass
req = urllib2.Request('http://instance-data/latest/meta-data/reservation-id')
global reservationid, jmxPass, clustername
reservationid = urllib2.urlopen(req).read()
jmxPass = reservationid
clustername = reservationid
# Try to get EC2 User Data
userDataExists = False
try:
req = urllib2.Request('http://instance-data/latest/user-data/')
global userdata
userdata = urllib2.urlopen(req).read()
userDataExists = True
except Exception, e:
logger.info("No User Data was set. Naming cluster the same as the reservation ID.")
logger.exe('sudo add-apt-repository "deb http://www.apache.org/dist/cassandra/debian 08x main"')
logger.exe('sudo apt-get update')
logger.exe('sudo apt-get install -y cassandra')
logger.exe('sudo rm -rf /var/lib/cassandra/*')
logger.exe('sudo service cassandra stop')
def initial_configurations():
# Begin configuration this is only run once in Public Packages
if not conf.get_config("AMI", "CurrentStatus"):
# Configure DataStax variables
try:
import ds2_configure
ds2_configure.run()
except:
conf.set_config("AMI", "Error", "Exception seen in %s. Please check ~/datastax_ami/ami.log for more info." % 'ds1_launcher.py')
logger.exception('ds1_launcher.py')
# Change permission back to being ubuntu's and cassandra's
logger.exe('sudo chown -hR ubuntu:ubuntu /home/ubuntu')
logger.exe('sudo chown -hR cassandra:cassandra /raid0/cassandra', False)
logger.exe('sudo chown -hR cassandra:cassandra /mnt/cassandra', False)
# Ensure permissions
directory_list = [
('/home/ubuntu', 'ubuntu', 'ubuntu'),
('/raid0/cassandra', 'cassandra', 'cassandra'),
('/mnt/cassandra', 'cassandra', 'cassandra')
]
for directory in directory_list:
if os.path.isdir(directory[0]):
logger.info('Checking permissions for: %s' % directory[0])
attempt = 0
max_attempts = 10
permissions_set = False
while attempt < max_attempts:
logger.info('Attempt #%s' % attempt)
stat_info = os.stat(directory[0])
uid = stat_info.st_uid
gid = stat_info.st_gid
user = pwd.getpwuid(uid)[0]
group = grp.getgrgid(gid)[0]
if user == directory[1] and group == directory[2]:
permissions_set = True
break
attempt += 1
time.sleep(1)
if not permissions_set:
logger.warn('Permissions not set correctly. Please run manually:')
logger.warn('sudo chown -hR %s:%s %s' % (directory[1], directory[2], directory[0]))
logger.warn('sudo service dse restart')
else:
logger.info('Permissions set for %s as %s:%s' % (directory[0], user, group))
else:
logger.info('Skipping initial configurations.')
def additional_post_configurations():
logger.exe('sudo apt-get install s3cmd')
# Setup HADOOP_HOME for ubuntu
file_to_open = '/home/ubuntu/.profile'
logger.exe('sudo chmod 777 ' + file_to_open)
with open(file_to_open, 'a') as f:
f.write("""
export HADOOP_HOME=/usr/share/dse/hadoop
""")
logger.exe('sudo chmod 644 ' + file_to_open)
# Setup HADOOP_HOME for root
os.chdir('/root')
file_to_open = '.profile'
logger.exe('sudo chmod 777 ' + file_to_open)
with open(file_to_open, 'w') as f:
f.write("""
export HADOOP_HOME=/usr/share/dse/hadoop
""")
logger.exe('sudo chmod 644 ' + file_to_open)
os.chdir('/home/ubuntu')
pass
def get_git_reset_arg(commitish):
if not commitish:
return ''
(commit_id, err) = logger.exe('git rev-parse ' + commitish)
if err:
return ''
# If the commit-ish is a valid commit id, use it as-is.
# Otherwise, prefix with the remote (always origin).
if commit_id.strip() == commitish:
return commit_id
else:
return 'origin/' + commitish
def get_email_auth():
# Setup global variables
config_data['smtp'] = None
config_data['port'] = None
config_data['username'] = None
config_data['password'] = None
# Ensure that the ring output is in the log
logger.exe('nodetool -h localhost ring')
# Parse Email information
if conf.get_config("AMI", "Email"):
try:
raw_email = conf.get_config("AMI", "Email")
read = raw_email.split(':')
logger.info("Parsed Email Options: {0}".format(read))
config_data['smtp'] = read[0]
config_data['port'] = read[1]
config_data['username'] = read[2]
config_data['password'] = read[3]
except:
print "[ERROR] No emails will be sent. Error during parsing."
def opscenter_installation():
if instance_data['launchindex'] == 0 and options.opscenter != "no":
logger.info('Installing OpsCenter...')
if conf.get_config("AMI", "Type") == "Community":
logger.exe('sudo apt-get -y install opscenter-free libssl0.9.8')
elif conf.get_config("AMI", "Type") == "Enterprise":
logger.exe('sudo apt-get -y install opscenter libssl0.9.8')
logger.exe('sudo service opscenterd stop')
elif options.opscenter == "no":
conf.set_config("OpsCenter", "NoOpsCenter", True)
def start_services():
# Wait for system setup changes to settle
time.sleep(5)
# Actually start the application
if conf.get_config("AMI", "Type") == "Community" or conf.get_config(
"AMI", "Type") == "False":
logger.info('Starting DataStax Community...')
logger.exe('sudo service cassandra restart')
elif conf.get_config("AMI", "Type") == "Enterprise":
logger.info('Starting DataStax Enterprise...')
logger.exe('sudo service dse restart')
# Wait 30 seconds for the Cassandra services to fully boot
time.sleep(30)
# Ensure that cassandra doesn't die shortly after first boot
# I've seen issues with a device not being available... but this is after the full raid
if not conf.get_config("AMI", "CompletedFirstBoot"):
start_time = time.time()
logger.info('Checking for 15 seconds to ensure Cassandra stays up...')
while time.time() - start_time < 15:
cassandra_running = False
if not logger.exe('nodetool info', expectError=True)[1]:
cassandra_running = True
if not cassandra_running:
# Restart the application
if conf.get_config("AMI",
"Type") == "Community" or conf.get_config(
"AMI", "Type") == "False":
logger.info('Restarting DataStax Community...')
logger.exe('sudo service cassandra restart')
elif conf.get_config("AMI", "Type") == "Enterprise":
logger.info('Restarting DataStax Enterprise...')
logger.exe('sudo service dse restart')
time.sleep(3)
time.sleep(1)
def construct_core_site():
if conf.get_config("AMI", "Type") == "Enterprise":
with open('/etc/dse/hadoop/core-site.xml', 'r') as f:
core_site = f.read()
hadoop_tmp_dir = os.path.join(conf.get_config("AMI", "MountDirectory"),
'hadoop')
tmp_dir = '\n <!-- AMI configuration -->\n <property>\n <name>hadoop.tmp.dir</name>\n <value>%s/${user.name}</value>\n </property>\n</configuration>' % hadoop_tmp_dir
core_site = core_site.replace('</configuration>', tmp_dir)
logger.exe('sudo mkdir -p %s' % hadoop_tmp_dir)
logger.exe('sudo chown -R cassandra:cassandra %s' % hadoop_tmp_dir)
hadoop_ubuntu_dir = os.path.join(hadoop_tmp_dir, 'ubuntu')
logger.exe('sudo mkdir -p %s' % hadoop_ubuntu_dir)
logger.exe('sudo chown -R ubuntu:ubuntu %s' % hadoop_ubuntu_dir)
with open('/etc/dse/hadoop/core-site.xml', 'w') as f:
f.write(core_site)
def restart_tasks():
logger.info("AMI Type: " + str(conf.get_config("AMI", "Type")))
# Mount all attached drives
logger.exe('sudo mount -a')
# Disable swap
logger.exe('sudo swapoff --all')
# Ensure the correct blockdev readahead since this sometimes resets after restarts
if conf.get_config('AMI', 'raid_readahead'):
logger.exe('sudo blockdev --setra %s /dev/md0' % (conf.get_config('AMI', 'raid_readahead')), expectError=True)
# Add repos
if conf.getConfig("AMI", "Type") == "Enterprise":
logger.pipe('echo "deb http://' + options.username + ':' + options.password + '@debian.datastax.com/enterprise stable main"', 'sudo tee -a /etc/apt/sources.list.d/datastax.sources.list')
else:
logger.pipe('echo "deb http://debian.datastax.com/community stable main"', 'sudo tee -a /etc/apt/sources.list.d/datastax.sources.list')
logger.pipe('curl -s http://installer.datastax.com/downloads/ubuntuarchive.repo_key', 'sudo apt-key add -')
logger.pipe('curl -s http://opscenter.datastax.com/debian/repo_key', 'sudo apt-key add -')
logger.pipe('curl -s http://debian.datastax.com/debian/repo_key', 'sudo apt-key add -')
if options and options.dev:
logger.pipe('echo "deb ' + options.dev.split(',')[0] + ' maverick main"', 'sudo tee -a /etc/apt/sources.list.d/datastax.sources.list')
logger.pipe('curl -s ' + options.dev.split(',')[1], 'sudo apt-key add -')
# Perform the install
logger.exe('sudo apt-get update')
logger.exe('sudo apt-get update')
time.sleep(5)
logger.info('Performing deployment install...')
if conf.getConfig("AMI", "Type") == "Community":
logger.exe('sudo apt-get install -y apache-cassandra1')
logger.exe('sudo rm -rf /var/lib/cassandra/*')
logger.exe('sudo service cassandra stop')
logger.exe('sudo apt-get install -y dsc-demos')
elif conf.getConfig("AMI", "Type") == "Enterprise":
logger.exe('sudo apt-get install -y dse-full')
logger.exe('sudo rm -rf /var/lib/cassandra/*')
logger.exe('sudo service dse stop')
# Remove the presaved information from startup
logger.exe('sudo rm -rf /var/lib/cassandra')
def installCQLsh():
logger.exe('wget http://downloads.datastax.com/community/datastax-cqlsh.tar.gz', False)
logger.exe('tar xf datastax-cqlsh.tar.gz')
logger.exe('chown -R ubuntu:ubuntu datastax-cqlsh')
logger.exe('rm datastax-cqlsh.tar.gz')
logger.exe('mv datastax-cqlsh /home/ubuntu')
def mountRAID():
# Only create raid0 once. Mount all times in init.d script.
if not conf.getConfig("AMI", "RAIDCreated"):
# Remove EC2 default /mnt from fstab
fstab = ''
fileToOpen = '/etc/fstab'
logger.exe('sudo chmod 777 ' + fileToOpen)
with open(fileToOpen, 'r') as f:
for line in f:
if not "/mnt" in line:
fstab += line
with open(fileToOpen, 'w') as f:
f.write(fstab)
logger.exe('sudo chmod 644 ' + fileToOpen)
# Create a list of devices
devices = glob.glob("/dev/sd*")
devices.remove('/dev/sda1')
devices.sort()
logger.info('Unformatted devices: ' + str(devices))
# Check if there are enough drives to start a RAID set
if len(devices) > 1:
# Make sure the devices are umounted, then run fdisk on each device
logger.info('Clear "invalid flag 0x0000 of partition table 4" by issuing a write, then running fdisk on each device...')
formatCommands = """echo 'n
p
1
t
fd
w'"""
for device in devices:
logger.info('Confirming devices are not mounted:')
logger.exe('sudo umount ' + device, False)
logger.pipe("echo 'w'", 'sudo fdisk -c -u ' + device)
logger.pipe(formatCommands, 'sudo fdisk -c -u ' + device)
# Create a list of partitions to RAID
logger.exe('sudo fdisk -l')
partitions = glob.glob("/dev/sd*[0-9]")
partitions.remove('/dev/sda1')
partitions.sort()
logger.info('Partitions about to be added to RAID0 set: ' + str(partitions))
# Make sure the partitions are umounted and create a list string
partionList = ''
for partition in partitions:
logger.info('Confirming partitions are not mounted:')
logger.exe('sudo umount ' + partition, False)
partionList += partition + ' '
partionList = partionList.strip()
logger.info('Creating the RAID0 set:')
time.sleep(5)
logger.pipe('yes', 'sudo mdadm --create /dev/md0 --chunk=256 --level=0 --raid-devices=' + str(len(devices)) + ' ' + partionList, False)
logger.pipe('echo DEVICE ' + partionList, 'sudo tee /etc/mdadm/mdadm.conf')
logger.pipe('mdadm --detail --scan', 'sudo tee -a /etc/mdadm/mdadm.conf')
time.sleep(5)
logger.exe('blockdev --setra 65536 /dev/md0')
logger.info('Formatting the RAID0 set:')
time.sleep(5)
logger.exe('sudo mkfs.xfs -f /dev/md0')
# Configure fstab and mount the new RAID0 device
raidMnt = '/raid0'
logger.pipe("echo '/dev/md0\t" + raidMnt + "\txfs\tdefaults,nobootwait,noatime\t0\t0'", 'sudo tee -a /etc/fstab')
logger.exe('sudo mkdir ' + raidMnt)
logger.exe('sudo mount -a')
logger.exe('sudo mkdir -p ' + raidMnt + '/cassandra/')
logger.exe('sudo chown -R ubuntu:ubuntu ' + raidMnt + '/cassandra')
logger.info('Showing RAID0 details:')
logger.exe('cat /proc/mdstat')
logger.exe('echo "15000" > /proc/sys/dev/raid/speed_limit_min')
logger.exe('sudo mdadm --detail /dev/md0')
else:
# Make sure the device is umounted, then run fdisk on the device
logger.info('Clear "invalid flag 0x0000 of partition table 4" by issuing a write, then running fdisk on the device...')
formatCommands = """echo 'd
n
p
1
t
83
w'"""
logger.exe('sudo umount ' + devices[0])
logger.pipe("echo 'w'", 'sudo fdisk -c -u ' + devices[0])
logger.pipe(formatCommands, 'sudo fdisk -c -u ' + devices[0])
# Create a list of partitions to RAID
logger.exe('sudo fdisk -l')
partitions = glob.glob("/dev/sd*[0-9]")
partitions.remove('/dev/sda1')
partitions.sort()
logger.info('Formatting the new partition:')
logger.exe('sudo mkfs.xfs -f ' + partitions[0])
# Configure fstab and mount the new formatted device
mntPoint = '/mnt'
logger.pipe("echo '" + partitions[0] + "\t" + mntPoint + "\txfs\tdefaults,nobootwait,noatime\t0\t0'", 'sudo tee -a /etc/fstab')
logger.exe('sudo mkdir ' + mntPoint, False)
logger.exe('sudo mount -a')
logger.exe('sudo mkdir -p ' + mntPoint + '/cassandra')
logger.exe('sudo chown -R cassandra:cassandra ' + mntPoint + '/cassandra')
# Change cassandra.yaml to point to the new data directories
with open(confPath + 'cassandra.yaml', 'r') as f:
yaml = f.read()
if len(partitions) > 1:
yaml = yaml.replace('/var/lib/cassandra/data', raidMnt + '/cassandra/data')
yaml = yaml.replace('/var/lib/cassandra/saved_caches', raidMnt + '/cassandra/saved_caches')
yaml = yaml.replace('/var/lib/cassandra/commitlog', raidMnt + '/cassandra/commitlog')
else:
yaml = yaml.replace('/var/lib/cassandra/data', mntPoint + '/cassandra/data')
yaml = yaml.replace('/var/lib/cassandra/saved_caches', mntPoint + '/cassandra/saved_caches')
yaml = yaml.replace('/var/lib/cassandra/commitlog', mntPoint + '/cassandra/commitlog')
with open(confPath + 'cassandra.yaml', 'w') as f:
f.write(yaml)
# Remove the old cassandra folders
subprocess.Popen("sudo rm -rf /var/log/cassandra/*", shell=True)
subprocess.Popen("sudo rm -rf /var/lib/cassandra/*", shell=True)
logger.exe('sudo chown -R cassandra:cassandra /var/lib/cassandra')
logger.exe('sudo chown -R cassandra:cassandra /var/log/cassandra')
# Never create raid array again
conf.setConfig("AMI", "RAIDCreated", True)
logger.info("Mounted Raid.\n")
#!/usr/bin/env python
### Script provided by DataStax.
import logger
import conf
# Update the AMI codebase if it's its first booot
if not conf.get_config("AMI", "CompletedFirstBoot"):
logger.exe('git pull')
# Start AMI start code
try:
import ds1_launcher
ds1_launcher.run()
except:
logger.exception('ds0_updater.py')
#!/usr/bin/env python
### Script provided by DataStax.
import os
import logger
import conf
if not conf.getConfig("AMI", "CompletedFirstBoot"):
logger.exe('git pull')
logger.exe('python ds1_launcher.py', False)
def create_cassandra_directories(mnt_point, device):
logger.pipe(
"echo '{0}\t{1}\txfs\tdefaults,nobootwait\t0\t0'".format(
device, mnt_point), 'sudo tee -a /etc/fstab')
logger.exe('sudo mount -a')
if conf.get_config("AMI", "RaidOnly"):
output = logger.exe('id cassandra', expectError=True)
if output[1] or 'no such user' in output[0].lower():
logger.pipe(
'yes',
'sudo adduser --no-create-home --disabled-password cassandra')
while True:
output = logger.exe('id cassandra', expectError=True)
if not output[1] and not 'no such user' in output[0].lower():
break
time.sleep(1)
output = logger.exe('id opscenter-agent', expectError=True)
if output[1] or 'no such user' in output[0].lower():
logger.pipe(
'yes',
'sudo adduser --no-create-home --disabled-password opscenter-agent'
)
while True:
output = logger.exe('id opscenter-agent', expectError=True)
if not output[1] and not 'no such user' in output[0].lower():
break
time.sleep(1)
logger.exe('sudo mkdir -p {0}'.format(
os.path.join(mnt_point, 'cassandra', 'logs')))
logger.exe('sudo chown -R cassandra:cassandra {0}'.format(
os.path.join(mnt_point, 'cassandra')))
# Create symlink for Cassandra data
logger.exe('sudo rm -rf /var/lib/cassandra')
logger.exe('sudo ln -s {0} /var/lib/cassandra'.format(
os.path.join(mnt_point, 'cassandra')))
logger.exe('sudo chown -R cassandra:cassandra /var/lib/cassandra')
# Create symlink for Cassandra logs
logger.exe('sudo rm -rf /var/log/cassandra')
logger.exe('sudo ln -s {0} /var/log/cassandra'.format(
os.path.join(mnt_point, 'cassandra', 'logs')))
logger.exe('sudo chown -R cassandra:cassandra /var/log/cassandra')
# Create symlink for OpsCenter logs
if instance_data['launchindex'] == 0 and options.opscenter != "no":
logger.exe('sudo mkdir -p {0}'.format(
os.path.join(mnt_point, 'opscenter', 'logs')))
logger.exe('sudo rm -rf /var/log/opscenter')
logger.exe('sudo ln -s {0} /var/log/opscenter'.format(
os.path.join(mnt_point, 'opscenter', 'logs')))
logger.exe('sudo chown -R root:root /var/log/opscenter')
logger.exe('sudo chown -R root:root {0}'.format(
os.path.join(mnt_point, 'opscenter')))
# Create symlink for DataStax Agent logs
logger.exe('sudo mkdir -p {0}'.format(
os.path.join(mnt_point, 'datastax-agent', 'logs')))
logger.exe('sudo rm -rf /var/log/datastax-agent')
logger.exe('sudo ln -s {0} /var/log/datastax-agent'.format(
os.path.join(mnt_point, 'datastax-agent', 'logs')))
logger.exe(
'sudo chown -R opscenter-agent:opscenter-agent /var/log/datastax-agent'
)
logger.exe('sudo touch /var/log/datastax-agent/agent.log')
logger.exe('sudo touch /var/log/datastax-agent/startup.log')
logger.exe('sudo chown -R cassandra:cassandra {0}'.format(
os.path.join(mnt_point, 'cassandra')))
logger.exe('sudo chown -R opscenter-agent:opscenter-agent {0}'.format(
os.path.join(mnt_point, 'datastax-agent')))
def clear_motd():
# To clear the default MOTD
logger.exe('sudo rm -rf /etc/motd')
logger.exe('sudo touch /etc/motd')
def clean_installation():
logger.info('Performing deployment install...')
# Hold onto baked limits conf before installation
logger.exe(
'sudo mv /etc/security/limits.d/cassandra.conf /etc/security/limits.d/cassandra.conf.bak'
)
if conf.get_config("AMI", "Type") == "Community":
if options.release and options.release.startswith('1.0'):
cassandra_release = options.release
if cassandra_release == '1.0.11-1':
cassandra_release = '1.0.11'
logger.exe(
'sudo apt-get install -y python-cql datastax-agent cassandra={0} dsc={1}'
.format(cassandra_release, options.release))
conf.set_config('AMI', 'package', 'dsc')
conf.set_config('Cassandra', 'partitioner', 'random_partitioner')
elif options.release and options.release.startswith('1.1'):
dsc_release = cassandra_release = options.release
if not dsc_release in ['1.1.1', '1.1.2', '1.1.3', '1.1.5']:
dsc_release = dsc_release + '-1'
logger.exe(
'sudo apt-get install -y python-cql datastax-agent cassandra={0} dsc1.1={1}'
.format(cassandra_release, dsc_release))
conf.set_config('AMI', 'package', 'dsc1.1')
conf.set_config('Cassandra', 'partitioner', 'random_partitioner')
elif options.release and options.release.startswith('1.2'):
dsc_release = cassandra_release = options.release
dsc_release = dsc_release + '-1'
logger.exe(
'sudo apt-get install -y python-cql datastax-agent cassandra={0} dsc12={1}'
.format(cassandra_release, dsc_release))
conf.set_config('AMI', 'package', 'dsc12')
conf.set_config('Cassandra', 'partitioner', 'murmur')
conf.set_config('Cassandra', 'vnodes', 'True')
elif options.release and options.release.startswith('2.0'):
dsc_release = cassandra_release = options.release
dsc_release = dsc_release + '-1'
logger.exe(
'sudo apt-get install -y python-cql datastax-agent cassandra={0} dsc20={1}'
.format(cassandra_release, dsc_release))
conf.set_config('AMI', 'package', 'dsc20')
conf.set_config('Cassandra', 'partitioner', 'murmur')
conf.set_config('Cassandra', 'vnodes', 'True')
else:
logger.exe(
'sudo apt-get install -y python-cql datastax-agent dsc20')
conf.set_config('AMI', 'package', 'dsc20')
conf.set_config('Cassandra', 'partitioner', 'murmur')
conf.set_config('Cassandra', 'vnodes', 'True')
logger.exe('sudo service cassandra stop')
elif conf.get_config("AMI", "Type") == "Enterprise":
config_data['conf_path'] = os.path.expanduser("/etc/dse/cassandra/")
if options.release:
install_list = 'sudo apt-get install -y dse-full={0} dse={0} dse-demos={0} dse-hive={0} dse-libcassandra={0} dse-libhadoop={0} dse-libhive={0} dse-libpig={0} dse-pig={0}'
if options.release.startswith('1'):
logger.exe(install_list.format(options.release))
conf.set_config('AMI', 'package', 'dse-full')
conf.set_config('Cassandra', 'partitioner',
'random_partitioner')
elif options.release.startswith('2'):
install_list += ' dse-liblog4j={0} dse-libsolr={0} dse-libsqoop={0} dse-libtomcat={0}'
if options.release.startswith(
'2.1') or options.release.startswith('2.2'):
install_list += ' dse-libmahout={0}'
logger.exe(install_list.format(options.release))
conf.set_config('AMI', 'package', 'dse-full')
conf.set_config('Cassandra', 'partitioner',
'random_partitioner')
elif options.release.startswith('3'):
install_list += ' dse-liblog4j={0} dse-libsolr={0} dse-libsqoop={0} dse-libtomcat={0} dse-libmahout={0} dse-libhadoop-native={0}'
logger.exe(install_list.format(options.release))
conf.set_config('AMI', 'package', 'dse-full')
if options.release.startswith('3.0'):
conf.set_config('Cassandra', 'partitioner',
'random_partitioner')
else:
conf.set_config('Cassandra', 'partitioner', 'murmur')
conf.set_config('Cassandra', 'vnodes', 'False')
else:
exit_path(
"--release should be in the format similar to `1.0.2-1` or `2.0`."
)
else:
logger.exe('sudo apt-get install -y dse-full')
conf.set_config('AMI', 'package', 'dse-full')
conf.set_config('Cassandra', 'partitioner', 'murmur')
conf.set_config('Cassandra', 'vnodes', 'False')
logger.exe('sudo service dse stop')
# Remove the presaved information from startup
logger.exe('sudo rm -rf /var/lib/cassandra')
logger.exe('sudo rm -rf /var/log/cassandra')
logger.exe('sudo mkdir -p /var/lib/cassandra')
logger.exe('sudo mkdir -p /var/log/cassandra')
logger.exe('sudo chown -R cassandra:cassandra /var/lib/cassandra')
logger.exe('sudo chown -R cassandra:cassandra /var/log/cassandra')
# Replace baked image conf after installation
logger.exe(
'sudo mv /etc/security/limits.d/cassandra.conf.bak /etc/security/limits.d/cassandra.conf'
)
def additionalDevConfigurations():
if isDEV:
logger.exe('sudo rm -rf /raid0/cassandra')
logger.exe('sudo rm -rf /var/lib/cassandra')
logger.exe('sudo rm -rf /var/log/cassandra')
logger.exe('ls -al')
req = urllib2.Request('http://instance-data/latest/meta-data/local-ipv4')
internalip = urllib2.urlopen(req).read()
req = urllib2.Request('http://instance-data/latest/meta-data/public-hostname')
publichostname = urllib2.urlopen(req).read()
req = urllib2.Request('http://instance-data/latest/meta-data/ami-launch-index')
launchindex = int(urllib2.urlopen(req).read())
smtp = None
port = None
username = None
password = None
logger.exe('nodetool -h localhost ring')
if conf.getConfig("AMI", "Email"):
try:
rawEmail = conf.getConfig("AMI", "Email")
read = rawEmail.split(':')
logger.info("Parsed Email Options: " + str(read))
smtp = read[0]
port = read[1]
username = read[2]
password = read[3]
except:
print "[ERROR] No emails will be sent. Error during parsing."
def checkAndLaunchOpsCenter():
global launchindex
if int(launchindex) == 0 and conf.getConfig("OpsCenter", "DNS") and not conf.getConfig("AMI", "CompletedFirstBoot") and not conf.getConfig("OpsCenter", "NoOpsCenter"):
logger.exe('sudo service opscenterd restart')
conf.setConfig("AMI", "CompletedFirstBoot", True)
def clearMOTD():
# To clear the default MOTD
logger.exe('sudo rm -rf /etc/motd')
logger.exe('sudo touch /etc/motd')
elif options.deployment == "08x":
logger.pipe('echo "deb http://www.apache.org/dist/cassandra/debian 08x main"', 'sudo tee -a /etc/apt/sources.list.d/datastax.sources.list')
elif options.deployment == "brisk":
logger.pipe('echo "deb http://debian.datastax.com/maverick maverick main"', 'sudo tee -a /etc/apt/sources.list.d/datastax.sources.list')
conf.setConfig("AMI", "Type", "Brisk")
global confPath
confPath = os.path.expanduser("/etc/brisk/cassandra/")
else:
logger.warn('Unable to parse Deployment Type. Defaulting to 0.8.x.')
logger.pipe('echo "deb http://www.apache.org/dist/cassandra/debian 08x main"', 'sudo tee -a /etc/apt/sources.list.d/datastax.sources.list')
else:
logger.info('Installing the latest version of Cassandra.')
logger.pipe('echo "deb http://www.apache.org/dist/cassandra/debian 08x main"', 'sudo tee -a /etc/apt/sources.list.d/datastax.sources.list')
# Perform the install
logger.exe('sudo apt-get update')
time.sleep(5)
logger.info('Performing deployment install...')
if conf.getConfig("AMI", "Type") == "Cassandra":
logger.exe('sudo apt-get install -y cassandra')
logger.exe('sudo rm -rf /var/lib/cassandra/*')
logger.exe('sudo service cassandra stop')
elif conf.getConfig("AMI", "Type") == "Brisk":
logger.exe('sudo apt-get install -y brisk-full')
logger.exe('sudo apt-get install -y brisk-demos')
logger.exe('sudo rm -rf /var/lib/cassandra/*')
logger.exe('sudo service brisk stop')
if options and options.email:
logger.info('Setting up diagnostic email using: ' + options.email)
conf.setConfig("AMI", "Email", options.email)
def mount_raid(devices):
# Make sure the devices are umounted, then run fdisk on each device
logger.info(
'Clear "invalid flag 0x0000 of partition table 4" by issuing a write, then running fdisk on each device...'
)
formatCommands = "echo 'n\np\n1\n\n\nt\nfd\nw'"
for device in devices:
logger.info('Confirming devices are not mounted:')
logger.exe('sudo umount {0}'.format(device), expectError=True)
logger.pipe("echo 'w'", 'sudo fdisk -c -u {0}'.format(device))
logger.pipe(formatCommands, 'sudo fdisk -c -u {0}'.format(device))
# Create a list of partitions to RAID
logger.exe('sudo fdisk -l')
partitions = glob.glob('/dev/xvd*[0-9]')
if '/dev/xvda1' in partitions:
partitions.remove('/dev/xvda1')
partitions.sort()
logger.info(
'Partitions about to be added to RAID0 set: {0}'.format(partitions))
# Make sure the partitions are umounted and create a list string
partion_list = ''
for partition in partitions:
logger.info('Confirming partitions are not mounted:')
logger.exe('sudo umount ' + partition, expectError=True)
partion_list = ' '.join(partitions).strip()
logger.info('Creating the RAID0 set:')
time.sleep(3) # was at 10
conf.set_config("AMI", "CurrentStatus", "Raid creation")
# Continuously create the Raid device, in case there are errors
raid_created = False
while not raid_created:
logger.exe(
'sudo mdadm --create /dev/md0 --chunk=256 --level=0 --raid-devices={0} {1}'
.format(len(partitions), partion_list),
expectError=True)
raid_created = True
logger.pipe('echo DEVICE {0}'.format(partion_list),
'sudo tee /etc/mdadm/mdadm.conf')
time.sleep(5)
# New parsing and elimination of the name= field due to 12.04's new RAID'ing methods
response = logger.exe('sudo mdadm --examine --scan')[0]
response = ' '.join(response.split(' ')[0:-1])
with open('/etc/mdadm/mdadm.conf', 'a') as f:
f.write(response)
logger.exe('sudo update-initramfs -u')
time.sleep(10)
conf.set_config('AMI', 'raid_readahead', 128)
logger.exe('sudo blockdev --setra %s /dev/md0' %
(conf.get_config('AMI', 'raid_readahead')))
logger.info('Formatting the RAID0 set:')
time.sleep(10)
raidError = logger.exe('sudo mkfs.xfs -f /dev/md0',
expectError=True)[1]
if raidError:
logger.exe('sudo mdadm --stop /dev/md_d0', expectError=True)
logger.exe('sudo mdadm --zero-superblock /dev/sdb1',
expectError=True)
raid_created = False
# Configure fstab and mount the new RAID0 device
mnt_point = '/raid0'
logger.exe('sudo mkdir {0}'.format(mnt_point))
create_cassandra_directories(mnt_point=mnt_point, device='/dev/md0')
logger.info('Showing RAID0 details:')
logger.exe('cat /proc/mdstat')
logger.exe('sudo mdadm --detail /dev/md0')
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html
logger.pipe('echo "30720"', 'sudo tee /proc/sys/dev/raid/speed_limit_min')
return mnt_point
#!/usr/bin/env python
### Script provided by DataStax.
import os, subprocess, shlex, sys, time, urllib2
import logger
import conf
# Begin configuration this is only run once in Public Packages
if os.path.isfile('ds2_configure.py'):
# Configure DataStax variables
read = logger.exe('python ds2_configure.py', False)
stderr = read[1]
stderr = stderr.replace('yes: standard output: Broken pipe', '').strip()
stderr = stderr.replace('yes: write error', '').strip()
if len(stderr) > 0:
# TODO: later
# logger.error("DEBUG")
# logger.error(stderr)
# logger.error("/DEBUG")
pass
# Set ulimit hard limits
logger.pipe('echo "* soft nofile 32768"', 'sudo tee -a /etc/security/limits.conf')
logger.pipe('echo "* hard nofile 32768"', 'sudo tee -a /etc/security/limits.conf')
logger.pipe('echo "root soft nofile 32768"', 'sudo tee -a /etc/security/limits.conf')
logger.pipe('echo "root hard nofile 32768"', 'sudo tee -a /etc/security/limits.conf')
# Create /raid0
logger.exe('sudo mount -a')
