/
fabfile.py
485 lines (445 loc) · 17.6 KB
/
fabfile.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
#!/usr/bin/python
# fabric libs
from fabric.api import *
from fabric.contrib.files import *
# useful standard lib to get passwords from prompt and not disaplay
import getpass
# useful sysytem commands
import os
import subprocess
import re
import commands
import time
import datetime
# global fallback user
env.user='ben'
# don't brick on servers that are not live
env.skip_bad_hosts=True
# turn off silly shell logins, no .bash|.rc|_profile.*
env.shell="/bin/bash -c"
# colorize errors
env.colorize_errors = True
# hosts should be single quoted and comma seperated
def all_internal_servers():
'''
array of all internal servers
'''
env.hosts=[]
# centeral config/management/ldap server
def config_host():
'''
This is a string with your management server
'''
env.hosts=''
env.user='root'
# hosts should be single quoted and comma seperated
def xen_servers():
'''
list of all xen servers (ip/dns),
consider only running against one in each pool
'''
env.user='root'
env.hosts=[]
# never store pass in text will get later from prompt
ops_pass=''
ops_pass_confirm=''
admin_pass=''
admin_pass_confirm=''
# kills any zombie processes
# cleans up processes
def purgeZombiesWithFire(process):
'''
cleans up a process. usage: fab -f fabfile.py -H some,hosts purgeZombiesWithFire:process=collectd
'''
env.user='rightscale'
sudo('service ' + process + ' stop; wait 5; pkill -9 ' + process + '; ps aux | grep ' + process + '; service ' + process + ' start; ps aux| grep ' + process + ';')
def restartService(service, action):
'''
restarts a service on a deb/ubuntu/centos/rhel system.
can work with services regardless of location or mechanism.
'''
initd='/etc/init.d/'
service_location=run('which service')
print 'service_location: ',service_location
if exists(service_location):
print 'using service method'
sudo(service + ' ' + action)
else:
print 'using init.d method'
sudo(initd + service + ' ' + action)
def dnsRestart():
'''
checks if you have bind9/dnsmasq and restart respective service
'''
dnsmasq_exists=run('dpkg --get-selections | awk \'{ print $1 }\' | grep --line-regexp "dnsmasq" | grep wc -l')
bind9_exists=run('dpkg --get-selections | awk \'{ print $1 }\' | grep --line-regexp "bind9" | grep wc -l')
resolvconf_exists('dpkg --get-selections | awk \'{ print $1 }\' | grep --line-regexp "resolvconf" | grep wc -l')
if dnsmasq_exists == 1:
sudo('/etc/init.d/dnsmasq restart')
if bind_exists == 1:
sudo('/etc/init.d/bind9 restart')
if resolvconf_exists == 1:
sudo('/etc/init.d/resolvconf restart')
sudo('/etc/init.d/bind9 restart')
# pings host and determines if live
def pingHost():
'''
pings a host let you know results, usage pingHost:ip=ip/dns
'''
for h in env.hosts:
response = os.system("ping -c 1 " + h)
if response == 0:
# ping is sucess, machine is live
print 'there is a live host on this ip address'
confirm = raw_input('confirm you wish to proceed [y/n]: ')
# retatard proof script (force user to confirm changes)
while confirm != 'y':
confirm = raw_input('confirm you wish to proceed [y/n]: ')
print "confirm: ", confirm
else:
print 'there is no live host on this ip address'
def configDynIfaceStatic(iface_x):
'''
reconfigs given iface from dynamic to static, using jinja template
'''
# overide defualt user
# env.user='root'
# get IP of server logged into
env.iface_x=iface_x
oldIP = run('ifconfig ' + env.iface_x + ' | grep "inet addr:" | cut -d: -f2 | awk \'{ print $1}\'')
run('whoami')
# set dns_servers to empty array
env.dns_servers=[]
# getting desired IP
env.server_ip=raw_input('please enter desired IP address: if it will remain the same press enter.')
if env.server_ip=='':
env.server_ip=oldIP
# check if host is alive
print 'pinging ', env.server_ip
pingHost()
# getting gateway, netmask, dns-servers
env.gateway = sudo('route -n | head -n 3 | awk \'{print $2}\' | egrep -v "IP|Gateway"')
env.netmask = sudo('ifconfig ' + env.iface_x + ' | grep Mask | cut -d: -f4')
num_dns=sudo('cat /etc/resolv.conf | grep ^nameserver | awk \'{print $2}\' | wc -l')
env.dns_servers=sudo("cat /etc/resolv.conf | grep ^nameserver | awk \'{print $2}\'").splitlines()
# modify /etc/network/interfaces
source_file = 'templates/interfaces'
destination_file='/etc/network/interfaces'
upload_template(source_file, destination_file, use_jinja=True, context=env, mode=0644,use_sudo=True)
run('cat /etc/network/interfaces')
confirm = raw_input('please confirm that ip output of network interfaces is good [y/n]: ')
# fail count
fail_count=0
# while you did not confirm network configs are good
while confirm != 'y':
fail_count += 1
env.server_ip=raw_input('You fool! your fail count is currently ' + str(fail_count) + '. Input desired ip or I shall taunt you again: ')
# reupload template with CORRECT ip
upload_template(source_file, destination_file, use_jinja=True, context=env, mode=0644,use_sudo=True)
# sudo('sed -i "s|' + config_host + '|' + new_ip + '|g"'' /etc/network/interfaces')
run('cat /etc/network/interfaces')
# confirm working?
confirm = raw_input('please confirm that ip output of network interfaces is good [y/n]: ')
else:
print 'restarting networking'
# sudo('service networking restart')
# will configure existing servers with ldap client packages/files to authenticate
def ldapClientConfig(rootbinddn,ldapip):
'''
installs/configures ldap clients based on template. rootbinddn should looki like admin.domain.tld
'''
# installs neccessary packages
sudo('DEBCONF_FRONTEND="noninteractive" apt-get install -y libpam-ldap libnss-ldap nss-updatedb libnss-db')
# setting up vars for ldap.conf template
ldapip=ldap_ip
env.base_dc=basedc.split('.',3)
# uploading templates
upload_template('templates/ldap/nsswitch', '/etc/nsswitch', use_jinja=True, context=None, mode=0644,use_sudo=True)
# setting base pam.d base
pb='/etc/pam.d/'
upload_template('templates/ldap/pam.d/common-account', pb +'common-account', use_jinja=True, context=None, mode=0644,use_sudo=True)
upload_template('templates/ldap/pam.d/common-auth', pb +'common-auth', use_jinja=True, context=None, mode=0644,use_sudo=True)
upload_template('templates/ldap/pam.d/common-password', pb +'common-password', use_jinja=True, context=None, mode=0644,use_sudo=True)
upload_template('templates/ldap/pam.d/common-session', pb +'common-session', use_jinja=True, context=None, mode=0644,use_sudo=True)
upload_template('templates/ldap/ldap.conf', '/etc/ldap.conf', use_jinja=True, context=env, mode=0644,use_sudo=True)
# will rename servers to reflect their real name not using localhost OR ubuntu
def nameMyServer():
'''
rename the server based on user input,
to keep server named same hit enter when prompted
'''
# get ip of server
server_ip = run('ifconfig eth0 | grep "inet addr:" | cut -d: -f2 | awk \'{ print $1}\'')
# get old server name
old_server_name = run('hostname')
# get new server name
print 'if you would not like to rename this host just hit enter'
server_name = raw_input('what is the name of this server, my ip is: ' + server_ip +' : ')
# if 0 you do not wish to rename
if len(server_name) == 0:
exit(0)
# rename server
else:
# replaces old_hostname with new one in /etc/hostname
sudo('sed -i "s|' + old_server_name + '|' + server_name + '|g"'' /etc/hostname')
# replaces old_hostname with new one in /etsc/hosts
if old_server_name == 'localhost':
print 'tricky...'
sudo('sed -i "s|127.0.0.1 ' + old_server_name + '|127.0.0.1 ' + server_name + '|g"'' /etc/hosts')
sudo('sed -i 1i"127.0.0.1 localhost" /etc/hosts')
sudo('hostname '+ server_name)
else:
sudo('sed -i "s|' + old_server_name + '|' + server_name + '|g"'' /etc/hosts')
sudo('hostname ' + server_name)
# adds a users authorized key to each system / Needs to have pushed to config_host manually
# should be run like this: fab -f fab.py syncUserKeys:user_id='useryouwishtosync'
def syncUserKeys(user_id):
'''
syncs the user supplied SSH keys from config_host
'''
# get username
env.user=user_id
run('whoami')
# checks if the homedir and .ssh folder exist
if not exists('/home/'+ user_id + '/.ssh'):
run('echo need to create homedir')
# rsync the homedir ignoring host keys
run('rsync -arvP -e "ssh -o StrictHostKeyChecking=no" ' + config_host + ':~/ /home/' + user_id + '/')
else:
# already exists
run('echo homedir and ssh already set up')
# pushes root ssh keys from SOME_HOST to each host
def syncRootKeys():
'''
syncs all of roots ssh
keys/authorized_hosts/etc from config_host
'''
# overide default user
env.user='root'
run('whoami')
# rsyncs roots ssh keys from condfig_host
run('rsync -arvP -e "ssh -o StrictHostKeyChecking=no" root@' + config_host + ':~/.ssh/ ~/.ssh/')
# deletes any local instance of specified user
def delLocalAdmin(local_admin):
'''
deletes the local_admin specified on command line
'''
# overide default user
env.user='root'
run('whoami')
# checking existance
if exists('/home/' + local_admin):
# blast away
run('userdel ' + local_admin)
run('rm -Rf /home/' + local_admin)
else:
print 'these are not the droids you are looking for, move along...'
# this is for creating a local admin
def addLocalAdmin():
'''
creates a new local admin,
the password prompted must be minimum required
len of standard admin_users default is 20
'''
# overide default user
env.user='root'
run('whoami')
# prompt for user
admin_user = raw_input('enter username: ')
# these are global to ensure they persist after each host is done
global admin_pass
global admin_pass_confirm
if exists('/home/' + admin_user):
print 'ops exists not creating, will sync ops keys and exit'
syncUserKeys(admin_user)
exit(0)
# You should adjust the number below to the len of your admin pass. This is an extra protection
# against someone typing this wrong.
while len(admin_pass) >= 20:
# defines function passPrompt (lambda) to get a password and a confirm password
passPrompt = lambda: (getpass.getpass(), getpass.getpass('Retype password: '))
# these vars restpectively get the input of the password you type
print 'Passwords will not be visable!'
admin_pass, admin_pass_confirm = passPrompt()
# ensures the two passwords match
while admin_pass != admin_pass_confirm:
print 'You Fool! You must type the same pw twice!'
admin_pass, admin_pass_confirm = passPrompt()
# the password is now >= 20 chars and you have typed the same password twice
else:
print 'admin_user: ' + admin_user
print 'admin_pass: ' + admin_pass
print 'password has been saved correctly'
# creates user in sudo group, bash shell, and no password
sudo('useradd ' + admin_user + ' -G sudo -m -s /bin/bash')
# uses chpasswd to 'create' a password with the password we captured earlier
sudo('echo "%s:%s" | chpasswd' % (admin_user, admin_pass))
# creates backup dirs
def createBackUpDirs():
'''
creates backup dirs on xen SR with the tag backup-sr
'''
# needs root
env.user = 'root'
env.key_filename = "/root/.ssh/id_rsa"
hn = run('hostname')
# gets dates for file/folder name
year = datetime.date.today().strftime("%Y")
month = datetime.date.today().strftime("%m")
day = datetime.date.today().strftime("%d")
# get sr uuid NOTE: create xen tag 'backup-sr' for the SR you want to backup to
sr_uuid = run('xe sr-list tags=backup-sr | grep uuid | cut -f2 -d":" | tr -d [:blank:]')
# check if dir exists
if not exists('/var/run/sr-mount/' + str(sr_uuid) + '/' + year + '/' + month + '-' + day + '/' ):
run('mkdir -p /var/run/sr-mount/' + sr_uuid + '/' + year + '/' + month + '-' + day +'/' )
print 'created folder for today'
else:
print 'back dir for today already exists, move along'
# ensures we start with a fresh log each time
run('cat /dev/null > /var/run/sr-mount/' + sr_uuid + '/' + year + '/' + month + '-' + day +'/backup.log' )
# function gets list of VM's
def getListOfVMs():
'''
gets a list of all VM's in a xen pool
'''
# need to run as root
env.user='root'
env.key_filename = "/root/.ssh/id_rsa"
# get hostname
xs = run('hostname')
print xs
# base xen list command
xe_list = 'xe vm-list is-control-domain=false is-a-snapshot=false'
# awk statement to get the xs_name (name of the server according to xen) and uuid
awk_state = 'awk \'{if ( $0 ~ /uuid/) {uuid=$5} if ($0 ~ /name-label/) {$1=$2=$3="";vmname=$0; printf "%s:%s\\n", vmname, uuid}}\''
# sed statement gets rid of whitespace
sed_state = 'sed -e "s/^[ \t]*//"'
# this is the command run to get a list of xen hosts located on the xen server,
# it will create a file: list-of-vms-to-back-up.txt and it will be formateed
# as xs_name:uuid
list_vm_cmd = run('' + xe_list +' | '+ awk_state + ' | ' + sed_state + ' > list-vms-backup.txt')
list_of_uuid = run('awk -F":" \'{ print $2 }\' list-vms-backup.txt > list-vm-uuids-backup.txt')
list_of_names = run('awk -F":" \'{ print $1 }\' list-vms-backup.txt > list-vm-names-backup.txt')
# list_of_names = run('awk -F":" \'{ print $1 }\' list-vms-backup.txt | tr -d [:blank:] > list-vm-names-backup.txt')
run('rsync -arvP list-*.* root@' + config_host + ':/xs-backup-lists/' + xs + '/' )
# function to back uo a single VM
def backUpVM(uuid, xs_name):
'''
backs up a single VM with uuid=/UUID/OF/VM and xs_name=/NAME/OF/VM
using the follwoing methodology:
1) checks and ejects dvd from vm if neccessary
2) takes snapshot of VM
3) removes any lingering copies
4) backup vm from snapshot
5) fixes permissions
6) deletes snapshot we created...we are not slobs
'''
# need to run as root
env.user='root'
env.key_filename = "/root/.ssh/id_rsa"
today = str(datetime.date.today())
# determine start time
stime = time.time()
# cast variables as strings to protect from spaces
uuid = str(uuid)
xs_name = re.escape(str(xs_name))
# dates for file names
# gets dates for file/folder name
year = datetime.date.today().strftime("%Y")
month = datetime.date.today().strftime("%m")
day = datetime.date.today().strftime("%d")
# gets the sr base path
sr_base = '/var/run/sr-mount'
sr_uuid = str(run('xe sr-list tags=backup-sr | grep uuid | cut -f2 -d":" | tr -d [:blank:]'))
filename = str(sr_base + '/' + sr_uuid + '/'+ year + '/' + month + '-' + day + '/' + xs_name + ".xva")
#this ejects any cd
dvdstate = run('xe vm-cd-list uuid='+ uuid + ' | grep "empty ( RO)" | awk \'{print $4}\'')
if dvdstate == "false":
print 'removing any cd\'s from: ', xs_name
run('xe vm-cd-eject vm=' + xs_name)
#this deletes existsing backup if named same
print 'filename: ', filename
print 'running initial snapshot'
# this command creates a snapshot
snapshot_vm_cmd = run('xe vm-snapshot uuid=' + uuid + ' new-name-label=' + xs_name + '-' + today)
snapshot_uuid = str(commands.getoutput(snapshot_vm_cmd).replace("sh: 1: ","").replace(": not found",""))
print 'done creating snapshot: ', snapshot_uuid
# this command sets HA to NAY for template
print 'setting template for HA to NAY'
backup_vm_cmd = 'xe template-param-set is-a-template=false ha-always-run=false uuid=' + snapshot_uuid
commands.getoutput(backup_vm_cmd)
print 'done setting template options'
# delete any lingering copies
print 'checking if ' + filename + 'exists'
run('rm -f ' + filename)
# this exports the snapshot we created as a vm
backup_vm_cmd = run('xe vm-export vm=' + snapshot_uuid + ' filename=' + filename)
commands.getoutput(backup_vm_cmd)
print 'done creating clone'
run('chmod 660 ' + filename)
etime = time.time()
ttime = (etime - stime)
run('echo ' + filename + '-' + str(ttime) + ' >> ' + sr_base + '/' + sr_uuid + '/' + year + '/' + month + '-' + day + '/backup.log')
#removing old snapshot
run('xe snapshot-uninstall snapshot-uuid=' + snapshot_uuid + ' force=true')
print 'does this still exist? ', snapshot_uuid
# function to backup all vms returned by getOfListofVMs()
def backUpAllVMs():
'''
will backup VM's returned by getListofVMs()
'''
# will run as root if using -H xen_servers
env.user='root'
env.key_filename = "/root/.ssh/id_rsa"
# gets hostname of xs (label)
xs = run('hostname')
# makes sure that dir for today exists
createBackUpDirs()
# gets the list of vm
getListOfVMs()
# initialize arrays
uuids = []
names = []
# open files and get them into arrays
uuids = open('/xs-backup-lists/' + xs + '/list-vm-uuids-backup.txt').read().splitlines()
names = open('/xs-backup-lists/' + xs + '/list-vm-names-backup.txt').read().splitlines()
# resets count
count = 0
# for every uuid in uuids file
for u in range(0, len(uuids) -1):
count +=1
# backup the vm passing the uuid and xen label/name
print 'uuid: ', uuids[count]
print 'name: ', names[count]
backUpVM(uuids[count],names[count])
print 'all uuids: ', uuids
print 'all names: ', names
rotateBackUps()
# rotates backups
def rotateBackUps():
'''
rotates and deletes backups with retention policy:
1) compresses all exported vm's (not already
compressed...that would be silly)
that are older than 1 day (2days old)
2) deletes all backups older than 8 days
'''
env.user = 'root'
# gets the sr
sr_base = '/var/run/sr-mount'
sr_uuid = run('xe sr-list tags=backup-sr | grep uuid | cut -f2 -d":" | tr -d [:blank:]')
sr_path = sr_base + '/' + sr_uuid
# date stuff
year = datetime.date.today().strftime("%Y")
# this will get the dat in 'month day_of_month' format for files
file_date_cmd = run("ls -lt | awk '{print $6,$7}'")
# getting dates in same format
today = run('date +"%b %d"')
# mtime +1 is GREATER than 1 day.
remove_empty_dir = run('find ' + sr_path + '/' + year + '/' + ' -type d -empty -delete')
two_days_ago = run('find ' + sr_path + '/' + year + '/' + ' -type f -not -name "*.gz" -mtime +1 -exec gzip {} \;')
eight_days_ago = run('find ' + sr_path + '/' + year + '/' + ' -mtime +8 -delete')
remove_empty_dir = run('find ' + sr_path + '/' + year + '/' + ' -type d -empty -delete')
print 'today: ', today
print 'two days ago: ', two_days_ago
print 'older than eight days old: ', eight_days_ago