-
Notifications
You must be signed in to change notification settings - Fork 0
/
fabfile.py
668 lines (536 loc) · 20.2 KB
/
fabfile.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
# vim: ai ts=4 sts=4 et sw=4 ft=python fdm=indent et foldlevel=0
# fabric task file for deploying a new CentOS 7 AWS EC2 instance
#
# usage:
# fab it (deploys and provision a new EC2 instance)
# fab destroy (destroys the current EC2 instance)
# fab up (boots an existing or a new EC2 instance)
# fab down (shutdowns an existing EC2 instance)
#
# state is kept locally in a file called state.json, it contains metadata
# related to the existing EC2 instance.
#
# the following environment variables must be set:
# AWS_AMI
# AWS_INSTANCE_TYPE
# AWS_ACCESS_KEY_ID
# AWS_ACCESS_KEY_FILENAME
# AWS_ACCESS_KEY_PAIR
# AWS_ACCESS_REGION
# AWS_SECRET_ACCESS_KEY
from fabric.api import sudo, local, warn_only, task, env, execute
from subprocess import check_output
from time import sleep
from fabric.colors import green as _green, yellow as _yellow, red as _red
import os
import json
import socket
from textwrap import dedent
from urlparse import urljoin
try:
import boto.ec2
from boto.ec2.blockdevicemapping import BlockDeviceType
from boto.ec2.blockdevicemapping import BlockDeviceMapping
from boto.ec2.blockdevicemapping import EBSBlockDeviceType
except:
# we assume we are running in a virtualenv
install_python_module('boto')
import boto.ec2
def connect_to_ec2():
""" returns a connection object to AWS EC2 """
conn = boto.ec2.connect_to_region(env.ec2_region,
aws_access_key_id=env.ec2_key,
aws_secret_access_key=env.ec2_secret)
return conn
def is_there_state():
""" checks is there is valid state available on disk """
if os.path.isfile('data.json'):
return True
else:
return False
def is_ssh_available(host, port=22):
""" checks if ssh port is open """
s = socket.socket()
try:
s.connect((host, port))
return True
except socket.error, e:
return False
def wait_for_ssh(host, port=22, timeout=600):
""" probes the ssh port and waits until it is available """
yellow('waiting for ssh...')
for iteration in xrange(1, timeout):
if is_ssh_available(host, port):
green('ssh is now available.')
return True
else:
yellow('waiting for ssh...')
sleep(1)
def green(msg):
""" prints it back in green """
print(_green(msg))
def yellow(msg):
""" prints it back in yellow """
print(_yellow(msg))
def red(msg):
""" prints it back in red """
print(_red(msg))
def create_server():
"""
Creates EC2 Instance and saves it state in a local json file
"""
# looks for an existing 'data.json' file, so that we don't start
# additional ec2 instances when we don't need them.
#
if is_there_state():
return True
else:
conn = connect_to_ec2()
print(_green("Started..."))
print(_yellow("...Creating EC2 instance..."))
# we need a larger boot device to store our cached images
dev_sda1 = EBSBlockDeviceType()
dev_sda1.size = 120
bdm = BlockDeviceMapping()
bdm['/dev/sda1'] = dev_sda1
# get an ec2 ami image object with our choosen ami
image = conn.get_all_images(env.ec2_ami)[0]
# start a new instance
reservation = image.run(1, 1,
key_name=env.ec2_key_pair,
security_groups=env.ec2_security,
block_device_map = bdm,
instance_type=env.ec2_instancetype)
# and get our instance_id
instance = reservation.instances[0]
# add a tag to our instance
conn.create_tags([instance.id], {"Name": env.ec2_instance_name})
# and loop and wait until ssh is available
while instance.state == u'pending':
yellow("Instance state: %s" % instance.state)
sleep(10)
instance.update()
wait_for_ssh(instance.public_dns_name)
green("Instance state: %s" % instance.state)
green("Public dns: %s" % instance.public_dns_name)
# finally save the details or our new instance into the local state file
save_state_locally(instance.id)
def save_state_locally(instance_id):
""" queries EC2 for details about a particular instance_id and
stores those details locally
"""
data = get_ec2_info(instance_id)
with open('data.json', 'w') as f:
json.dump(data, f)
def load_state_from_disk():
""" saves state in a local 'data.json' file so that it can be
reused between fabric runs.
"""
if is_there_state():
import json
with open('data.json', 'r') as f:
data = json.load(f)
return data
else:
return False
@task
def print_ec2_info():
""" outputs information about our EC2 instance """
_state = load_state_from_disk()
if _state:
data = get_ec2_info(_state['id'])
green("Instance state: %s" % data['state'])
green("Public dns: %s" % data['public_dns_name'])
green("Ip address: %s" % data['ip_address'])
green("volume: %s" % data['volume'])
green("user: %s" % env.user)
green("ssh -i %s %s@%s" % (env.key_filename,
env.user,
data['ip_address']))
def get_ec2_info(instance_id):
""" queries EC2 for details about a particular instance_id
"""
conn = connect_to_ec2()
instance = conn.get_only_instances(
filters={'instance_id': instance_id}
)[0]
data = {}
data['public_dns_name'] = instance.public_dns_name
data['id'] = instance.id
data['ip_address'] = instance.ip_address
data['architecture'] = instance.architecture
data['state'] = instance.state
try:
volume = conn.get_all_volumes(
filters={'attachment.instance-id': instance.id})[0].id
data['volume'] = volume
except:
data['volume'] = ''
return data
@task
def rsync():
""" syncs the src code to the remote box """
from fabric.context_managers import lcd
green('syncing code to remote box...')
data = load_state_from_disk()
if 'SOURCE_PATH' in os.environ:
with lcd(os.environ['SOURCE_PATH']):
local("rsync -a "
"--info=progress2 "
"--exclude .git "
"--exclude .tox "
"--exclude .vagrant "
"--exclude venv "
". "
"-e 'ssh -C -i " + env.ec2_key_filename + "' "
"%s@%s:" % (env.user, data['ip_address']))
else:
print('please export SOURCE_PATH before running rsync')
exit(1)
@task
def status():
print_ec2_info()
@task
def up():
""" boots an existing ec2_instance, or creates a new one if needed """
# if we don't have a state file, then its likely we need to create a new
# ec2 instance.
if is_there_state() is False:
create_server()
else:
conn = connect_to_ec2()
# there is a data.json file, which contains our ec2 instance_id
data = load_state_from_disk()
# boot the ec2 instance
instance = conn.start_instances(instance_ids=[data['id']])[0]
while instance.state != "running":
print(_yellow("Instance state: %s" % instance.state))
sleep(10)
instance.update()
# the ip_address has changed so we need to get the latest data from ec2
data = get_ec2_info(data['id'])
# and make sure we don't return until the instance is fully up
wait_for_ssh(data['ip_address'])
# lets update our local state file with the new ip_address
save_state_locally(instance.id)
env.hosts = data['ip_address']
print_ec2_info()
@task
def down():
""" shutdown of an existing EC2 instance """
conn = connect_to_ec2()
# checks for a valid state file, containing the details our ec2 instance
if is_there_state() is False:
# we can't shutdown the instance, if we don't know which one it is
return False
else:
# get the instance_id from the state file, and stop the instance
data = load_state_from_disk()
instance = conn.stop_instances(instance_ids=[data['id']])[0]
while instance.state != "stopped":
print(_yellow("Instance state: %s" % instance.state))
sleep(10)
instance.update()
@task
def halt():
down()
@task
def destroy():
""" terminates the instance """
if is_there_state() is False:
return True
else:
conn = connect_to_ec2()
_state = load_state_from_disk()
data = get_ec2_info(_state['id'])
instance = conn.terminate_instances(instance_ids=[data['id']])[0]
yellow('destroying instance ...')
while instance.state != "terminated":
print(_yellow("Instance state: %s" % instance.state))
sleep(10)
instance.update()
volume = data['volume']
if volume:
yellow('destroying EBS volume ...')
conn.delete_volume(volume)
os.unlink('data.json')
@task
def terminate():
destroy()
def install_python_module(name):
""" instals a python module using pip """
local('pip install %s' % name)
def disable_selinux():
""" disables selinux """
from fabric.contrib.files import sed, contains
if contains(filename='/etc/selinux/config',
text='SELINUX=enforcing'):
sed('/etc/selinux/config',
'SELINUX=enforcing', 'SELINUX=disabled', use_sudo=True)
if contains(filename='/etc/selinux/config',
text='SELINUXTYPE=enforcing'):
sed('/etc/selinux/config',
'SELINUXTYPE=enforcing', 'SELINUX=targeted', use_sudo=True)
def yum_install(**kwargs):
"""
installs a yum package
"""
for pkg in list(kwargs['packages']):
if is_package_installed(pkg) is False:
if 'repo' in kwargs:
green("installing %s from repo %s ..." % (pkg, repo))
sudo("yum install -y --quiet --enablerepo=%s %s" % (repo, pkg))
else:
green("installing %s ..." % pkg)
sudo("yum install -y --quiet %s" % pkg)
def yum_install_from_url(pkg_name, url):
""" installs a pkg from a url
p pkg_name: the name of the package to install
p url: the full URL for the rpm package
"""
from fabric.api import settings
from fabric.context_managers import hide
if is_package_installed(pkg_name) is False:
green("installing %s from %s" % (pkg_name, url))
with settings(hide('warnings', 'running', 'stdout', 'stderr'),
warn_only=True, capture=True):
result = sudo("yum install --quiet -y %s" % url)
if result.return_code == 0:
return True
elif result.return_code == 1:
return False
else: #print error to user
print result
raise SystemExit()
def systemd(service, start=True, enabled=True, unmask=False):
""" manipulates systemd services """
from fabric.api import settings
from fabric.context_managers import hide
with settings(hide('warnings', 'running', 'stdout', 'stderr'),
warn_only=True, capture=True):
if start:
sudo('systemctl start %s' % service)
else:
sudo('systemctl stop %s' % service)
if enabled:
sudo('systemctl enable %s' % service)
else:
sudo('systemctl disable %s' % service)
if unmask:
sudo('systemctl unmask %s' % service)
def sleep_for_one_minute():
local(sleep(60))
def reboot():
sudo('shutdown -r now')
def is_package_installed(pkg):
""" checks if a particular rpm package is installed """
from fabric.api import settings
from fabric.context_managers import hide
with settings(hide('warnings', 'running', 'stdout', 'stderr'),
warn_only=True, capture=True):
result = sudo("rpm -q %s" % pkg)
if result.return_code == 0:
return True
elif result.return_code == 1:
return False
else: #print error to user
print result
raise SystemExit()
def install_os_updates():
""" installs OS updates """
sudo("yum -y --quiet update")
def install_development_packages():
""" Update the kernel and install some development tools necessary for
building the ZFS kernel module. """
yum_install(packages = ["kernel-devel", "kernel", "kernel-headers",
"dkms", "gcc", "git", "make", "psutils-perl", "lsof", "rsync"])
def add_epel_yum_repository():
""" Install a repository that provides epel packages/updates """
yum_install(packages=["epel-release"])
def arch():
""" returns the current cpu archictecture """
from fabric.api import settings
from fabric.context_managers import hide
with settings(hide('warnings', 'running', 'stdout', 'stderr'),
warn_only=True, capture=True):
result = sudo('rpm -E %dist').strip()
return result
def create_docker_group():
""" creates the docker group """
from fabric.contrib.files import contains
if not contains('/etc/group', 'docker', use_sudo=True):
sudo("groupadd docker")
def enable_firewalld_service():
""" install and enables the firewalld service """
yum_install(packages=['firewalld'])
systemd(service='firewalld', unmask=True)
def add_firewall_service(service, permanent=True):
""" adds a firewall rule """
yum_install(packages=['firewalld'])
from fabric.api import settings
from fabric.context_managers import hide
with settings(hide('warnings', 'running', 'stdout', 'stderr'),
warn_only=True, capture=True):
p = ''
if permanent:
p = '--permanent'
sudo('firewall-cmd --add-service %s %s' % (service, p))
def add_firewall_port(port, permanent=True):
""" adds a firewall rule """
yum_install(packages=['firewalld'])
from fabric.api import settings
from fabric.context_managers import hide
with settings(hide('warnings', 'running', 'stdout', 'stderr'),
warn_only=True, capture=True):
p = ''
if permanent:
p = '--permanent'
sudo('firewall-cmd --add-port %s %s' % (port, p))
@task
def ssh(*cli):
from itertools import chain
""" opens a ssh shell to the host """
data = load_state_from_disk()
local('ssh -t -i %s %s@%s %s' % (env['ec2_key_filename'],
env['user'], data['ip_address'],
"".join(chain.from_iterable(cli))))
def install_docker():
""" installs docker """
yum_install(packages=['docker', 'docker-registry'])
systemd('docker.service')
def pull_docker_image(image):
""" pulls a docker image """
from fabric.api import settings
from fabric.context_managers import hide
with settings(hide('warnings', 'running', 'stdout', 'stderr'),
warn_only=True, capture=True):
result = sudo("docker pull %s" % image)
if result.return_code == 0:
return True
elif result.return_code == 1:
return False
else: #print error to user
print result
raise SystemExit()
def cache_docker_images():
""" pulls some docker images locally """
green('refreshing docker images...')
for image in ["busybox",
"clusterhq/mongodb",
"redis",
"clusterhq/flask",
"python:2.7-slim"]:
pull_docker_image(image)
def check_for_missing_environment_variables():
""" double checks that the minimum environment variables have been setup """
env_var_missing = []
for env_var in ['AWS_KEY_PAIR',
'AWS_KEY_FILENAME',
'AWS_SECRET_ACCESS_KEY',
'AWS_ACCESS_KEY_ID']:
if not env_var in os.environ:
env_var_missing.append(env_var)
if env_var_missing:
print('the following environment variables must be set:')
for env_var in env_var_missing:
print(env_var)
return True
def does_container_exist(container):
from fabric.api import settings
with settings(warn_only=True):
result = sudo('docker inspect %s' % container)
print('*********************************************')
red(result.return_code)
if result.return_code is 0:
return True
else:
return False
def get_image_id(image):
result = sudo("docker images | grep %s | awk '{print $3}'" % image)
return result
def get_container_id(container):
result = sudo("docker ps -a | grep %s | awk '{print $1}'" % container)
return result
def does_image_exist(image):
from fabric.api import settings
with settings(warn_only=True):
result = sudo('docker images')
if image in result:
return True
else:
return False
def remove_image(image):
sudo('docker rmi -f %s' % get_image_id(image))
def remove_container(container):
sudo('docker rm -f %s' % get_container_id(container))
def git_clone_grafana():
sudo('rm -rf graphite_docker')
sudo('git clone https://github.com/SamSaffron/graphite_docker.git')
def build_metrics_platform_image():
sudo('cd graphite_docker && docker build -t metrics_platform_img .')
@task
def deploy_metrics_platform():
if does_container_exist('metrics_platform'):
yellow('removing old data-platform container ...')
remove_container('metrics_platform')
if does_image_exist('metrics_platform_img'):
yellow('removing old metrics_platform image ...')
remove_image('metrics_platform_img')
git_clone_grafana()
build_metrics_platform_image()
sudo('test -e `pwd`/data || mkdir `pwd`/data')
sudo('chmod 777 `pwd`/data')
sudo('docker run -d --name metrics_platform -it -v `pwd`/data:/data '
'-p 14000:80 -p 14001:3000 -p 2003:2003 -p 8125:8125/udp'
' metrics_platform_img')
add_firewall_port('14000/tcp')
add_firewall_port('14001/tcp')
add_firewall_port('8125/udp')
add_firewall_port('2003/tcp')
@task
def it():
""" runs the full stack """
execute(up)
# ec2 hosts get their ip addresses using dhcp, we need to know the new
# ip address of our box before we continue our provisioning tasks.
# we load the state from disk, and store the ip in ec2_host#
ec2_host="%s@%s" %(env.user, load_state_from_disk()['ip_address'])
execute(disable_selinux, hosts=ec2_host)
execute(down, hosts=ec2_host)
execute(up, hosts=ec2_host)
ec2_host="%s@%s" %(env.user, load_state_from_disk()['ip_address'])
execute(install_os_updates, hosts=ec2_host)
execute(add_epel_yum_repository, hosts=ec2_host)
execute(install_development_packages, hosts=ec2_host)
execute(enable_firewalld_service, hosts=ec2_host)
execute(install_docker, hosts=ec2_host)
execute(create_docker_group, hosts=ec2_host)
execute(deploy_metrics_platform, hosts=ec2_host)
def main():
""" loads our environment variables into the env dict.
checks for existing local state and loads the env dict with the state
from the last fabric run.
We store the state in a local file as we need to keep track of the
ec2 instance id and ip_address so that we can run provision multiple
times
"""
if check_for_missing_environment_variables():
exit(1)
env.ec2_ami = os.getenv('AWS_AMI', 'ami-c7d092f7')
env.ec2_instance_name = 'data-platform'
env.ec2_instancetype = os.getenv('AWS_INSTANCE_TYPE', 't2.micro')
env.ec2_key = os.environ['AWS_ACCESS_KEY_ID']
env.ec2_key_filename = os.environ['AWS_KEY_FILENAME'] # path to ssh key
env.ec2_key_pair = os.environ['AWS_KEY_PAIR']
env.ec2_region = os.getenv('AWS_REGION', 'us-west-2')
env.ec2_secret = os.environ['AWS_SECRET_ACCESS_KEY']
env.ec2_security = ['data-platform']
env.user = 'centos'
env.disable_known_hosts = True
env.key_filename = env.ec2_key_filename
if is_there_state() is False:
pass
else:
data = load_state_from_disk()
env.hosts = data['ip_address']
main()