forked from qszhuan/datawinners
/
fabfile.py
367 lines (272 loc) · 12.9 KB
/
fabfile.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
# vim: ai ts=4 sts=4 et sw=4 encoding=utf-8
import getpass
from fabric.api import run, env
from fabric.context_managers import cd, settings
import os
from datetime import datetime
DATAWINNERS = 'datawinners'
MANGROVE = 'mangrove'
TODAY_IN_UTC = str(datetime.utcnow().date()).replace('-', '')
ENVIRONMENT_CONFIGURATIONS = {
"showcase": "showcase_local_settings.py",
"test": "test_local_settings.py",
"master": "showcase_local_settings.py",
"beta": "local_settings.py",
"production": "prod_local_settings.py",
"ec2": "ec2_local_settings.py",
"qa": "local_settings_qa.py"
}
ENVIRONMENT_VES = {
"showcase": "/home/mangrover/ve",
"production": "/home/mangrover/ve",
"ec2": "/home/mangrover/.virtualenvs/datawinners",
"qa": "/home/twer/virtual_env/datawinner",
"test": "/home/twer/virtual_env/datawinner",
}
ENVIRONMENT_TOMCAT = {
"showcase": "/home/mangrover/tomcat",
"production": "/home/mangrover/tomcat7",
"ec2": "/home/mangrover/tomcat7",
}
ENVIRONMENT_JENKINS_JOB = {
MANGROVE : 'Mangrove-develop',
DATAWINNERS: 'Datawinners'
}
def git_clone_datawinners_if_not_present(code_dir):
if run("test -d %s" % code_dir).failed:
run("git clone git://github.com/mangroveorg/datawinners.git %s" % code_dir)
def git_clone_mangrove_if_not_present(code_dir):
if run("test -d %s" % code_dir).failed:
run("git clone git://github.com/mangroveorg/mangrove.git %s" % code_dir)
def activate_and_run(virtual_env, command):
run('source %s/bin/activate && %s' % (virtual_env, command))
def branch_exists(branch_name):
return not run("git branch -a|grep %s" % branch_name).failed
def sync_branch(branch):
run("git checkout %s" % branch)
run("git pull origin %s" % branch)
def delete_if_branch_exists(build_number):
if branch_exists(build_number):
run("git branch -D %s" % build_number)
def restart_gunicorn(virtual_env):
if gunicorn_is_running():
stop_gunicorn()
start_gunicorn(virtual_env)
def gunicorn_is_running():
return not run("pgrep gunicorn").failed
def stop_gunicorn():
run("kill -9 `pgrep gunicorn`")
def start_gunicorn(virtual_env):
activate_and_run(virtual_env, "gunicorn_django -D -b 0.0.0.0:8000 --pid=mangrove_gunicorn")
def restart_servers():
stop_servers()
start_servers()
def stop_servers():
run("sudo /etc/init.d/nginx stop")
run("sudo /etc/init.d/uwsgi stop")
print 'server stopped...'
def start_servers():
run("sudo /etc/init.d/uwsgi start")
run("sudo /etc/init.d/nginx start")
print 'server started..'
def set_mangrove_commit_sha(branch, mangrove_build_number):
if mangrove_build_number == 'lastSuccessfulBuild':
mangrove_build_number = run("curl http://178.79.163.33:8080/job/Mangrove-%s/lastSuccessfulBuild/buildNumber" % (branch,))
run("echo 'Checking the mangrove commit sha for build number %s'" % mangrove_build_number)
run("export MANGROVE_COMMIT_SHA=`curl -s http://178.79.163.33:8080/job/Mangrove-%s/%s/artifact/last_successful_commit_sha`" % (
branch, mangrove_build_number))
run("echo MANGROVE_COMMIT_SHA=$MANGROVE_COMMIT_SHA" )
def set_datawinner_commit_sha(datawinner_build_number):
if datawinner_build_number == 'lastSuccessfulBuild':
datawinner_build_number = run("curl http://178.79.163.33:8080/job/Datawinners/lastSuccessfulBuild/buildNumber")
run("echo 'Checking the datawinner commit sha for build number %s'" % datawinner_build_number)
run("export DATAWINNER_COMMIT_SHA=`curl -s http://178.79.163.33:8080/job/Datawinners/%s/artifact/last_successful_commit_sha`" % (
datawinner_build_number))
run("echo DATAWINNER_COMMIT_SHA=$DATAWINNER_COMMIT_SHA" )
def check_out_mangrove_code(mangrove_build_number, mangrove_code_dir, branch, virtual_env):
git_clone_mangrove_if_not_present(mangrove_code_dir)
with cd(mangrove_code_dir):
run("cd %s" % mangrove_code_dir)
run("git reset --hard HEAD")
sync_branch(branch)
delete_if_branch_exists(mangrove_build_number)
run("git checkout -b %s $MANGROVE_COMMIT_SHA" % (mangrove_build_number, ))
run("git checkout .")
activate_and_run(virtual_env, "pip install -r requirements.pip")
activate_and_run(virtual_env, "python setup.py develop")
def check_out_datawinners_code(datawinner_build_number, datawinners_code_dir, branch, virtual_env):
git_clone_datawinners_if_not_present(datawinners_code_dir)
with cd(datawinners_code_dir):
run("cd %s" % datawinners_code_dir)
run("git reset --hard HEAD")
sync_branch(branch)
delete_if_branch_exists(datawinner_build_number)
run("git checkout -b %s $DATAWINNER_COMMIT_SHA" % (datawinner_build_number, ))
run("git checkout .")
activate_and_run(virtual_env, "pip install -r requirements.pip")
def deploy(mangrove_build_number, datawinner_build_number, home_dir, virtual_env, branch="develop", environment="showcase"):
"""build_number : hudson build number to be deployed
home_dir: directory where you want to deploy the source code
virtual_env : path to your virtual_env folder
"""
set_mangrove_commit_sha(branch, mangrove_build_number)
set_datawinner_commit_sha(datawinner_build_number)
mangrove_code_dir = home_dir + '/mangrove'
datawinners_code_dir = home_dir + '/datawinners'
with settings(warn_only=True):
check_out_mangrove_code(mangrove_build_number, mangrove_code_dir, branch, virtual_env)
check_out_datawinners_code(datawinner_build_number, datawinners_code_dir, branch, virtual_env)
with cd(datawinners_code_dir + '/datawinners'):
run("cd %s/datawinners" % datawinners_code_dir)
run("cp %s local_settings.py" % (ENVIRONMENT_CONFIGURATIONS[environment],))
activate_and_run(virtual_env, "python manage.py syncdb --noinput")
activate_and_run(virtual_env, "python manage.py migrate")
activate_and_run(virtual_env, "python manage.py recreatedb")
activate_and_run(virtual_env, "python manage.py compilemessages")
activate_and_run(virtual_env, "python manage.py loadshapes")
if environment == "test":
restart_gunicorn(virtual_env)
else:
restart_servers()
def killfirefox():
with settings(warn_only=True):
run("killall firefox")
def showcase():
env.user = "mangrover"
env.hosts = ["178.79.161.90"]
env.key_filename = ["/var/lib/jenkins/.ssh/id_rsa"]
env.warn_only = True
def qa():
env.user = "twer"
env.hosts = ["10.18.2.237"]
env.key_filename = ["/home/dw/.ssh/id_rsa"]
env.warn_only = True
def test():
env.user = "twer"
env.hosts = ["10.18.2.237"]
env.key_filename = ["/Users/twer/.ssh/id_rsa"]
env.warn_only = True
def local():
env.user = "mangrover"
env.hosts = ["127.0.0.1"]
env.key_filename = ["/var/lib/jenkins/.ssh/id_rsa"]
def production():
env.user = "mangrover"
env.hosts = ["178.79.185.34"]
env.key_filename = ["/var/lib/jenkins/.ssh/id_rsa"]
env.warn_only = True
def ec2():
env.user = "mangrover"
env.hosts = ["54.243.31.50"]
env.key_filename = ["/var/lib/jenkins/.ssh/id_rsa"]
env.warn_only = True
def local_test():
env.user = getpass.getuser()
env.hosts = ["localhost"]
env.key_filename = ["%s/.ssh/id_rsa" % os.getenv("HOME")]
env.warn_only = True
def anonymous():
run("uname -a")
def commit_sha_from_build_number(jenkins_job_name, build_number):
if build_number == 'lastSuccessfulBuild':
build_number = run("curl http://178.79.163.33:8080/job/%s/lastSuccessfulBuild/buildNumber" % (jenkins_job_name,))
print("Retrieving the commit sha for build number %s of jenkins job %s" % (build_number, jenkins_job_name,))
commit_sha = run("curl -s http://178.79.163.33:8080/job/%s/%s/artifact/last_successful_commit_sha" % (jenkins_job_name, build_number))
print("%s_commit_sha: %s" % (jenkins_job_name, commit_sha))
return commit_sha
def _project_dir(code_dir, project_name):
project_dir = os.path.join(code_dir, project_name)
if run("test -d %s" % project_dir).failed:
run('git clone git://github.com/mangroveorg/%s.git %s' % (project_name, project_dir))
return project_dir
def checkout_project(context, project_name):
run("git reset --hard HEAD")
run("git checkout develop")
run("git pull --rebase")
if context.branch in ['develop', 'origin/develop']:
start_point = commit_sha_from_build_number(ENVIRONMENT_JENKINS_JOB[project_name], context.build_numbers[project_name])
else:
start_point = context.branch
run("git checkout -B %s %s" % (TODAY_IN_UTC, start_point))
def install_requirement(virtual_env):
activate_and_run(virtual_env, "pip install -r requirements.pip")
def post_checkout_datawinners(virtual_env):
install_requirement(virtual_env)
def post_checkout_mangrove(virtual_env):
install_requirement(virtual_env)
activate_and_run(virtual_env, "python setup.py develop")
def deploy_project(context, project_name, post_checkout_function):
with cd(_project_dir(context.code_dir, project_name)):
checkout_project(context, project_name)
post_checkout_function(context.virtual_env)
def check_out_latest_custom_reports_code_for_production(code_dir):
custom_reports_dir = _project_dir(code_dir, "custom_reports")
with cd(custom_reports_dir):
run("git reset --hard HEAD")
run("git checkout develop")
run("git pull origin develop")
if not run("git branch -a|grep %s" % TODAY_IN_UTC).failed:
run("git branch -D %s" % TODAY_IN_UTC)
run("git checkout -b %s HEAD" % TODAY_IN_UTC)
run("git checkout .")
def _make_sure_code_dir_exists(context):
run('mkdir -p %s' % context.code_dir)
def replace_setting_file_for_environment(environment):
run("cp %s local_settings.py" % ENVIRONMENT_CONFIGURATIONS[environment])
def migrate_couchdb(context):
if context.couch_migration_file:
with cd('%s/datawinners' % context.code_dir):
activate_and_run(context.virtual_env, "python %s" % context.couch_migration_file)
def link_repo(context, link_name):
run('rm -f %s' % link_name)
run('ln -s %s %s' % (os.path.join(context.code_dir, link_name), link_name))
def _deploy_mangrove(context):
deploy_project(context, MANGROVE, post_checkout_mangrove)
link_repo(context, MANGROVE)
def _deploy_datawinners(context):
deploy_project(context, DATAWINNERS, post_checkout_datawinners)
with cd(os.path.join(context.code_dir, DATAWINNERS, DATAWINNERS)):
replace_setting_file_for_environment(context.environment)
activate_and_run(context.virtual_env, "python manage.py migrate")
activate_and_run(context.virtual_env, "python manage.py compilemessages")
activate_and_run(context.virtual_env, "python manage.py syncviews syncall")
migrate_couchdb(context)
link_repo(context, DATAWINNERS)
class Context(object):
def __init__(self, mangrove_build_number, datawinner_build_number, code_dir, environment, branch_name, virtual_env, couch_migration_file):
self.build_numbers = {MANGROVE: mangrove_build_number, DATAWINNERS: datawinner_build_number}
self.code_dir = code_dir
self.environment = environment
self.branch = branch_name
self.virtual_env = virtual_env
self.couch_migration_file = couch_migration_file
def production_deploy(mangrove_build_number="lastSuccessfulBuild",
datawinner_build_number="lastSuccessfulBuild",
code_dir="/home/twer/workspace",
environment = 'showcase',
branch_name='develop',
couch_migration_file=None):
virtual_env = ENVIRONMENT_VES[environment]
context = Context(mangrove_build_number, datawinner_build_number, code_dir, environment, branch_name, virtual_env, couch_migration_file)
_make_sure_code_dir_exists(context)
_deploy_mangrove(context)
_deploy_datawinners(context)
remove_cache(context)
restart_servers()
def custom_reports_deploy(code_dir, environment = 'showcase'):
check_out_latest_custom_reports_code_for_production(code_dir)
with cd('%s/bin/' % ENVIRONMENT_TOMCAT[environment]):
run('./catalina.sh stop')
with cd('%s/webapps/birt-viewer/' % ENVIRONMENT_TOMCAT[environment]):
run('rm crs')
run('ln -s %s/custom_reports/crs/ crs' % code_dir)
with cd('%s/bin/' % ENVIRONMENT_TOMCAT[environment]):
run('./catalina.sh start')
def deploy_to_qa():
production_deploy(code_dir="/home/twer/workspace",environment="qa")
def test_deploy_against_qa_machine():
production_deploy(code_dir="/home/twer/workspace_for_script_test", branch_name="origin/release",environment="test")
def remove_cache(context):
with cd(os.path.join(context.code_dir, DATAWINNERS, DATAWINNERS, 'media')):
run('rm -rf CACHE/js/*')
run('rm -rf CACHE/css/*')