-
Notifications
You must be signed in to change notification settings - Fork 1
/
nightlybuilder.py
executable file
·332 lines (299 loc) · 10.5 KB
/
nightlybuilder.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
#!/usr/bin/env python
# Andrew Sutherland <dr3wsuth3rland@gmail.com>
import argparse
from datetime import datetime
import json
import logging
import os
import shutil
import subprocess
import Queue
# local
from drewis import __version__
from drewis import html,rsync,android
from drewis.utils import *
# handle commandline args
parser = argparse.ArgumentParser(description="Drew's builder script")
parser.add_argument('--version', action='version', version='%(prog)s ' + __version__)
parser.add_argument('target', help="Device(s) to build",
nargs='+')
parser.add_argument('--source', help="Path to android tree",
default=os.getcwd())
parser.add_argument('--host', help="Hostname for upload")
parser.add_argument('--user', help="Username for upload host")
parser.add_argument('--remotedir', help="Remote path for uploads")
parser.add_argument('--localdir', help="Local path for uploads")
parser.add_argument('--nosync', help="Skip sync",
action="store_true")
parser.add_argument('--nobuild', help="Skip build",
action="store_true")
args = parser.parse_args()
# static vars
DATE = datetime.now().strftime('%Y.%m.%d')
def main(args):
# script logging
log_dir = os.path.join(args.source, 'nightly_logs')
try:
if not os.path.isdir(log_dir):
os.mkdir(log_dir)
except OSError:
pass
scriptlog = os.path.join(log_dir, 'scriptlog-' + DATE + '.log')
logging.basicConfig(filename=scriptlog, level=logging.INFO,
format='%(levelname)-8s %(message)s')
# for total runtime
script_start = datetime.now()
logging.info(script_start)
# set vars for uploading/mirroring
if not args.user:
droid_user = os.getenv('DROID_USER')
else:
droid_user = args.user
if not args.host:
droid_host = os.getenv('DROID_HOST')
else:
droid_host = args.host
if not args.remotedir:
droid_path = os.getenv('DROID_PATH')
else:
droid_path = args.remotedir
if not args.localdir:
droid_mirror = os.getenv('DROID_MIRROR')
if not droid_mirror:
droid_mirror = os.getenv('DROID_LOCAL_MIRROR')
else:
droid_mirror = args.localdir
# we must put the builds somewhere
if not droid_mirror:
mirroring = False
if droid_host and droid_user and droid_path:
uploading = True
else:
logging.error('DROID_MIRROR not set')
logging.error('DROID_HOST or DROID_USER or DROID_PATH not set')
logging.error('no where put builds. BAILING!!')
exit()
else:
mirroring = True
if droid_host and droid_user and droid_path:
uploading = True
else:
uploading = False
# cd working dir
previous_working_dir = os.getcwd()
os.chdir(args.source)
# we want group write
os.umask(002)
# make the remote directories
if uploading:
upload_path = os.path.join(droid_path, DATE)
try:
subprocess.check_call(['ssh', '%s@%s' % (droid_user, droid_host),
'test -d %s || mkdir -p %s' % (upload_path,upload_path)])
except subprocess.CalledProcessError as e:
logging.error('ssh returned %d while making directories' %
(e.returncode))
uploading = False
if not mirroring:
logging.error('no where to put builds. BAILING!!')
exit()
else:
# upload thread
upq = Queue.Queue()
t1 = rsync.rsyncThread(upq,
'%s@%s:%s' % (droid_user, droid_host, upload_path),
message='Uploaded')
t1.setDaemon(True)
t1.start()
if mirroring:
mirror_path = os.path.join(droid_mirror, DATE)
try:
if not os.path.isdir(mirror_path):
os.makedirs(mirror_path)
subprocess.call(['chmod','775','%s' % (mirror_path)])
except OSError as e:
logging.error('failed to make mirror dir: %s' % (e))
mirroring = False
if not uploading:
logging.error('no where to put builds. BAILING!!')
exit()
else:
# mirror thread
m_q = Queue.Queue()
t2 = rsync.rsyncThread(m_q,
mirror_path,
message='Copied')
t2.setDaemon(True)
t2.start()
#
# Syncing
#
if not args.nosync:
# common directory for all changelogs
changelog_dir = os.path.join(os.path.realpath(os.getcwd()), 'nightly_changelogs')
try:
if not os.path.isdir(changelog_dir):
os.mkdir(changelog_dir)
except OSError:
pass
# changelog
changelog = os.path.join(changelog_dir, 'changelog-' + DATE + '.log')
# sync the tree
if android.reposync():
logging.error('Sync failed. Skipping the build')
args.nobuild = True
# Remove out so we dont upload yesterdays build
if os.path.isdir('out'):
shutil.rmtree('out')
else:
android.get_changelog(DATE,changelog)
# create the html changelog
if os.path.exists(changelog):
logging.info('Created changelog for %s' % DATE)
html_changelog = os.path.join(changelog_dir, 'changelog-' + DATE + '.html')
cl = html.Create()
cl.title('Changelog')
cl.css('body {font-family:"Lucida Console", Monaco, monospace;font-size:0.9em;}')
clbody = html.parse_file(changelog)
cl.header(clbody[0])
cl.body(html.add_line_breaks(clbody[1:]))
cl.write(html_changelog)
# add changelog to rsync queues
if uploading:
upq.put(html_changelog)
if mirroring:
m_q.put(html_changelog)
else:
logging.info('Skipped sync')
#
# Building
#
# export vars for the build script
os.putenv('NIGHTLY_BUILD', 'true')
# for zip storage
if os.path.isdir('/dev/shm'):
temp_dir = '/dev/shm/tmp-nightlybuilder_zips'
else:
temp_dir = '/tmp/tmp-nightlybuilder_zips'
if not os.path.isdir(temp_dir):
os.mkdir(temp_dir)
# keep track of builds
build_start = datetime.now()
# for json manifest
json_info = []
# Targets to run through the squisher
squisher_targets = (
'passion',
'bravo',
'supersonic',
'inc',
)
# build each target
for target in args.target:
if not args.nobuild:
target_start = datetime.now()
pkg = 'otapackage'
if target in squisher_targets:
pkg = 'squishedpackage'
if android.build(target,pkg):
continue # Failed
else:
logging.info('Built %s in %s' %
(target, pretty_time(datetime.now() - target_start)))
# find and add the zips to the rsync queues
zips = []
target_out_dir = os.path.join('out', 'target', 'product', target)
if os.path.isdir(target_out_dir):
for f in os.listdir(target_out_dir):
if f.startswith('ev') and f.endswith('.zip'):
zips.append(f)
if zips:
for z in zips:
json_info.append({
'date': DATE,
'device': target,
'count': 0,
'message': 'Nightly build for %s' % target,
'md5sum': md5sum(os.path.join(target_out_dir, z)),
'name': z,
'size': os.path.getsize(os.path.join(target_out_dir, z)),
'type': 'nightly',
'location': '%s/%s' % (DATE,z),
})
shutil.copy2(os.path.join(target_out_dir, z),os.path.join(temp_dir, z))
if uploading:
upq.put(os.path.join(temp_dir, z))
if mirroring:
m_q.put(os.path.join(temp_dir, z))
else:
logging.warning('No zips found for %s' % target)
# write total buildtime
logging.info('Built all targets in %s' %
(pretty_time(datetime.now() - build_start)))
# write manifest
if json_info:
json_info.sort(key=lambda d:d['device'])
try:
f = open(os.path.join(temp_dir,'info.json'),'w')
except IOError as e:
logging.error('Failed to open info.json: %s' % (e))
else:
with f:
json.dump(json_info, f, indent=2)
if uploading:
upq.put(os.path.join(temp_dir,'info.json'))
if mirroring:
m_q.put(os.path.join(temp_dir,'info.json'))
# for website
''' # Not using this any more
if mirroring:
main_manifest = os.path.join(droid_mirror,'manifest.json')
try:
f = open(main_manifest,'r')
except IOError as e:
logging.warning('%s' % e)
else:
with f:
entries = json.load(f)
for e in json_info:
entries.append(e)
try:
f = open(main_manifest,'w')
except IOError as e:
logging.warning('%s' % e)
else:
with f:
json.dump(entries,f,indent=2)
'''
# wait for builds to finish uploading/mirroring
if mirroring:
m_q.join()
if uploading:
upq.join()
# cleanup
shutil.rmtree(temp_dir)
logging.info('Total run time: %s' %
(pretty_time(datetime.now() - script_start)))
#
# Finish up
#
# create html scriptlog
if os.path.exists(scriptlog):
html_scriptlog = os.path.join(log_dir, 'scriptlog-' + DATE + '.html')
sl = html.Create()
sl.title('Nightly Log')
sl.css('body {font-family:"Lucida Console", Monaco, monospace;font-size:0.9em;}')
sl.header(DATE)
sl.body(html.add_line_breaks(html.parse_file(scriptlog)))
sl.write(html_scriptlog)
# add log to rsync queues
if uploading:
upq.put(html_scriptlog)
upq.join()
if mirroring:
m_q.put(html_scriptlog)
m_q.join()
# cd previous working dir
os.chdir(previous_working_dir)
if __name__ == "__main__":
main(args)