-
Notifications
You must be signed in to change notification settings - Fork 0
/
convert.py
404 lines (330 loc) · 11.3 KB
/
convert.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import shutil
import errno
import sys
import optparse
import string
import smtplib
import datetime
from datetime import date
import subprocess
from thread_pool import ThreadPool
from application_lock import ApplicationLock
"""
Verbosity
"""
VERBOSE=True
"""
Debug
"""
DEBUG=False # = Remove empty folders after processing
"""
Log File
"""
LOGFILE='/tmp/convert_' + datetime.datetime.now().strftime("%Y-%b-%d_%H-%M-%S") + '.log'
"""
Conversion options
"""
TIF_TO_JP2=True
JP2_TO_JPEG=True
"""
Conversion tools
"""
JP2_TOOL='kdu_compress'
JPEG_TOOL='convert'
"""
Kakadu kdu_compress runtime options
"""
KDU_OPTIONS = \
'-quiet -rate 1.5 Creversible=yes Clayers=1 Clevels=7 \"Cprecincts={256,256},{256,256},{128,128}\" \"Corder=RPCL\" \"ORGgen_plt=yes\" \"ORGtparts=R\" \"Cblk={64,64}\" Cuse_sop=yes'
"""
ImageMagick convert runtime options
"""
JPEG_DEST_FILES = [['half.jpg', '1500x2100'],['quarter.jpg',
'1200x1500'], ['thumb.jpg', '200x200']]
"""
Email alert using GMail
"""
EMAIL = 'email@email.edu'
SUBJECT = 'Image Convert Report ' + str(date.today())
FROM_EMAIL = 'email@gmail.com'
USERNAME = 'email'
PASSWORD = 'password'
"""
END configuration options
"""
class logBuffer:
def __init__(self):
self.content = []
def write(self, string):
self.content.append(string)
emaillog = logBuffer()
"""
Parse variables passed to the program
"""
def parseOptions():
usage = 'usage: %prog [options]'
description = 'Process a directory containing image files and convert from TIFF to JPEG2000, and JPEG2000 to a variety of JPEG derivatives. Prerequisites include Kakadu kdu_compress and ImageMagick compiled with JasPer (for JPEG2000 support). The application supports multi-threading for multi proc/core systems. Email, command line options, verbosity, and other configuration options can be modified directly in convert.py'
parser = optparse.OptionParser(usage=usage, description=description)
parser.add_option(
'-s',
'--source',
action='store',
dest='source',
default='',
help='Source directory to be processed: Absolute or relative path.'
)
parser.add_option(
'-d',
'--destination',
action='store',
dest='destination',
default='',
help='Destination directory of processed images: Absolute or relative path.'
)
parser.add_option(
'-t',
'--threads',
action='store',
dest='threads',
default=12,
help='Set number of threads to execute at once. default = 12'
)
parser.add_option(
'-b',
'--broken',
action='store',
dest='broken',
default='_broken',
help='Directory to place files left unprocessed due to error. Absolute or relative path. default destination/_broken'
)
return parser.parse_args()
"""
Send an email using the appropriate global configuration
@param _message Message to be sent via email
"""
def sendEmail(_message):
body = string.join(('From: %s' % FROM_EMAIL, 'To: %s' % EMAIL,
'Subject: %s' % SUBJECT, '', _message), '\r\n')
server = smtplib.SMTP('smtp.gmail.com:587')
server.starttls()
server.login(USERNAME,PASSWORD)
# server.set_debuglevel(1)
server.sendmail(FROM_EMAIL, EMAIL, body)
server.quit()
"""
Log output to local filesystem
"""
def logOutput(_message):
f = open(LOGFILE, 'w')
f.write(_message)
f.close()
"""
Make directory if it doesn't exist
@param _dir Directory to be tested and created
"""
def makeDir(_dir):
try:
os.makedirs(_dir)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
print >>emaillog, 'Unable to create ' + _dir
sendEmail(''.join(emaillog.content))
sys.exit()
"""
Check if program exists in user path
@param _name Name of application
"""
def checkProgram(_name):
for dir in os.environ['PATH'].split(':'):
prog = os.path.join(dir, _name)
if os.path.exists(prog):
return prog
"""
Converts TIFF to JP2
@param _threads Number of concurrent threads to execute
@param _app Application used to process, kdu_compress default
@param _source Source directory (can include subdirectories)
@param _destination Destination directory (does not need to exist)
@param _broken Directory to store broken images that are left unprocessed
@param _options Options to pass to _app
@param _verbose Verbosity
"""
def tif_to_jp2(
_threads,
_app,
_source,
_destination,
_broken,
_options,
_verbose
):
testApp(_app)
t = ThreadPool(_threads)
for (root, dirs, files) in os.walk(_source):
subpath = root.replace(_source, '').lstrip('/')
if _broken not in subpath:
jp2Path = os.path.join(_destination,subpath)
makeDir(jp2Path)
if any(".tif" in s for s in files):
print >>emaillog, 'Converting contents of ' + subpath + ' from TIF to JP2'
for file in files:
if file.endswith('.tif'):
tiff = os.path.join(root, file)
jp2 = os.path.join(_destination, subpath,
os.path.splitext(file)[0] + '.jp2')
tiffcopy = os.path.join(_destination,subpath,file)
command = _app + ' -i ' + tiff + ' -o ' + jp2 + ' ' \
+ _options
command_post = 'shutil.move(\'' + tiff + '\',\'' + tiffcopy + '\')'
if _verbose == True:
print 'Creating ' + jp2
t.add_task(executeConversion,command,command_post,tiff,_destination,_broken,file,jp2)
t.await_completion()
"""
Converts JP2 to JPEG derivatives
@param _threads Number of concurrent threads to execute
@param _app Application used to process, ImageMagick convert default
@param _source Source directory (can include subdirectories)
@param _destination Destination directory (does not need to exist)
@param _broken Directory to store broken images that are left unprocessed
@param _jpegs Array of names,sizes of various required derivatives
@param _verbose Verbosity
"""
def jp2_to_jpeg(
_threads,
_app,
_source,
_destination,
_broken,
_jpegs,
_verbose
):
testApp(_app)
t = ThreadPool(_threads)
for (root, dirs, files) in os.walk(_destination):
subpath = root.replace(_destination, '').lstrip('/')
if _broken not in subpath:
if any(".jp2" in s for s in files):
print >>emaillog, 'Converting contents of ' + subpath + ' from JP2 to JPEG'
for (output_file, size) in _jpegs:
for file in files:
if file.endswith('.jp2'):
jp2 = os.path.join(root, file)
newfile = os.path.join(root,
os.path.splitext(file)[0]) + '_' \
+ output_file
command = _app + ' -size ' + size + " " + jp2 \
+ ' -resize ' + size + ' ' + newfile
if _verbose == True:
print 'Creating ' + newfile
t.add_task(executeConversion,command,None,jp2,_source,_broken,file,newfile)
t.await_completion()
"""
Tests application exists and exits if not found
@param _app Application defined
"""
def testApp(_app):
if str(checkProgram(_app)) == "None":
print >>emaillog, _app + ' not found. Exiting.'
sendEmail(''.join(emaillog.content))
sys.exit()
else:
print _app + ' found.'
"""
Process conversion commands defined above and report errors
@param _command Command to execute
@param _command_post Post processing command to execute
@param _srcfile Full path + file name being performed upon
@param _destination Destination directory (does not need to exist)
@param _broken Directory to store broken images that are left unprocessed
@param _file File that command is acting upon
@param _create File with full path that is being created
"""
def executeConversion(
_command,
_command_post,
_srcfile,
_destination,
_broken,
_file,
_create
):
proc = subprocess.Popen(_command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True )
output = proc.stderr.read()
if output:
print >>emaillog, '\n--'
print >>emaillog, 'Error encountered running the following command: '
print >>emaillog, _command
print >>emaillog, 'Output: '
print >>emaillog, output
print >>emaillog, 'Moved to following directory for inspection: \n' + os.path.join(_destination,_broken,_file)
if (os.path.exists(_srcfile)):
shutil.move(_srcfile, os.path.join(_destination,_broken,_file))
print >>emaillog, 'Removing file created by this process: \n' + _create
if (os.path.exists(_create)):
os.remove(_create)
else:
if _command_post:
exec _command_post
def removeEmptyFolders(path):
if not os.path.isdir(path):
return
# remove empty subfolders
files = os.listdir(path)
if len(files):
for f in files:
fullpath = os.path.join(path, f)
if os.path.isdir(fullpath):
removeEmptyFolders(fullpath)
# if folder empty, delete it
files = os.listdir(path)
if len(files) == 0:
print "Removing empty folder:", path
os.rmdir(path)
def main():
# Fetch those options
(options, args) = parseOptions()
# Test source exists
if os.path.isdir(options.source):
# Create destination if doesn't exist
if TIF_TO_JP2 == True:
if not os.path.isdir(options.destination):
print options.destination + ' does not exist. Creating...'
makeDir(options.destination)
# Create broken directory if doesn't exist
if not os.path.isdir(options.broken):
print options.broken + ' does not exist. Creating...'
makeDir(os.path.join(options.destination,options.broken))
# Process images
if TIF_TO_JP2 == True:
if VERBOSE == True:
print 'Ready to begin TIFF to JP2 transformation.'
tif_to_jp2(int(options.threads),JP2_TOOL,
options.source,options.destination,options.broken,
KDU_OPTIONS,VERBOSE)
if JP2_TO_JPEG == True:
if VERBOSE == True:
print 'Ready to begin JP2 to JPEG transformation.'
jp2_to_jpeg(int(options.threads),JPEG_TOOL,
options.source,options.destination,options.broken,
JPEG_DEST_FILES,VERBOSE)
# Test if errors exist, email if true
if emaillog.content:
sendEmail(''.join(emaillog.content))
logOutput(''.join(emaillog.content))
# Remove any empty directories in source or destination
if DEBUG == False:
removeEmptyFolders(options.source)
removeEmptyFolders(options.destination)
if __name__ == '__main__':
applock = ApplicationLock ('/tmp/convert.lock')
if (applock.lock()):
main()
applock.unlock()
else:
print ('Unable to obtain lock, exiting')