def get_media_category(config, path):
if path.endswith('/'):
path = path[:-1]
last_dir = path[path.rfind('/')+1:]
xdcamDir = config.get('CategoryFolders', 'sony-xdcam')
videoDir = config.get('CategoryFolders', 'video')
fostexDir = config.get('CategoryFolders', 'fostex-audio')
marantzDir = config.get('CategoryFolders', 'marantz-audio')
audioDir = config.get('CategoryFolders', 'audio')
imageDir = config.get('CategoryFolders', 'image')
if last_dir == xdcamDir:
return 'sony-xdcam'
elif last_dir == videoDir:
return 'video'
elif last_dir == fostexDir:
return 'fostex-audio'
elif last_dir == marantzDir:
return 'marantz-audio'
elif last_dir == audioDir:
return 'audio'
elif last_dir == imageDir:
return 'image'
else:
print 'Path must terminate with one of the following directory names: '+xdcamDir+', '+videoDir+', '+fostexDir+', '+marantzDir+', '+audioDir+', '+imageDir
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def session_device_check(config, session_id, device_code):
if not ltoUtil.valid_chars(session_id):
print 'Invalid characters used in session id.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
if not ltoUtil.valid_chars(device_code):
print 'Invalid characters used in device_code.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
media_exists_for_session_device_qry = 'exists(/session[@id="'+session_id+'"]//device[@code="'+device_code+'"]/*)'
if not db_session_id_exists(config, session_id):
print 'session id: '+session_id+' does not yet exist.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
if not db_device_code_exists(config, device_code):
print 'device code: '+device_code+' does not exist.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
xquery_result = ltoUtil.exec_url_xquery(config, ltoUtil.get_transcript_url(config)+'/data', media_exists_for_session_device_qry)
if ltoUtil.get_parsed_xquery_value(xquery_result) == ['true']:
print 'At least one media item has already been associated with session: '+session_id+' and device: '+device_code
print
print 'To see the associated media items for this session open the following link in your browser:'
print 'http://'+ltoUtil.get_host_port(config)+ltoUtil.get_transcript_url(config)+'/data?_query=/session[@id="'+session_id+'"]//mediaMetadata'
print
cont = raw_input('Are you sure you want to continue? [y/n]: ')
if cont == 'y':
return
else:
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def check_media_ids_exist(config, items, domain):
for i in items:
if not lto_db_media_id_exists(domain, i, config):
print 'There does not exist a '+domain+' file with the id '+i+' in the tape index files.'
print 'Maybe the tar archive containing it is yet to be written to tape.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def db_add_media_xml(config, db_media_xml_doc, username, password):
deviceElement = db_media_xml_doc.getElementsByTagName('device')[0]
session_id = deviceElement.getAttribute('sessionId')
device_code = deviceElement.getAttribute('code')
base64string = base64.encodestring('%s:%s' % (username, password))[:-1]
authheader = "Basic %s" % base64string
headers = {'Authorization': authheader}
conn = httplib.HTTPConnection(ltoUtil.get_host_port(config))
if len(deviceElement.childNodes) == 0:
print 'Internal Error: No session-media xml nodes generated.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
for mediaElem in deviceElement.childNodes:
if mediaElem.nodeType == xml.dom.Node.ELEMENT_NODE:
element_xml = str(mediaElem.toxml())
#Hack to put id as the first attribute
element_xml = string.replace(element_xml, '_id="', 'id="')
params = urllib.urlencode({'sessionId': session_id, 'deviceCode': device_code, 'mediaXML': element_xml})
try:
conn.request('GET', ltoUtil.get_transcript_url(config)+'/xquery/import-media-element.xql?'+params, None, headers)
response = conn.getresponse()
data = response.read()
if '<exception>' in data:
print ltoUtil.get_xquery_exception_msg(data)
print ltoUtil.get_script_name()+' script terminated.'
return False
elif response.status == 401 and response.reason == 'Unauthorized':
print 'Authentication Failed.'
return False
except Exception, e:
print 'Unable to connect to database'
return False
def check_tape_build_dir_contents(dir):
if len(os.listdir(dir)) == 0:
print 'The tape build directory '+dir+' is empty!'
print 'You must first manually transfer a set of tar,xml file pairs here (from the tars directory).'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
for f in os.listdir(dir):
if os.path.isdir(os.path.join(dir, f)):
print 'The tape build directory: '+dir+' must not contain any sub-directories.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
elif not (f.endswith('.tar') or f.endswith('.xml')):
print 'The tape build directory: '+dir+' contains unidentified files.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
elif f.endswith('.tar'):
fid = f[:-4]
if not os.path.exists(os.path.join(dir, fid+'.xml')):
print 'The tape build directory: '+dir+' contains tar files without matching xml files.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
elif f.endswith('.xml'):
fid = f[:-4]
if not os.path.exists(os.path.join(dir, fid+'.tar')):
print 'The tape build directory: '+dir+' contains xml files without matching tar files.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def check_total_size(config, vectors):
total_size = 0
for v in vectors:
total_size += v[2]
free_bytes = ltoUtil.get_freespace(ltoUtil.get_restore_dir(config))
if total_size > free_bytes - (1024*1024):
print 'Insufficient free space on drive ('+ltoUtil.format_bytes_to_gbs(free_bytes)+') to restore all files ('+ltoUtil.format_bytes_to_gbs(total_size)+').'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def parse_files(args):
ids = []
for f in args:
if not os.path.exists(f):
print 'Cannot find file '+f
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
fh = open(f, 'r')
for line in fh.readlines():
if not (line.strip() == None or line.strip() == ""):
ids.append(line.strip())
return ids
def get_media_ids(config, session_ids, domain):
list = []
for s in session_ids:
if ltoTarUtil.db_session_id_exists(config, s):
media_ids_qry = '/session[@id="'+s+'"]//'+domain+'/xs:string(@id)'
xquery_result = ltoUtil.exec_url_xquery(config, ltoUtil.get_transcript_url(config)+'/data', media_ids_qry)
list.extend(ltoUtil.get_parsed_xquery_value(xquery_result))
else:
print s+' is not a recognised session id.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
return list
def move_build_virtual_tape_files(config, tape_id):
dest = config.get('Dirs', 'virtual_tape_dir')+'/pending/'+tape_id
try:
if not os.path.exists(dest):
os.mkdir(dest)
else:
ltoUtil.delete_dir_content(dest)
except OSError, e:
print 'Unable to create virtual tape directory: '+dest
print 'OSError '+str(e.errno)+': '+e.strerror
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def get_session_full_name(config, sessionId):
params = urllib.urlencode({'sessionId':sessionId})
conn = httplib.HTTPConnection(ltoUtil.get_host_port(config))
try:
conn.request('GET', ltoUtil.get_transcript_url(config)+'/xquery/get-session-full-name.xql?'+params, None, {})
response = conn.getresponse()
data = response.read()
if '<exception>' in data:
print ltoUtil.get_xquery_exception_msg(data)
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
except httplib.HTTPException, e:
print 'Unable to connect to database'
def move_tar_files(config, session_id, device_code):
tb = ltoUtil.get_tar_build_dir(config)
archive_id = session_id+'-'+device_code
tar = tb+'/'+archive_id+'.tar'
xml = tb+'/'+archive_id+'.xml'
dest = config.get('Dirs', 'tar_archive_dir')
try:
print '\nMoving '+archive_id+'.tar to '+dest
ltoUtil.move(tar, dest)
print '\nMoving '+archive_id+'.xml to '+dest
ltoUtil.move(xml, dest)
except Exception, e:
print '\nUnable to move archive files to: '+dest
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def get_item_vectors(config, items, domain):
bs = int(ltoUtil.get_blocksize(config))
ids = string.join(items, ',')
params = urllib.urlencode({'domain':domain, 'ids':ids})
conn = httplib.HTTPConnection(ltoUtil.get_host_port(config))
try:
conn.request('POST', ltoUtil.get_lto_url(config)+'/xquery/get-file-lto-vector.xql?'+params, None, {})
response = conn.getresponse()
data = response.read()
if '<exception>' in data:
print ltoUtil.get_xquery_exception_msg(data)
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
except httplib.HTTPException, e:
print 'Unable to connect to database'
def db_get_next_media_id(session_id, domain, config):
event_type = session_id[:session_id.find('-')]
params = urllib.urlencode({'domain': domain, 'eventType': event_type})
conn = httplib.HTTPConnection(ltoUtil.get_host_port(config))
try:
conn.request('GET', ltoUtil.get_transcript_url(config)+'/xquery/get-next-media-id.xql?'+params, None, {})
response = conn.getresponse()
data = response.read()
if '<exception>' in data:
print ltoUtil.get_xquery_exception_msg(data)
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
else:
return data
except httplib.HTTPException, e:
print 'Unable to connect to database'
def diskspace_check(config, tape_id):
verify_dir = ltoUtil.get_tape_verify_dir(config)
free_bytes = ltoUtil.get_freespace(verify_dir)
index_path = ltoUtil.get_tape_written_dir(config)+'/'+tape_id+'/'+tape_id+'.xml'
tape_xml_doc = xml.dom.minidom.parse(index_path)
tar_elems = tape_xml_doc.getElementsByTagName('tar')
max_tar_size = 0
for t in tar_elems:
tar_size = int(t.getAttribute('size'))
if tar_size > max_tar_size:
max_tar_size = tar_size
if max_tar_size > free_bytes - (1024*1024):
free_gbs = ltoUtil.format_bytes_to_gbs(free_bytes)
needed_gbs = ltoUtil.format_bytes_to_gbs(max_tar_size)
print 'There is only '+free_gbs+' available space on '+verify_dir+'. At least '+needed_gbs+' is needed to perform the tape verification.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def get_new_media_id(session_id, domain, category, config, previous_id, filepath):
if category == 'sony-xdcam':
id = generate_new_id(session_id, domain, config, previous_id)
elif category == 'video':
id = get_id_from_filename(filepath, domain, config)
elif category == 'fostex-audio':
id = generate_new_id(session_id, domain, config, previous_id)
elif category == 'marantz-audio':
id = generate_new_id(session_id, domain, config, previous_id)
elif category == 'audio':
id = get_id_from_filename(filepath, domain, config)
elif category == 'image':
id = generate_new_id(session_id, domain, config, previous_id)
else:
print 'Undefined media category: '+category
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
return id
def check_tape_build_size(config):
dir = ltoUtil.get_tape_build_dir(config)
min_gbs = config.getint('Tape', 'min_gb')
max_gbs = config.getint('Tape', 'max_gb')
min_size = int(min_gbs)*1024*1024*1024
max_size = int(max_gbs)*1024*1024*1024
index_size_mb = config.getint('Tape', 'index_size_mb')
index_size = index_size_mb*1024*1024
size = ltoUtil.get_dir_total_size(dir) + index_size
size_gb = float(size)/(1024*1024*1024)
if size > max_size:
print 'The total size of the files in the tape build directory: '+dir+' is over the maximum limit of '+str(max_gbs)+'GB.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
elif size < min_size:
print 'The total size of the files in the tape build directory: '+dir+' is under the minimum limit of '+str(min_gbs)+'GB.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
else:
print 'Virtual tape size: '+'%3.2f'%size_gb+'GB'
def get_id_from_filename(filepath, domain, config):
fn = ltoUtil.get_filename(filepath)
id = string.lower(fn[:fn.rfind('.')])
if re.match('^[a-z]{1,3}-?[0-9]+$', id):
alpha = re.search('^[a-z]+', id)
alpha_index = alpha.end()
num = re.search('[0-9]+$', id)
num_index = num.start()
alpha_part = id[:alpha_index]
#This is to remove any leading zeros (which will prevent ids from matching)
num_part = int(id[num_index:])
if num_part == 0:
print 'Media filename cannot have a zero digit component.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
#Insert hyphen if un-hyphenated
id = alpha_part+'-'+str(num_part)
if db_media_id_exists(domain, id, config):
print 'The '+domain+' media id: '+id+' has already been used.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
else:
print 'Wrong format for media filename: '+fn
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
return id
def generate_db_media_xml_element(media_path, domain, category, media_id, config):
if media_in_category(media_path, category, config):
if category == 'sony-xdcam':
mfile = open(media_path, 'r')
num_bytes_mp4_xml_data = 1143
try:
mfile.seek(-num_bytes_mp4_xml_data, 2)
except IOError, io:
print 'Unable to read data from '+category+' file '+media_path
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
mp4xml = mfile.read()
timestamp = get_sonyxdcam_timestamp(mp4xml)
timecode = get_sonyxdcam_timecode(mp4xml)
original_id = get_original_id(media_path)
duration = get_media_duration(media_path)
auto_split = get_autosplit(media_path)
attributes = {'timestamp':timestamp, 'originalId':original_id, 'duration':duration, 'autoSplit':auto_split, 'timecode':timecode}
elif category == 'video':
original_id = get_original_id(media_path)
duration = get_media_duration(media_path)
attributes = {'originalId':original_id, 'duration':duration}
elif category == 'audio':
original_id = get_original_id(media_path)
duration = get_media_duration(media_path)
attributes = {'originalId':original_id, 'duration':duration}
elif category == 'fostex-audio':
mfile = open(media_path, 'r')
num_bytes_bwf_xml_data = 10945
try:
mfile.seek(num_bytes_bwf_xml_data, 0)
except IOError, io:
print 'Unable to read data from '+category+' file '+media_path
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
bwfxml = mfile.read(615)
timestamp = get_fostexaudio_timestamp(bwfxml)
original_id = get_original_id(media_path)
duration = get_media_duration(media_path)
attributes = {'timestamp':timestamp, 'originalId':original_id, 'duration':duration}
def check_restore_args(opts, args):
if len(args) == 0:
print 'Missing arguments.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
if opts.use_session_ids == True and opts.use_files == True:
print 'You cannot specify both the -s and -f options simultaneously.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
if not (opts.media_type == None or opts.media_type == 'video' or opts.media_type == 'audio' or opts.media_type == 'image'):
print 'Invalid media type: "'+opts.media_type+'".'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def db_import_tape_xml(config, tape_xml_doc):
tape_xml = str(tape_xml_doc.toxml())
print '\nImporting tape index to database:\n'
username = raw_input('username: '******'password: '******'%s:%s' % (username, password)))
authheader = "Basic %s" % base64string
headers = {"Authorization": authheader, "Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
params = urllib.urlencode({'tapeXML': tape_xml})
conn = httplib.HTTPConnection(ltoUtil.get_host_port(config))
try:
conn.request('POST', ltoUtil.get_lto_url(config)+'/xquery/import-tape-element.xql', params, headers)
response = conn.getresponse()
data = response.read()
if '<exception>' in data:
print ltoUtil.get_xquery_exception_msg(data)
print 'Unable to update database.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
elif 'HTTP ERROR: 404' in data:
print '\nHTTP ERROR: 404'
print data[data.find('<title>')+7:data.find('</title>')]
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
elif response.status == 401 and response.reason == 'Unauthorized':
print 'Authentication Failed.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
else:
print '\nDatabase updated (tape id: '+data+').'
except httplib.HTTPException, e:
print e.msg
print 'Unable to connect to database'
print ltoUtil.get_script_name()+' script terminated.'
conn.close()
sys.exit(2)
def media_file_types_check(config, category, path):
category_filetypes = config.get('CategoryFileTypes', category).split(',')
media_file_count = 0
total_file_size = 0
non_category_files = []
for dirpath, dirnames, filenames in os.walk(path):
for file in filenames:
if media_in_domain(file, 'video', config) or media_in_domain(file, 'audio', config) or media_in_domain(file, 'image', config):
if not media_in_category(file, category, config):
non_category_files.append(os.path.join(dirpath, file))
else:
media_file_count += 1
total_file_size += ltoUtil.get_filesize(os.path.join(dirpath, file))
free_bytes = ltoUtil.get_freespace(ltoUtil.get_tar_build_dir(config))
if (len(non_category_files) > 0):
print 'WARNING: The following unexpected media files were found in the folder: '+path+'\n'
for f in non_category_files:
print f
print '\nExpecting only '+string.upper(string.join(category_filetypes, ','))+' files for "'+category+'" category.'
print '\nNOTE: This situation should only occur if when one device has generated multiple media formats. (e.g. a video camera taking stills/audio as well as video)'
print 'Usually this indicates that the files have been misplaced, in which case the user should manually remove them before re-running the script.\n'
proceed = raw_input('Do you still want to proceed, including these files in the archive? [y/n]: ')
if proceed == 'y':
return
else:
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
elif media_file_count == 0:
print 'No recognised media files were found in: '+path
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
elif total_file_size > free_bytes - (1024*1024):
print 'The total file size of the media files to be archived ('+ltoUtil.format_bytes_to_gbs(total_file_size)+') exceeds the free space available in '+ltoUtil.get_tar_build_dir(config)+' ('+ltoUtil.format_bytes_to_gbs(free_bytes)+').'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
print 'Unable to read data from '+category+' file '+media_path
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
bwfxml = mfile.read(615)
timestamp = get_fostexaudio_timestamp(bwfxml)
original_id = get_original_id(media_path)
duration = get_media_duration(media_path)
attributes = {'timestamp':timestamp, 'originalId':original_id, 'duration':duration}
elif category == 'marantz-audio':
mfile = open(media_path, 'r')
num_bytes_wf_str_data = 364
try:
mfile.seek(num_bytes_wf_str_data, 0)
except IOError, io:
print 'Unable to read data from '+category+' file '+media_path
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
str = mfile.read(18)
timestamp = get_marantzaudio_timestamp(str)
original_id = get_original_id(media_path)
duration = get_media_duration(media_path)
attributes = {'timestamp':timestamp, 'originalId':original_id, 'duration':duration}
elif category == 'audio':
duration = get_media_duration(media_path)
attributes = {'originalId':original_id,'duration':duration}
elif category == 'image':
timestamp = get_image_timestamp(media_path)
original_id = get_original_id(media_path)
attributes = {'timestamp':timestamp, 'originalId':original_id}
else:
original_id = get_original_id(media_path)
def check_tar_args(args):
if not len(args) == 5:
print 'Exactly five arguments must be provided. (session id, device code, media path, username, password).'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def generate_proxy(config, domain, filepath):
proxy_dir = ltoUtil.get_proxy_dir(config)
thumb_dir = ltoUtil.get_thumb_dir(config)
proxy_suffix = 'proxy'
thumb_suffix = 'thumb'
if os.path.exists(proxy_dir+'/'+domain):
if (domain =='image' or domain == 'video'):
if os.path.exists(thumb_dir+'/'+domain):
if (domain == 'video'):
video_type = get_video_type(filepath)
if video_type == 'DV_PAL' or video_type == 'DV_NTSC' or video_type == 'MPEG2_H-14':
proxy_size = '384x288'
thumb_size = '96x72'
elif video_type == 'MPEG2_HL':
proxy_size = '512x288'
thumb_size = '128x72'
fn = ltoUtil.get_filename(filepath)
fn_proxy = fn[0:-4]+'_'+proxy_suffix+'.mp4'
fn_thumb = fn[0:-4]+'_'+thumb_suffix+'.jpg'
p = subprocess.Popen('ffmpeg -y -i '+filepath+' -pass 1 -vcodec libx264 -coder 1 -flags +loop -cmp +chroma -partitions -parti8x8-parti4x4-partp8x8-partp4x4-partb8x8 -me_method dia -subq 1 -me_range 16 -g 25 -keyint_min 25 -sc_threshold 40 -i_qfactor 0.71 -b_strategy 1 -qcomp 0.6 -qmin 10 -qmax 51 -qdiff 4 -bf 4 -refs 1 -directpred 3 -trellis 0 -flags2 -bpyramid-wpred-mixed_refs-dct8x8+fastpskip -s '+proxy_size+' -b 512k -bt 512k -threads 0 -f mp4 -an /dev/null && ffmpeg -y -i '+filepath+' -pass 2 -acodec libfaac -ar 44100 -ab 96k -vcodec libx264 -coder 1 -flags +loop -cmp +chroma -partitions +parti8x8+parti4x4+partp8x8+partb8x8 -me_method umh -subq 8 -me_range 16 -g 25 -keyint_min 25 -sc_threshold 40 -i_qfactor 0.71 -b_strategy 2 -qcomp 0.6 -qmin 10 -qmax 51 -qdiff 4 -bf 4 -refs 4 -directpred 3 -trellis 1 -flags2 +bpyramid+wpred+mixed_refs+dct8x8+fastpskip -s '+proxy_size+' -b 512k -bt 512k -g 25 -threads 0 -f mp4 '+proxy_dir+'/video/'+fn_proxy, shell=True)
sts = os.waitpid(p.pid, 0)
os.remove('ffmpeg2pass-0.log')
os.remove('x264_2pass.log')
p = subprocess.Popen('ffmpeg -y -i '+proxy_dir+'/video/'+fn_proxy+' -vcodec mjpeg -vframes 1 -an -f rawvideo -s '+thumb_size+' '+thumb_dir+'/video/'+fn_thumb, shell=True)
sts = os.waitpid(p.pid, 0)
elif (domain =='image'):
orientation = get_orientation(filepath)
file_type = ltoUtil.get_file_extn(filepath)
if orientation in [0,1,2,3,4] and file_type == 'jpg':
proxy_size = '1024x680'
thumb_size = '128x85'
elif orientation in [0,1,2,3,4] and file_type in ['nef', 'cr2']:
proxy_size = '1024x680'
thumb_size = '128x85'
elif orientation in [5,6,7,8] and file_type == 'jpg':
proxy_size = '1024x680'
thumb_size = '128x85'
#We need to do this since we are extracting the embedded image from the raw files - which has no EXIF data
elif orientation in [5,6,7,8] and file_type in ['nef', 'cr2']:
proxy_size = '680x1024'
thumb_size = '85x128'
fn = ltoUtil.get_filename(filepath)
fn_proxy = fn[0:-4]+'_'+proxy_suffix+'.jpg'
fn_thumb = fn[0:-4]+'_'+thumb_suffix+'.jpg'
if string.lower(file_type) == 'jpg':
p = subprocess.Popen('convert -thumbnail '+proxy_size+' -auto-orient -unsharp 0x.5 -quality 92 '+filepath+' '+proxy_dir+'/image/'+fn_proxy, shell=True)
sts = os.waitpid(p.pid, 0)
#Much faster to extract the embedded image from the RAW file instead of resizing
elif string.lower(file_type) == 'nef' or string.lower(file_type) == 'cr2':
p = subprocess.Popen('dcraw -ce '+filepath+' | convert -thumbnail '+proxy_size+' -auto-orient -unsharp 0x.5 -quality 92 - '+proxy_dir+'/image/'+fn_proxy, shell=True)
sts = os.waitpid(p.pid, 0)
p = subprocess.Popen('convert -thumbnail '+thumb_size+' -auto-orient '+proxy_dir+'/image/'+fn_proxy+' '+thumb_dir+'/image/'+fn_thumb, shell=True)
sts = os.waitpid(p.pid, 0)
else:
print 'thumbnail directory '+thumb_dir+'/'+domain+' does not exist.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
elif (domain =='audio'):
fn = ltoUtil.get_filename(filepath)
fn_proxy = fn[0:-4]+'_'+proxy_suffix+'.mp4'
p = subprocess.Popen('ffmpeg -y -i '+filepath+' -acodec libfaac -ar 44100 -ab 96k -ac 1 -f mp4 '+proxy_dir+'/audio/'+fn_proxy, shell=True)
sts = os.waitpid(p.pid, 0)
else:
print 'proxy directory '+proxy_dir+'/'+domain+' does not exist.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
def verify_tape(config, tape_id):
print '\nVerifying index'
bs = int(config.get('Tape', 'block_size_bytes'))
tape = config.get('Tape', 'device')
blocking_factor = bs/512
written_dir = ltoUtil.get_tape_written_dir(config)+'/'+tape_id
temp_dir = ltoUtil.get_temp_dir(config)
original_dir = temp_dir+'/verify-tape/'+tape_id+'/original'
archived_dir = temp_dir+'/verify-tape/'+tape_id+'/archived'
ltoUtil.create_dir(original_dir)
ltoUtil.create_dir(archived_dir)
tape_xml_file = tape_id+'.xml'
#Extract index file from tape
p = subprocess.Popen('tar -x -b '+str(blocking_factor)+' -C '+archived_dir+' -f '+tape+' '+tape_xml_file, shell=True)
sts = os.waitpid(p.pid, 0)
#Unpack index file from disk archive
index_tar_path = written_dir+'/'+tape_xml_file+'.tar'
if not os.path.exists(index_tar_path):
print 'Verification Error: The tar file '+index_tar_path+' does not exist.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
p = subprocess.Popen('tar -x -b '+str(blocking_factor)+' -C '+original_dir+' -f '+written_dir+'/'+tape_xml_file+'.tar ', shell=True)
sts = os.waitpid(p.pid, 0)
#Compare index files
original_index = original_dir+'/'+tape_xml_file
archived_index = archived_dir+'/'+tape_xml_file
if not os.path.exists(archived_index):
print 'Unable to restore tape index file '+tape_xml_file+' from tape. Please check the correct tape is in the drive.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
if not os.path.exists(original_index):
print 'Unable to extract tape index file '+tape_xml_file+' from '+written_dir+'/'+tape_xml_file+'.tar.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
if not ltoUtil.compare(original_index, archived_index):
print '\nIndex file verification failed.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
print 'OK'
print '\nVerifying tape\n'
position = 2
#Parse index file into memory
tape_xml_doc = xml.dom.minidom.parse(archived_index)
tar_elems = tape_xml_doc.getElementsByTagName('tar')
if len(tar_elems) == 0:
print 'XML Error: The tape index file '+archived_index+' has no "tar" elements.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
#Loop thru all tars on tape
disk_tarfiles_verified = [tape_xml_file+'.tar']
while True:
p = subprocess.Popen('mt -f '+tape+' fsf 1', shell=True)
sts = os.waitpid(p.pid, 0)
tapedump_dir = archived_dir+'/'+str(position)
diskdump_dir = original_dir+'/'+str(position)
ltoUtil.create_dir(tapedump_dir)
ltoUtil.create_dir(diskdump_dir)
session_id = ""
device_code = ""
#Extract tarfile members from tape
print 'Extracting tar archive #'+str(position)+' from tape'
p = subprocess.Popen('tar -x -b '+str(blocking_factor)+' -C '+tapedump_dir+' -f '+tape, shell=True, stderr=subprocess.PIPE)
stderr_value = p.stderr.read()
sts = os.waitpid(p.pid, 0)
if stderr_value == 'tar: This does not look like a tar archive\ntar: Error exit delayed from previous errors\n':
print 'End of Data reached on tape'
break
t = tar_elems[position - 2]
if int(t.getAttribute('position')) == position:
session_id = t.getAttribute('sessionId')
device_code = t.getAttribute('deviceCode')
tar_name = session_id+'-'+device_code+'.tar'
tarfile_path = written_dir+'/'+tar_name
if not os.path.exists(tarfile_path):
print 'Verification Error: The tar file '+tarfile_path+' does not exist.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
#Unpack corresponding tarfile on disk
print 'Extracting corresponding local tar archive '+tarfile_path+'\n'
p = subprocess.Popen('tar -x -b '+str(blocking_factor)+' -C '+diskdump_dir+' -f '+written_dir+'/'+tar_name, shell=True)
sts = os.waitpid(p.pid, 0)
#Compare the extracted files with each other (byte for byte), and ensure that index file is correct
tar_members = []
for file in os.listdir(diskdump_dir):
fp1 = os.path.join(diskdump_dir, file)
fp2 = os.path.join(tapedump_dir, file)
if not os.path.exists(fp2):
print '\nVerification Error: The file '+file+' was found on the local tape archive '+tarfile_path+', but not on the tape.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
md5 = ltoUtil.compare(fp1, fp2)
if not md5:
print '\nVerification Error: The file '+file+' failed the byte comparison with the local copy.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
else:
print file+' byte comparison with local copy successful'
ref_file = session_id+'-'+device_code+'-referenced-items.xml'
if not file == ref_file:
#Verify each file against index file
children = t.childNodes
for c in children:
if c.nodeType == c.ELEMENT_NODE and c.getAttribute('filename') == file:
if md5 == c.getAttribute('md5'):
print file+' MD5 check successful'
else:
print '\nVerification Error: The file '+file+' failed the md5 verification against the index file.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
grandchildren = c.childNodes
for g in grandchildren:
if g.nodeType == g.ELEMENT_NODE and g.getAttribute('filename') == file:
if md5 == g.getAttribute('md5'):
print file+' MD5 check successful'
else:
print '\nVerification Error: The file '+file+' failed the md5 verification against the index file.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
tar_members.append(file)
#Check that there are no unmatched tar members on the tape
for m in os.listdir(tapedump_dir):
if not m in tar_members:
print '\nVerification Error: The file '+m+' was found on the tape, but not on the corresponding local tape archive '+tarfile_path
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
disk_tarfiles_verified.append(tar_name)
else:
print 'Index File Error: Index file archives incorrectly ordered.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
#Delete temporary files
print '\n'
shutil.rmtree(original_dir)
shutil.rmtree(archived_dir)
position += 1
#Check that there are no unmatched tarfiles on the disk
for f in os.listdir(written_dir):
if f.endswith('.tar') and not f in disk_tarfiles_verified:
print 'Verification Error: The tar archive '+written_dir+'/'+f+' was not found on the tape.'
print ltoUtil.get_script_name()+' script terminated.'
sys.exit(2)
print 'Rewinding tape.'
p = subprocess.Popen('mt -f '+tape+' rewind', shell=True, stderr=subprocess.PIPE)
sts = os.waitpid(p.pid, 0)
ltoUtil.terminate_on_error(p.stderr.read())
print '\nTape successfully verified.'
