def get_media_category(config, path): if path.endswith('/'): path = path[:-1] last_dir = path[path.rfind('/')+1:] xdcamDir = config.get('CategoryFolders', 'sony-xdcam') videoDir = config.get('CategoryFolders', 'video') fostexDir = config.get('CategoryFolders', 'fostex-audio') marantzDir = config.get('CategoryFolders', 'marantz-audio') audioDir = config.get('CategoryFolders', 'audio') imageDir = config.get('CategoryFolders', 'image') if last_dir == xdcamDir: return 'sony-xdcam' elif last_dir == videoDir: return 'video' elif last_dir == fostexDir: return 'fostex-audio' elif last_dir == marantzDir: return 'marantz-audio' elif last_dir == audioDir: return 'audio' elif last_dir == imageDir: return 'image' else: print 'Path must terminate with one of the following directory names: '+xdcamDir+', '+videoDir+', '+fostexDir+', '+marantzDir+', '+audioDir+', '+imageDir print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def session_device_check(config, session_id, device_code): if not ltoUtil.valid_chars(session_id): print 'Invalid characters used in session id.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) if not ltoUtil.valid_chars(device_code): print 'Invalid characters used in device_code.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) media_exists_for_session_device_qry = 'exists(/session[@id="'+session_id+'"]//device[@code="'+device_code+'"]/*)' if not db_session_id_exists(config, session_id): print 'session id: '+session_id+' does not yet exist.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) if not db_device_code_exists(config, device_code): print 'device code: '+device_code+' does not exist.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) xquery_result = ltoUtil.exec_url_xquery(config, ltoUtil.get_transcript_url(config)+'/data', media_exists_for_session_device_qry) if ltoUtil.get_parsed_xquery_value(xquery_result) == ['true']: print 'At least one media item has already been associated with session: '+session_id+' and device: '+device_code print print 'To see the associated media items for this session open the following link in your browser:' print 'http://'+ltoUtil.get_host_port(config)+ltoUtil.get_transcript_url(config)+'/data?_query=/session[@id="'+session_id+'"]//mediaMetadata' print cont = raw_input('Are you sure you want to continue? [y/n]: ') if cont == 'y': return else: print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def check_media_ids_exist(config, items, domain): for i in items: if not lto_db_media_id_exists(domain, i, config): print 'There does not exist a '+domain+' file with the id '+i+' in the tape index files.' print 'Maybe the tar archive containing it is yet to be written to tape.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def db_add_media_xml(config, db_media_xml_doc, username, password): deviceElement = db_media_xml_doc.getElementsByTagName('device')[0] session_id = deviceElement.getAttribute('sessionId') device_code = deviceElement.getAttribute('code') base64string = base64.encodestring('%s:%s' % (username, password))[:-1] authheader = "Basic %s" % base64string headers = {'Authorization': authheader} conn = httplib.HTTPConnection(ltoUtil.get_host_port(config)) if len(deviceElement.childNodes) == 0: print 'Internal Error: No session-media xml nodes generated.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) for mediaElem in deviceElement.childNodes: if mediaElem.nodeType == xml.dom.Node.ELEMENT_NODE: element_xml = str(mediaElem.toxml()) #Hack to put id as the first attribute element_xml = string.replace(element_xml, '_id="', 'id="') params = urllib.urlencode({'sessionId': session_id, 'deviceCode': device_code, 'mediaXML': element_xml}) try: conn.request('GET', ltoUtil.get_transcript_url(config)+'/xquery/import-media-element.xql?'+params, None, headers) response = conn.getresponse() data = response.read() if '<exception>' in data: print ltoUtil.get_xquery_exception_msg(data) print ltoUtil.get_script_name()+' script terminated.' return False elif response.status == 401 and response.reason == 'Unauthorized': print 'Authentication Failed.' return False except Exception, e: print 'Unable to connect to database' return False
def check_tape_build_dir_contents(dir): if len(os.listdir(dir)) == 0: print 'The tape build directory '+dir+' is empty!' print 'You must first manually transfer a set of tar,xml file pairs here (from the tars directory).' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) for f in os.listdir(dir): if os.path.isdir(os.path.join(dir, f)): print 'The tape build directory: '+dir+' must not contain any sub-directories.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) elif not (f.endswith('.tar') or f.endswith('.xml')): print 'The tape build directory: '+dir+' contains unidentified files.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) elif f.endswith('.tar'): fid = f[:-4] if not os.path.exists(os.path.join(dir, fid+'.xml')): print 'The tape build directory: '+dir+' contains tar files without matching xml files.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) elif f.endswith('.xml'): fid = f[:-4] if not os.path.exists(os.path.join(dir, fid+'.tar')): print 'The tape build directory: '+dir+' contains xml files without matching tar files.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def check_total_size(config, vectors): total_size = 0 for v in vectors: total_size += v[2] free_bytes = ltoUtil.get_freespace(ltoUtil.get_restore_dir(config)) if total_size > free_bytes - (1024*1024): print 'Insufficient free space on drive ('+ltoUtil.format_bytes_to_gbs(free_bytes)+') to restore all files ('+ltoUtil.format_bytes_to_gbs(total_size)+').' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def parse_files(args): ids = [] for f in args: if not os.path.exists(f): print 'Cannot find file '+f print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) fh = open(f, 'r') for line in fh.readlines(): if not (line.strip() == None or line.strip() == ""): ids.append(line.strip()) return ids
def get_media_ids(config, session_ids, domain): list = [] for s in session_ids: if ltoTarUtil.db_session_id_exists(config, s): media_ids_qry = '/session[@id="'+s+'"]//'+domain+'/xs:string(@id)' xquery_result = ltoUtil.exec_url_xquery(config, ltoUtil.get_transcript_url(config)+'/data', media_ids_qry) list.extend(ltoUtil.get_parsed_xquery_value(xquery_result)) else: print s+' is not a recognised session id.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) return list
def move_build_virtual_tape_files(config, tape_id): dest = config.get('Dirs', 'virtual_tape_dir')+'/pending/'+tape_id try: if not os.path.exists(dest): os.mkdir(dest) else: ltoUtil.delete_dir_content(dest) except OSError, e: print 'Unable to create virtual tape directory: '+dest print 'OSError '+str(e.errno)+': '+e.strerror print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def get_session_full_name(config, sessionId): params = urllib.urlencode({'sessionId':sessionId}) conn = httplib.HTTPConnection(ltoUtil.get_host_port(config)) try: conn.request('GET', ltoUtil.get_transcript_url(config)+'/xquery/get-session-full-name.xql?'+params, None, {}) response = conn.getresponse() data = response.read() if '<exception>' in data: print ltoUtil.get_xquery_exception_msg(data) print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) except httplib.HTTPException, e: print 'Unable to connect to database'
def move_tar_files(config, session_id, device_code): tb = ltoUtil.get_tar_build_dir(config) archive_id = session_id+'-'+device_code tar = tb+'/'+archive_id+'.tar' xml = tb+'/'+archive_id+'.xml' dest = config.get('Dirs', 'tar_archive_dir') try: print '\nMoving '+archive_id+'.tar to '+dest ltoUtil.move(tar, dest) print '\nMoving '+archive_id+'.xml to '+dest ltoUtil.move(xml, dest) except Exception, e: print '\nUnable to move archive files to: '+dest print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def get_item_vectors(config, items, domain): bs = int(ltoUtil.get_blocksize(config)) ids = string.join(items, ',') params = urllib.urlencode({'domain':domain, 'ids':ids}) conn = httplib.HTTPConnection(ltoUtil.get_host_port(config)) try: conn.request('POST', ltoUtil.get_lto_url(config)+'/xquery/get-file-lto-vector.xql?'+params, None, {}) response = conn.getresponse() data = response.read() if '<exception>' in data: print ltoUtil.get_xquery_exception_msg(data) print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) except httplib.HTTPException, e: print 'Unable to connect to database'
def db_get_next_media_id(session_id, domain, config): event_type = session_id[:session_id.find('-')] params = urllib.urlencode({'domain': domain, 'eventType': event_type}) conn = httplib.HTTPConnection(ltoUtil.get_host_port(config)) try: conn.request('GET', ltoUtil.get_transcript_url(config)+'/xquery/get-next-media-id.xql?'+params, None, {}) response = conn.getresponse() data = response.read() if '<exception>' in data: print ltoUtil.get_xquery_exception_msg(data) print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) else: return data except httplib.HTTPException, e: print 'Unable to connect to database'
def diskspace_check(config, tape_id): verify_dir = ltoUtil.get_tape_verify_dir(config) free_bytes = ltoUtil.get_freespace(verify_dir) index_path = ltoUtil.get_tape_written_dir(config)+'/'+tape_id+'/'+tape_id+'.xml' tape_xml_doc = xml.dom.minidom.parse(index_path) tar_elems = tape_xml_doc.getElementsByTagName('tar') max_tar_size = 0 for t in tar_elems: tar_size = int(t.getAttribute('size')) if tar_size > max_tar_size: max_tar_size = tar_size if max_tar_size > free_bytes - (1024*1024): free_gbs = ltoUtil.format_bytes_to_gbs(free_bytes) needed_gbs = ltoUtil.format_bytes_to_gbs(max_tar_size) print 'There is only '+free_gbs+' available space on '+verify_dir+'. At least '+needed_gbs+' is needed to perform the tape verification.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def get_new_media_id(session_id, domain, category, config, previous_id, filepath): if category == 'sony-xdcam': id = generate_new_id(session_id, domain, config, previous_id) elif category == 'video': id = get_id_from_filename(filepath, domain, config) elif category == 'fostex-audio': id = generate_new_id(session_id, domain, config, previous_id) elif category == 'marantz-audio': id = generate_new_id(session_id, domain, config, previous_id) elif category == 'audio': id = get_id_from_filename(filepath, domain, config) elif category == 'image': id = generate_new_id(session_id, domain, config, previous_id) else: print 'Undefined media category: '+category print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) return id
def check_tape_build_size(config): dir = ltoUtil.get_tape_build_dir(config) min_gbs = config.getint('Tape', 'min_gb') max_gbs = config.getint('Tape', 'max_gb') min_size = int(min_gbs)*1024*1024*1024 max_size = int(max_gbs)*1024*1024*1024 index_size_mb = config.getint('Tape', 'index_size_mb') index_size = index_size_mb*1024*1024 size = ltoUtil.get_dir_total_size(dir) + index_size size_gb = float(size)/(1024*1024*1024) if size > max_size: print 'The total size of the files in the tape build directory: '+dir+' is over the maximum limit of '+str(max_gbs)+'GB.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) elif size < min_size: print 'The total size of the files in the tape build directory: '+dir+' is under the minimum limit of '+str(min_gbs)+'GB.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) else: print 'Virtual tape size: '+'%3.2f'%size_gb+'GB'
def get_id_from_filename(filepath, domain, config): fn = ltoUtil.get_filename(filepath) id = string.lower(fn[:fn.rfind('.')]) if re.match('^[a-z]{1,3}-?[0-9]+$', id): alpha = re.search('^[a-z]+', id) alpha_index = alpha.end() num = re.search('[0-9]+$', id) num_index = num.start() alpha_part = id[:alpha_index] #This is to remove any leading zeros (which will prevent ids from matching) num_part = int(id[num_index:]) if num_part == 0: print 'Media filename cannot have a zero digit component.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) #Insert hyphen if un-hyphenated id = alpha_part+'-'+str(num_part) if db_media_id_exists(domain, id, config): print 'The '+domain+' media id: '+id+' has already been used.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) else: print 'Wrong format for media filename: '+fn print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) return id
def generate_db_media_xml_element(media_path, domain, category, media_id, config): if media_in_category(media_path, category, config): if category == 'sony-xdcam': mfile = open(media_path, 'r') num_bytes_mp4_xml_data = 1143 try: mfile.seek(-num_bytes_mp4_xml_data, 2) except IOError, io: print 'Unable to read data from '+category+' file '+media_path print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) mp4xml = mfile.read() timestamp = get_sonyxdcam_timestamp(mp4xml) timecode = get_sonyxdcam_timecode(mp4xml) original_id = get_original_id(media_path) duration = get_media_duration(media_path) auto_split = get_autosplit(media_path) attributes = {'timestamp':timestamp, 'originalId':original_id, 'duration':duration, 'autoSplit':auto_split, 'timecode':timecode} elif category == 'video': original_id = get_original_id(media_path) duration = get_media_duration(media_path) attributes = {'originalId':original_id, 'duration':duration} elif category == 'audio': original_id = get_original_id(media_path) duration = get_media_duration(media_path) attributes = {'originalId':original_id, 'duration':duration} elif category == 'fostex-audio': mfile = open(media_path, 'r') num_bytes_bwf_xml_data = 10945 try: mfile.seek(num_bytes_bwf_xml_data, 0) except IOError, io: print 'Unable to read data from '+category+' file '+media_path print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) bwfxml = mfile.read(615) timestamp = get_fostexaudio_timestamp(bwfxml) original_id = get_original_id(media_path) duration = get_media_duration(media_path) attributes = {'timestamp':timestamp, 'originalId':original_id, 'duration':duration}
def check_restore_args(opts, args): if len(args) == 0: print 'Missing arguments.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) if opts.use_session_ids == True and opts.use_files == True: print 'You cannot specify both the -s and -f options simultaneously.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) if not (opts.media_type == None or opts.media_type == 'video' or opts.media_type == 'audio' or opts.media_type == 'image'): print 'Invalid media type: "'+opts.media_type+'".' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def db_import_tape_xml(config, tape_xml_doc): tape_xml = str(tape_xml_doc.toxml()) print '\nImporting tape index to database:\n' username = raw_input('username: '******'password: '******'%s:%s' % (username, password))) authheader = "Basic %s" % base64string headers = {"Authorization": authheader, "Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} params = urllib.urlencode({'tapeXML': tape_xml}) conn = httplib.HTTPConnection(ltoUtil.get_host_port(config)) try: conn.request('POST', ltoUtil.get_lto_url(config)+'/xquery/import-tape-element.xql', params, headers) response = conn.getresponse() data = response.read() if '<exception>' in data: print ltoUtil.get_xquery_exception_msg(data) print 'Unable to update database.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) elif 'HTTP ERROR: 404' in data: print '\nHTTP ERROR: 404' print data[data.find('<title>')+7:data.find('</title>')] print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) elif response.status == 401 and response.reason == 'Unauthorized': print 'Authentication Failed.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) else: print '\nDatabase updated (tape id: '+data+').' except httplib.HTTPException, e: print e.msg print 'Unable to connect to database' print ltoUtil.get_script_name()+' script terminated.' conn.close() sys.exit(2)
def media_file_types_check(config, category, path): category_filetypes = config.get('CategoryFileTypes', category).split(',') media_file_count = 0 total_file_size = 0 non_category_files = [] for dirpath, dirnames, filenames in os.walk(path): for file in filenames: if media_in_domain(file, 'video', config) or media_in_domain(file, 'audio', config) or media_in_domain(file, 'image', config): if not media_in_category(file, category, config): non_category_files.append(os.path.join(dirpath, file)) else: media_file_count += 1 total_file_size += ltoUtil.get_filesize(os.path.join(dirpath, file)) free_bytes = ltoUtil.get_freespace(ltoUtil.get_tar_build_dir(config)) if (len(non_category_files) > 0): print 'WARNING: The following unexpected media files were found in the folder: '+path+'\n' for f in non_category_files: print f print '\nExpecting only '+string.upper(string.join(category_filetypes, ','))+' files for "'+category+'" category.' print '\nNOTE: This situation should only occur if when one device has generated multiple media formats. (e.g. a video camera taking stills/audio as well as video)' print 'Usually this indicates that the files have been misplaced, in which case the user should manually remove them before re-running the script.\n' proceed = raw_input('Do you still want to proceed, including these files in the archive? [y/n]: ') if proceed == 'y': return else: print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) elif media_file_count == 0: print 'No recognised media files were found in: '+path print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) elif total_file_size > free_bytes - (1024*1024): print 'The total file size of the media files to be archived ('+ltoUtil.format_bytes_to_gbs(total_file_size)+') exceeds the free space available in '+ltoUtil.get_tar_build_dir(config)+' ('+ltoUtil.format_bytes_to_gbs(free_bytes)+').' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
print 'Unable to read data from '+category+' file '+media_path print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) bwfxml = mfile.read(615) timestamp = get_fostexaudio_timestamp(bwfxml) original_id = get_original_id(media_path) duration = get_media_duration(media_path) attributes = {'timestamp':timestamp, 'originalId':original_id, 'duration':duration} elif category == 'marantz-audio': mfile = open(media_path, 'r') num_bytes_wf_str_data = 364 try: mfile.seek(num_bytes_wf_str_data, 0) except IOError, io: print 'Unable to read data from '+category+' file '+media_path print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) str = mfile.read(18) timestamp = get_marantzaudio_timestamp(str) original_id = get_original_id(media_path) duration = get_media_duration(media_path) attributes = {'timestamp':timestamp, 'originalId':original_id, 'duration':duration} elif category == 'audio': duration = get_media_duration(media_path) attributes = {'originalId':original_id,'duration':duration} elif category == 'image': timestamp = get_image_timestamp(media_path) original_id = get_original_id(media_path) attributes = {'timestamp':timestamp, 'originalId':original_id} else: original_id = get_original_id(media_path)
def check_tar_args(args): if not len(args) == 5: print 'Exactly five arguments must be provided. (session id, device code, media path, username, password).' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def generate_proxy(config, domain, filepath): proxy_dir = ltoUtil.get_proxy_dir(config) thumb_dir = ltoUtil.get_thumb_dir(config) proxy_suffix = 'proxy' thumb_suffix = 'thumb' if os.path.exists(proxy_dir+'/'+domain): if (domain =='image' or domain == 'video'): if os.path.exists(thumb_dir+'/'+domain): if (domain == 'video'): video_type = get_video_type(filepath) if video_type == 'DV_PAL' or video_type == 'DV_NTSC' or video_type == 'MPEG2_H-14': proxy_size = '384x288' thumb_size = '96x72' elif video_type == 'MPEG2_HL': proxy_size = '512x288' thumb_size = '128x72' fn = ltoUtil.get_filename(filepath) fn_proxy = fn[0:-4]+'_'+proxy_suffix+'.mp4' fn_thumb = fn[0:-4]+'_'+thumb_suffix+'.jpg' p = subprocess.Popen('ffmpeg -y -i '+filepath+' -pass 1 -vcodec libx264 -coder 1 -flags +loop -cmp +chroma -partitions -parti8x8-parti4x4-partp8x8-partp4x4-partb8x8 -me_method dia -subq 1 -me_range 16 -g 25 -keyint_min 25 -sc_threshold 40 -i_qfactor 0.71 -b_strategy 1 -qcomp 0.6 -qmin 10 -qmax 51 -qdiff 4 -bf 4 -refs 1 -directpred 3 -trellis 0 -flags2 -bpyramid-wpred-mixed_refs-dct8x8+fastpskip -s '+proxy_size+' -b 512k -bt 512k -threads 0 -f mp4 -an /dev/null && ffmpeg -y -i '+filepath+' -pass 2 -acodec libfaac -ar 44100 -ab 96k -vcodec libx264 -coder 1 -flags +loop -cmp +chroma -partitions +parti8x8+parti4x4+partp8x8+partb8x8 -me_method umh -subq 8 -me_range 16 -g 25 -keyint_min 25 -sc_threshold 40 -i_qfactor 0.71 -b_strategy 2 -qcomp 0.6 -qmin 10 -qmax 51 -qdiff 4 -bf 4 -refs 4 -directpred 3 -trellis 1 -flags2 +bpyramid+wpred+mixed_refs+dct8x8+fastpskip -s '+proxy_size+' -b 512k -bt 512k -g 25 -threads 0 -f mp4 '+proxy_dir+'/video/'+fn_proxy, shell=True) sts = os.waitpid(p.pid, 0) os.remove('ffmpeg2pass-0.log') os.remove('x264_2pass.log') p = subprocess.Popen('ffmpeg -y -i '+proxy_dir+'/video/'+fn_proxy+' -vcodec mjpeg -vframes 1 -an -f rawvideo -s '+thumb_size+' '+thumb_dir+'/video/'+fn_thumb, shell=True) sts = os.waitpid(p.pid, 0) elif (domain =='image'): orientation = get_orientation(filepath) file_type = ltoUtil.get_file_extn(filepath) if orientation in [0,1,2,3,4] and file_type == 'jpg': proxy_size = '1024x680' thumb_size = '128x85' elif orientation in [0,1,2,3,4] and file_type in ['nef', 'cr2']: proxy_size = '1024x680' thumb_size = '128x85' elif orientation in [5,6,7,8] and file_type == 'jpg': proxy_size = '1024x680' thumb_size = '128x85' #We need to do this since we are extracting the embedded image from the raw files - which has no EXIF data elif orientation in [5,6,7,8] and file_type in ['nef', 'cr2']: proxy_size = '680x1024' thumb_size = '85x128' fn = ltoUtil.get_filename(filepath) fn_proxy = fn[0:-4]+'_'+proxy_suffix+'.jpg' fn_thumb = fn[0:-4]+'_'+thumb_suffix+'.jpg' if string.lower(file_type) == 'jpg': p = subprocess.Popen('convert -thumbnail '+proxy_size+' -auto-orient -unsharp 0x.5 -quality 92 '+filepath+' '+proxy_dir+'/image/'+fn_proxy, shell=True) sts = os.waitpid(p.pid, 0) #Much faster to extract the embedded image from the RAW file instead of resizing elif string.lower(file_type) == 'nef' or string.lower(file_type) == 'cr2': p = subprocess.Popen('dcraw -ce '+filepath+' | convert -thumbnail '+proxy_size+' -auto-orient -unsharp 0x.5 -quality 92 - '+proxy_dir+'/image/'+fn_proxy, shell=True) sts = os.waitpid(p.pid, 0) p = subprocess.Popen('convert -thumbnail '+thumb_size+' -auto-orient '+proxy_dir+'/image/'+fn_proxy+' '+thumb_dir+'/image/'+fn_thumb, shell=True) sts = os.waitpid(p.pid, 0) else: print 'thumbnail directory '+thumb_dir+'/'+domain+' does not exist.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) elif (domain =='audio'): fn = ltoUtil.get_filename(filepath) fn_proxy = fn[0:-4]+'_'+proxy_suffix+'.mp4' p = subprocess.Popen('ffmpeg -y -i '+filepath+' -acodec libfaac -ar 44100 -ab 96k -ac 1 -f mp4 '+proxy_dir+'/audio/'+fn_proxy, shell=True) sts = os.waitpid(p.pid, 0) else: print 'proxy directory '+proxy_dir+'/'+domain+' does not exist.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2)
def verify_tape(config, tape_id): print '\nVerifying index' bs = int(config.get('Tape', 'block_size_bytes')) tape = config.get('Tape', 'device') blocking_factor = bs/512 written_dir = ltoUtil.get_tape_written_dir(config)+'/'+tape_id temp_dir = ltoUtil.get_temp_dir(config) original_dir = temp_dir+'/verify-tape/'+tape_id+'/original' archived_dir = temp_dir+'/verify-tape/'+tape_id+'/archived' ltoUtil.create_dir(original_dir) ltoUtil.create_dir(archived_dir) tape_xml_file = tape_id+'.xml' #Extract index file from tape p = subprocess.Popen('tar -x -b '+str(blocking_factor)+' -C '+archived_dir+' -f '+tape+' '+tape_xml_file, shell=True) sts = os.waitpid(p.pid, 0) #Unpack index file from disk archive index_tar_path = written_dir+'/'+tape_xml_file+'.tar' if not os.path.exists(index_tar_path): print 'Verification Error: The tar file '+index_tar_path+' does not exist.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) p = subprocess.Popen('tar -x -b '+str(blocking_factor)+' -C '+original_dir+' -f '+written_dir+'/'+tape_xml_file+'.tar ', shell=True) sts = os.waitpid(p.pid, 0) #Compare index files original_index = original_dir+'/'+tape_xml_file archived_index = archived_dir+'/'+tape_xml_file if not os.path.exists(archived_index): print 'Unable to restore tape index file '+tape_xml_file+' from tape. Please check the correct tape is in the drive.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) if not os.path.exists(original_index): print 'Unable to extract tape index file '+tape_xml_file+' from '+written_dir+'/'+tape_xml_file+'.tar.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) if not ltoUtil.compare(original_index, archived_index): print '\nIndex file verification failed.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) print 'OK' print '\nVerifying tape\n' position = 2 #Parse index file into memory tape_xml_doc = xml.dom.minidom.parse(archived_index) tar_elems = tape_xml_doc.getElementsByTagName('tar') if len(tar_elems) == 0: print 'XML Error: The tape index file '+archived_index+' has no "tar" elements.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) #Loop thru all tars on tape disk_tarfiles_verified = [tape_xml_file+'.tar'] while True: p = subprocess.Popen('mt -f '+tape+' fsf 1', shell=True) sts = os.waitpid(p.pid, 0) tapedump_dir = archived_dir+'/'+str(position) diskdump_dir = original_dir+'/'+str(position) ltoUtil.create_dir(tapedump_dir) ltoUtil.create_dir(diskdump_dir) session_id = "" device_code = "" #Extract tarfile members from tape print 'Extracting tar archive #'+str(position)+' from tape' p = subprocess.Popen('tar -x -b '+str(blocking_factor)+' -C '+tapedump_dir+' -f '+tape, shell=True, stderr=subprocess.PIPE) stderr_value = p.stderr.read() sts = os.waitpid(p.pid, 0) if stderr_value == 'tar: This does not look like a tar archive\ntar: Error exit delayed from previous errors\n': print 'End of Data reached on tape' break t = tar_elems[position - 2] if int(t.getAttribute('position')) == position: session_id = t.getAttribute('sessionId') device_code = t.getAttribute('deviceCode') tar_name = session_id+'-'+device_code+'.tar' tarfile_path = written_dir+'/'+tar_name if not os.path.exists(tarfile_path): print 'Verification Error: The tar file '+tarfile_path+' does not exist.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) #Unpack corresponding tarfile on disk print 'Extracting corresponding local tar archive '+tarfile_path+'\n' p = subprocess.Popen('tar -x -b '+str(blocking_factor)+' -C '+diskdump_dir+' -f '+written_dir+'/'+tar_name, shell=True) sts = os.waitpid(p.pid, 0) #Compare the extracted files with each other (byte for byte), and ensure that index file is correct tar_members = [] for file in os.listdir(diskdump_dir): fp1 = os.path.join(diskdump_dir, file) fp2 = os.path.join(tapedump_dir, file) if not os.path.exists(fp2): print '\nVerification Error: The file '+file+' was found on the local tape archive '+tarfile_path+', but not on the tape.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) md5 = ltoUtil.compare(fp1, fp2) if not md5: print '\nVerification Error: The file '+file+' failed the byte comparison with the local copy.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) else: print file+' byte comparison with local copy successful' ref_file = session_id+'-'+device_code+'-referenced-items.xml' if not file == ref_file: #Verify each file against index file children = t.childNodes for c in children: if c.nodeType == c.ELEMENT_NODE and c.getAttribute('filename') == file: if md5 == c.getAttribute('md5'): print file+' MD5 check successful' else: print '\nVerification Error: The file '+file+' failed the md5 verification against the index file.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) grandchildren = c.childNodes for g in grandchildren: if g.nodeType == g.ELEMENT_NODE and g.getAttribute('filename') == file: if md5 == g.getAttribute('md5'): print file+' MD5 check successful' else: print '\nVerification Error: The file '+file+' failed the md5 verification against the index file.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) tar_members.append(file) #Check that there are no unmatched tar members on the tape for m in os.listdir(tapedump_dir): if not m in tar_members: print '\nVerification Error: The file '+m+' was found on the tape, but not on the corresponding local tape archive '+tarfile_path print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) disk_tarfiles_verified.append(tar_name) else: print 'Index File Error: Index file archives incorrectly ordered.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) #Delete temporary files print '\n' shutil.rmtree(original_dir) shutil.rmtree(archived_dir) position += 1 #Check that there are no unmatched tarfiles on the disk for f in os.listdir(written_dir): if f.endswith('.tar') and not f in disk_tarfiles_verified: print 'Verification Error: The tar archive '+written_dir+'/'+f+' was not found on the tape.' print ltoUtil.get_script_name()+' script terminated.' sys.exit(2) print 'Rewinding tape.' p = subprocess.Popen('mt -f '+tape+' rewind', shell=True, stderr=subprocess.PIPE) sts = os.waitpid(p.pid, 0) ltoUtil.terminate_on_error(p.stderr.read()) print '\nTape successfully verified.'