def handle_query(self, query, tsn):
mname = False
if 'Command' in query and len(query['Command']) >= 1:
command = query['Command'][0]
# If we are looking at the root container
if (command == 'QueryContainer' and
(not 'Container' in query or query['Container'][0] == '/')):
self.root_container()
return
if 'Container' in query:
# Dispatch to the container plugin
basepath = query['Container'][0].split('/')[0]
if self.do_command(query, command, basepath, tsn):
return
elif command == 'QueryItem':
path = query.get('Url', [''])[0]
splitpath = [x for x in unquote_plus(path).split('/') if x]
if splitpath and not '..' in splitpath:
if self.do_command(query, command, splitpath[0], tsn):
return
elif (command == 'QueryFormats' and 'SourceFormat' in query
and query['SourceFormat'][0].startswith('video')):
if config.is_ts_capable(tsn):
self.send_xml(VIDEO_FORMATS_TS)
else:
self.send_xml(VIDEO_FORMATS)
return
elif command == 'QueryServer':
self.send_xml(SERVER_INFO)
return
elif command in ('GetActiveTransferCount', 'GetTransferStatus'):
plugin = GetPlugin('video')
if hasattr(plugin, command):
method = getattr(plugin, command)
method(self, query)
return True
elif command in ('FlushServer', 'ResetServer'):
# Does nothing -- included for completeness
self.send_response(200)
self.send_header('Content-Length', '0')
self.end_headers()
self.wfile.flush()
return
# If we made it here it means we couldn't match the request to
# anything.
self.unsupported(query)
def handle_query(self, query, tsn):
if 'Command' in query and len(query['Command']) >= 1:
command = query['Command'][0]
# If we are looking at the root container
if (command == 'QueryContainer' and
(not 'Container' in query or query['Container'][0] == '/')):
self.root_container()
return
if 'Container' in query:
# Dispatch to the container plugin
basepath = query['Container'][0].split('/')[0]
if self.do_command(query, command, basepath, tsn):
return
elif command == 'QueryItem':
path = query.get('Url', [''])[0]
splitpath = [x for x in unquote_plus(path).split('/') if x]
if splitpath and not '..' in splitpath:
if self.do_command(query, command, splitpath[0], tsn):
return
elif (command == 'QueryFormats' and 'SourceFormat' in query and
query['SourceFormat'][0].startswith('video')):
if config.is_ts_capable(tsn):
self.send_xml(VIDEO_FORMATS_TS)
else:
self.send_xml(VIDEO_FORMATS)
return
elif command == 'QueryServer':
self.send_xml(SERVER_INFO)
return
elif command in ('GetActiveTransferCount', 'GetTransferStatus'):
plugin = GetPlugin('video')
if hasattr(plugin, command):
method = getattr(plugin, command)
method(self, query)
return True
elif command in ('FlushServer', 'ResetServer'):
# Does nothing -- included for completeness
self.send_response(200)
self.send_header('Content-Length', '0')
self.end_headers()
self.wfile.flush()
return
# If we made it here it means we couldn't match the request to
# anything.
self.unsupported(query)
def use_ts(self, tsn, file_path):
if config.is_ts_capable(tsn):
if file_path[-5:].lower() == '.tivo':
try:
flag = file(file_path).read(8)
except:
return False
if ord(flag[7]) & 0x20:
return True
elif config.has_ts_flag():
return True
return False
def use_ts(self, tsn, file_path):
if config.is_ts_capable(tsn):
if file_path[-5:].lower() == '.tivo':
try:
flag = file(file_path).read(8)
except:
return False
if ord(flag[7]) & 0x20:
return True
elif config.has_ts_flag():
return True
return False
def use_ts(self, tsn, file_path):
if config.is_ts_capable(tsn):
ext = os.path.splitext(file_path)[1].lower()
if ext == '.tivo':
try:
flag = file(file_path).read(8)
except:
return False
if ord(flag[7]) & 0x20:
return True
else:
opt = config.get_ts_flag()
if ((opt == 'auto' and ext in LIKELYTS) or
(opt in ['true', 'yes', 'on'])):
return True
return False
def use_ts(self, tsn, file_path):
if config.is_ts_capable(tsn):
ext = os.path.splitext(file_path)[1].lower()
if ext == '.tivo':
try:
flag = file(file_path).read(8)
except:
return False
if ord(flag[7]) & 0x20:
return True
else:
opt = config.get_ts_flag()
if ((opt == 'auto' and ext in LIKELYTS)
or (opt in ['true', 'yes', 'on'])):
return True
return False
def get_tivo_file(self, tivoIP, url, mak, togo_path):
# global status
status[url].update({'running': True, 'queued': False})
outfile = self.get_out_file(url, tivoIP, togo_path)
auth_handler.add_password('TiVo DVR', url, 'tivo', mak)
try:
ts_format = False
if status[url]['ts_format'] and config.is_ts_capable(
config.tivos_by_ip(tivoIP)):
ts_format = True
handle = self.tivo_open(url + '&Format=video/x-tivo-mpeg-ts')
else:
handle = self.tivo_open(url)
except Exception, msg:
status[url]['running'] = False
status[url]['error'] = str(msg)
return
data = []
tivoIP = ''
TotalItems = 0
ItemStart = 0
ItemCount = 0
title = ''
t = Template(NPL_TEMPLATE, filter=EncodeUnicode)
t.escape = escape
t.quote = quote
t.folder = folder
t.status = status
if tivoIP in queue:
t.queue = queue[tivoIP]
t.has_tivodecode = has_tivodecode
t.togo_mpegts = config.is_ts_capable(tsn)
t.tname = tivo_name
t.tivoIP = tivoIP
t.container = handler.cname
t.data = data
t.len = len
t.TotalItems = getint(TotalItems)
t.ItemStart = getint(ItemStart)
t.ItemCount = getint(ItemCount)
t.FirstAnchor = quote(FirstAnchor)
t.shows_per_page = shows_per_page
t.title = title
handler.send_html(str(t), refresh='300')
def get_tivo_file(self, tivoIP, url, mak, togo_path):
# global status
def ToGo(handler, query):
"""
HTTP command handler to download a set of recordings from a Tivo.
If there is already a thread downloading recordings from that Tivo,
the new recordings will be appended to the existing download task
list for that Tivo, otherwise a new task list will be created and
a thread spawned to process it.
"""
togo_path = config.get_togo('path')
for name, data in config.getShares():
if togo_path == name:
togo_path = data.get('path')
if togo_path:
tivoIP = query['TiVo'][0]
tsn = config.tivos_by_ip(tivoIP)
tivo_name = config.tivos[tsn].get('name', tivoIP)
tivo_mak = config.get_tsn('tivo_mak', tsn)
urls = query.get('Url', [])
decode = 'decode' in query
save = 'save' in query
ts_format = 'ts_format' in query and config.is_ts_capable(tsn)
sortable = bool(config.get_togo('sortable_names', False))
for theurl in urls:
status = {'url': theurl,
'running': False,
'queued': True,
'finished': False,
'showinfo': showinfo[theurl], # metadata information about the show
'decode': decode, # decode the downloaded tivo file
'save': save, # save the tivo file's metadata to a .txt file
'ts_format': ts_format, # download using transport stream otherwise program stream
'sortable': sortable, # name saved tivo file in a sortable manner
'error': '',
'rate': 0,
'size': 0,
'retry': 0,
'download_attempts': [], # information about each download attempt (used for sync error log)
'ts_error_packets': [], # list of TS packets w/ sync lost as tuples (packet_no, count)
'best_attempt_index': None, # index into download_attempts of the attempt w/ fewest errors
'best_file': '',
'best_error_count': None} # count of TS packets lost (sync byte was wrong) in 'best_file'
with active_tivos_lock:
if tivoIP in active_tivos:
with active_tivos[tivoIP]['lock']:
active_tivos[tivoIP]['queue'].append(status)
else:
# we have to add authentication info again because the
# download netloc may be different from that used to
# retrieve the list of recordings (and in fact the port
# is different, 443 to get the NPL and 80 for downloading).
auth_handler.add_password('TiVo DVR', urlsplit(theurl).netloc, 'tivo', tivo_mak)
logger.debug('ToGo: add password for TiVo DVR netloc: %s', urlsplit(theurl).netloc)
active_tivos[tivoIP] = {'tivoIP': tivoIP,
'lock': RLock(),
'thread': None,
'tivo_name': tivo_name,
'mak': tivo_mak,
'dest_path': togo_path,
'fn_format_info': {'episode': config.get_togo('episode_fn'),
'movie': config.get_togo('movie_fn')
},
'ts_error_mode': config.get_togo('ts_error_mode', 'ignore'),
'ts_max_retries': int(config.get_togo('ts_max_retries', 0)),
'queue': [status]}
active_tivos[tivoIP]['thread'] = TivoDownload(tivoIP, active_tivos, active_tivos_lock, tivo_open)
active_tivos[tivoIP]['thread'].start()
logger.info('[%s] Queued "%s" for transfer to %s',
time.strftime('%d/%b/%Y %H:%M:%S'),
unquote(theurl), togo_path)
urlstring = '<br>'.join([unquote(x) for x in urls])
message = TRANS_QUEUE % (urlstring, togo_path)
else:
message = MISSING
handler.redir(message, 5)
def NPL(handler, query):
"""
ToGo.NPL returns an html page displaying the now playing list (NPL)
from a particular TiVo device.
The query may specify:
- TiVo: the IPv4 address of the TiVo whose NPL is to be retrieved
- ItemCount: the number of shows/folders to put on the page (default: 50, max: 50)
- AnchorItem: the url identifying the 1st item in the retrieved list (default 1st item in folder)
- AnchorOffset: the offset from the AnchorItem to start the retrieval from (default 0)
- SortOrder:
- Recurse:
"""
def getint(thing):
try:
result = int(thing)
except: # pylint: disable=bare-except
result = 0
return result
shows_per_page = 50 # Change this to alter the number of shows returned (max is 50)
if 'ItemCount' in query:
shows_per_page = int(query['ItemCount'][0])
if shows_per_page > 50:
shows_per_page = 50
folder = ''
FirstAnchor = ''
has_tivodecode = bool(config.get_bin('tivodecode'))
has_tivolibre = bool(config.get_bin('tivolibre'))
if 'TiVo' in query:
tivoIP = query['TiVo'][0]
try:
tsn = config.tivos_by_ip(tivoIP)
attrs = config.tivos[tsn]
tivo_name = attrs.get('name', tivoIP)
tivo_mak = config.get_tsn('tivo_mak', tsn)
except config.Error as e:
logger.error('NPL: %s', e)
t = Template(ERROR_TEMPLATE)
t.e = e
t.additional_info = 'Your browser may have cached an old page'
handler.send_html(str(t))
return
protocol = attrs.get('protocol', 'https')
ip_port = '%s:%d' % (tivoIP, attrs.get('port', 443))
path = attrs.get('path', DEFPATH)
baseurl = '%s://%s%s' % (protocol, ip_port, path)
theurl = baseurl
if 'Folder' in query:
folder = query['Folder'][0]
theurl = urljoin(theurl, folder)
theurl += '&ItemCount=%d' % shows_per_page
if 'AnchorItem' in query:
theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
if 'AnchorOffset' in query:
theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
if 'SortOrder' in query:
theurl += '&SortOrder=' + query['SortOrder'][0]
if 'Recurse' in query:
theurl += '&Recurse=' + query['Recurse'][0]
if (theurl not in tivo_cache or
(time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
# if page is not cached or old then retrieve it
auth_handler.add_password('TiVo DVR', ip_port, 'tivo', tivo_mak)
logger.debug('NPL: (1) add password for TiVo DVR netloc: %s', ip_port)
try:
logger.debug("NPL.theurl: %s", theurl)
with tivo_open(theurl) as page:
tivo_cache[theurl] = {'thepage': minidom.parse(page),
'thepage_time': time.time()}
except IOError as e:
handler.redir(UNABLE % (tivoIP, html.escape(str(e))), 10)
return
xmldoc = tivo_cache[theurl]['thepage']
items = xmldoc.getElementsByTagName('Item')
TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
if items:
FirstAnchor = tag_data(items[0], 'Links/Content/Url')
data = []
for item in items:
entry = {}
for tag in ('CopyProtected', 'ContentType'):
value = tag_data(item, 'Details/' + tag)
if value:
entry[tag] = value
if entry['ContentType'].startswith('x-tivo-container'):
entry['Url'] = tag_data(item, 'Links/Content/Url')
entry['Title'] = tag_data(item, 'Details/Title')
entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
lc = tag_data(item, 'Details/LastCaptureDate')
if not lc:
lc = tag_data(item, 'Details/LastChangeDate')
entry['LastChangeDate'] = time.strftime('%b %d, %Y',
time.localtime(int(lc, 16)))
else:
keys = {'Icon': 'Links/CustomIcon/Url',
'Url': 'Links/Content/Url',
'Details': 'Links/TiVoVideoDetails/Url',
'SourceSize': 'Details/SourceSize',
'Duration': 'Details/Duration',
'CaptureDate': 'Details/CaptureDate'}
for key in keys:
value = tag_data(item, keys[key])
if value:
entry[key] = value
if 'SourceSize' in entry:
rawsize = entry['SourceSize']
entry['SourceSize'] = metadata.human_size(rawsize)
if 'Duration' in entry:
dur = getint(entry['Duration']) // 1000
entry['Duration'] = ('%d:%02d:%02d' %
(dur // 3600, (dur % 3600) // 60, dur % 60))
if 'CaptureDate' in entry:
entry['CaptureDate'] = time.strftime('%b %d, %Y',
time.localtime(int(entry['CaptureDate'], 16)))
dnld_url = entry['Url']
# the tivo download url seems to always be absolute, so is this necessary?
# I'm commenting it out -mjl 7/23/2017
#dnld_url = urljoin(baseurl, dnld_url)
if not dnld_url in showinfo:
showinfo[dnld_url] = ShowInfo()
showinfo[dnld_url].from_tivo_container_item(item)
entry.update(showinfo[dnld_url].get_old_basicmeta())
data.append(entry)
else:
data = []
tivoIP = ''
TotalItems = 0
ItemStart = 0
ItemCount = 0
title = ''
tsn = ''
tivo_name = ''
t = Template(NPL_TEMPLATE)
t.quote = quote
t.folder = folder
t.urlstatus = ToGo.get_urlstatus(tivoIP)
t.has_tivodecode = has_tivodecode
t.has_tivolibre = has_tivolibre
t.togo_mpegts = config.is_ts_capable(tsn)
t.tname = tivo_name
t.tivoIP = tivoIP
t.container = handler.cname
t.data = data
t.len = len
t.TotalItems = getint(TotalItems)
t.ItemStart = getint(ItemStart)
t.ItemCount = getint(ItemCount)
t.FirstAnchor = quote(FirstAnchor)
t.shows_per_page = shows_per_page
t.title = title
handler.send_html(str(t), refresh='300')
else:
data = []
tivoIP = ''
TotalItems = 0
ItemStart = 0
ItemCount = 0
title = ''
t = Template(NPL_TEMPLATE, filter=EncodeUnicode)
t.quote = quote
t.folder = folder
t.status = status
if tivoIP in queue:
t.queue = queue[tivoIP]
t.has_tivodecode = has_tivodecode
t.togo_mpegts = config.is_ts_capable(tsn)
t.tname = tivo_name
t.tivoIP = tivoIP
t.container = handler.cname
t.data = data
t.len = len
t.TotalItems = getint(TotalItems)
t.ItemStart = getint(ItemStart)
t.ItemCount = getint(ItemCount)
t.FirstAnchor = quote(FirstAnchor)
t.shows_per_page = shows_per_page
t.title = title
handler.send_html(str(t), refresh='300')
def get_tivo_file(self, tivoIP, url, mak, togo_path):
# global status
def ToGo(self, handler, query):
togo_path = config.get_server('togo_path')
for name, data in config.getShares():
if togo_path == name:
togo_path = data.get('path')
if togo_path:
tivoIP = query['TiVo'][0]
tsn = config.tivos_by_ip(tivoIP)
tivo_mak = config.get_tsn('tivo_mak', tsn)
urls = query.get('Url', [])
decode = 'decode' in query
save = 'save' in query
if 'postprocess' in query:
postprocess = query['postprocess'][0]
else:
postprocess = config.get_server('vrd_post_processing')
if 'postprocess_profile' in query:
postprocess_profile = query['postprocess_profile'][0]
else:
postprocess_profile = config.get_server('vrd_profile', '')
postprocess_decrypt = 'postprocess_decrypt' in query
if not postprocess_decrypt:
try:
postprocess_decrypt = config.config.getboolean(
'Server', 'vrd_decrypt_qsf')
except:
postprocess_decrypt = False
postprocess_delete = 'postprocess_delete' in query
if not postprocess_delete:
try:
postprocess_delete = config.config.getboolean(
'Server', 'vrd_delete_on_success')
except:
postprocess_delete = False
ts_format = 'ts_format' in query and config.is_ts_capable(tsn)
for theurl in urls:
if theurl in status:
del status[theurl]
status[theurl] = {
'running':
False,
'status':
'',
'error':
'',
'rate':
0,
'percent':
0,
'queued':
True,
'size':
0,
'postprocessing':
False,
'finished':
False,
'decode':
decode,
'save':
save,
'ts_format':
ts_format,
'postprocess':
postprocess,
'postprocess_profile':
postprocess_profile,
'postprocess_decrypt':
postprocess_decrypt,
'postprocess_delete':
postprocess_delete,
'retry':
0,
'ts_max_retries':
int(config.get_server('togo_ts_max_retries', 0)),
'ts_error_count':
0,
'best_file':
'',
'best_error_count':
0
}
if tivoIP in queue:
queue[tivoIP].append(theurl)
else:
queue[tivoIP] = [theurl]
thread.start_new_thread(
ToGo.process_queue,
(self, tivoIP, tivo_mak, togo_path))
logger.info('[%s] Queued "%s" for transfer to %s' %
(time.strftime('%d/%b/%Y %H:%M:%S'),
unquote(theurl), togo_path))
urlstring = '<br>'.join(
[unicode(unquote(x), 'utf-8') for x in urls])
message = TRANS_QUEUE % (urlstring, togo_path)
else:
message = MISSING
handler.redir(message, 5)
def get_out_file(self, url, tivoIP, togo_path):
# Use TiVo Desktop style naming
if url in basic_meta:
if 'title' in basic_meta[url]:
title = basic_meta[url]['title']
episodeTitle = ''
if 'episodeTitle' in basic_meta[url]:
episodeTitle = basic_meta[url]['episodeTitle']
recordDate = datetime.now()
if 'recordDate' in basic_meta[url]:
recordDate = datetime.fromtimestamp(
int(basic_meta[url]['recordDate'], 0), pytz.utc)
callSign = ''
if 'callsign' in basic_meta[url]:
callSign = basic_meta[url]['callsign']
count = 1
while True:
fileName = title
try:
sortable = config.config.getboolean(
'Server', 'togo_sortable_names')
except:
sortable = False
if sortable == True:
fileName += ' - '
fileName += recordDate.strftime('%Y-%m-%d')
if len(episodeTitle):
fileName += ' - \'\'' + episodeTitle + '\'\''
if len(callSign):
fileName += ' (' + callSign + ')'
else:
if len(episodeTitle):
fileName += ' - \'\'' + episodeTitle + '\'\''
fileName += ' (Recorded '
fileName += recordDate.strftime('%b %d, %Y')
if len(callSign):
fileName += ', ' + callSign
fileName += ')'
ts = status[url]['ts_format'] and config.is_ts_capable(
config.tivos_by_ip(tivoIP))
if not status[url]['decode']:
if ts:
fileName += ' (TS)'
else:
fileName += ' (PS)'
if count > 1:
fileName += ' (%d)' % count
if status[url]['decode']:
if ts:
fileName += '.ts'
else:
fileName += '.mpg'
else:
fileName += '.tivo'
for ch in BADCHAR:
fileName = fileName.replace(ch, BADCHAR[ch])
if os.path.isfile(os.path.join(togo_path, fileName)):
count += 1
continue
return os.path.join(togo_path, fileName)
# If we get here then use old style naming
parse_url = urlparse.urlparse(url)
name = unicode(unquote(parse_url[2]),
'utf-8').split('/')[-1].split('.')
try:
id = unquote(parse_url[4]).split('id=')[1]
name.insert(-1, ' - ' + id)
except:
pass
ts = status[url]['ts_format'] and config.is_ts_capable(
config.tivos_by_ip(tivoIP))
if status[url]['decode']:
if ts:
name[-1] = 'ts'
else:
name[-1] = 'mpg'
else:
if ts:
name.insert(-1, ' (TS)')
else:
name.insert(-1, ' (PS)')
nameHold = name
name.insert(-1, '.')
count = 2
newName = name
while (os.path.isfile(os.path.join(togo_path, ''.join(newName)))):
newName = nameHold
newName.insert(-1, ' (%d)' % count)
newName.insert(-1, '.')
count += 1
name = newName
name = ''.join(name)
for ch in BADCHAR:
name = name.replace(ch, BADCHAR[ch])
return os.path.join(togo_path, name)
def NPL(self, handler, query):
global basic_meta
shows_per_page = 50 # Change this to alter the number of shows returned
folder = ''
has_tivodecode = bool(config.get_bin('tivodecode'))
if 'TiVo' in query:
tivoIP = query['TiVo'][0]
tsn = config.tivos_by_ip(tivoIP)
togo_mpegts = config.is_ts_capable(tsn)
useragent = handler.headers.getheader('User-Agent', '')
tivo_name = config.tivo_names[tsn]
tivo_mak = config.get_tsn('tivo_mak', tsn)
theurl = ('https://' + tivoIP +
'/TiVoConnect?Command=QueryContainer&ItemCount=' +
str(shows_per_page) + '&Container=/NowPlaying')
if 'Folder' in query:
folder += query['Folder'][0]
theurl += '/' + folder
if 'AnchorItem' in query:
theurl += '&AnchorItem=' + quote(query['AnchorItem'][0])
if 'AnchorOffset' in query:
theurl += '&AnchorOffset=' + query['AnchorOffset'][0]
if (theurl not in tivo_cache or
(time.time() - tivo_cache[theurl]['thepage_time']) >= 60):
# if page is not cached or old then retreive it
auth_handler.add_password('TiVo DVR', tivoIP, 'tivo', tivo_mak)
try:
page = self.tivo_open(theurl)
except IOError, e:
handler.redir(UNABLE % tivoIP, 10)
return
tivo_cache[theurl] = {'thepage': minidom.parse(page),
'thepage_time': time.time()}
page.close()
xmldoc = tivo_cache[theurl]['thepage']
items = xmldoc.getElementsByTagName('Item')
TotalItems = tag_data(xmldoc, 'TiVoContainer/Details/TotalItems')
ItemStart = tag_data(xmldoc, 'TiVoContainer/ItemStart')
ItemCount = tag_data(xmldoc, 'TiVoContainer/ItemCount')
FirstAnchor = tag_data(items[0], 'Links/Content/Url')
title = tag_data(xmldoc, 'TiVoContainer/Details/Title')
data = []
for item in items:
entry = {}
entry['ContentType'] = tag_data(item, 'Details/ContentType')
for tag in ('CopyProtected', 'UniqueId'):
value = tag_data(item, 'Details/' + tag)
if value:
entry[tag] = value
if entry['ContentType'] == 'x-tivo-container/folder':
entry['Title'] = tag_data(item, 'Details/Title')
entry['TotalItems'] = tag_data(item, 'Details/TotalItems')
lc = tag_data(item, 'Details/LastCaptureDate')
if not lc:
lc = tag_data(item, 'Details/LastChangeDate')
entry['LastChangeDate'] = time.strftime('%b %d, %Y',
time.localtime(int(lc, 16)))
else:
keys = {'Icon': 'Links/CustomIcon/Url',
'Url': 'Links/Content/Url',
'SourceSize': 'Details/SourceSize',
'Duration': 'Details/Duration',
'CaptureDate': 'Details/CaptureDate'}
for key in keys:
value = tag_data(item, keys[key])
if value:
entry[key] = value
rawsize = entry['SourceSize']
entry['SourceSize'] = metadata.human_size(rawsize)
dur = int(entry['Duration']) / 1000
entry['Duration'] = ( '%d:%02d:%02d' %
(dur / 3600, (dur % 3600) / 60, dur % 60) )
entry['CaptureDate'] = time.strftime('%b %d, %Y',
time.localtime(int(entry['CaptureDate'], 16)))
url = entry['Url']
if url in basic_meta:
entry.update(basic_meta[url])
else:
basic_data = metadata.from_container(item)
entry.update(basic_data)
basic_meta[url] = basic_data
data.append(entry)