downloads_path = config.get('general',
'downloads_path',
fallback='/tmp/downloads/')
if not os.path.exists(downloads_path):
os.makedirs(downloads_path)
elif not os.path.isdir(downloads_path):
print('ERROR: downloads_path parameter points to file!')
sys.exit(1)
crawler = Crawler(config, 'captions')
crawler.get(config.get('captions', 'url'))
captions_list = []
for row in crawler.get_elements('tbody tr'):
items = crawler.get_elements('td', root=row)
filename = items[0].text
print("Current Filename is {}".format(filename))
url = crawler.get_attr('a', 'href', root=items[0])
crawler.download(url, filename)
convert_filename = filename.replace('.pdf', '.txt')
os.system("pdftotext '%s' '%s'" %
(downloads_path + 'Captions/' + filename,
downloads_path + 'Captions/' + convert_filename))
file_handle = open(downloads_path + 'Captions/' + convert_filename,
encoding="utf-8")
content = file_handle.readlines()
content = list(filter(None, [x.strip() for x in content]))
