def test_dl_dry_cancel():
bdl = bd.BulkDownloader('https://feeds.radiokawa.com/podcast_nawak.xml',
'./dl')
assert len(bdl.list_mp3()) > 0
cb = Callback()
cb.cancel()
bdl.download_mp3(dry_run=True, cb=cb)
def download_with_resume(url: str, path: str, cb: Callback = None) -> bool:
"""
Download a file pointed by url to a local path
@param url: URL to download
@param path: Local file to be saved
@param cb: Callback object
@return: True if the file was completely downloaded
"""
logging.debug("Downloading {} to {}".format(url, path))
# Clean existing file
if os.path.exists(path):
os.remove(path)
if cb and cb.is_cancelled():
return False
try:
r = requests.head(url, allow_redirects=True)
except requests.exceptions as e:
logging.error(e)
return False
if r.status_code < 200 or r.status_code > 302:
logging.error("Failed to reach {}, status is {}".format(url, r.status_code))
r.close()
return False
expected_size = int(r.headers.get("content-length"))
r.close()
if cb and cb.is_cancelled():
return False
chunk_size = 2**20
last_byte = 0
with open(path, 'wb') as f:
while last_byte < expected_size:
if cb and cb.is_cancelled():
return False
logging.debug("{} vs {}".format(last_byte, expected_size))
logging.debug("Starting download with already {}% of the file".
format((100*last_byte)/expected_size))
resume_header = {'Range': 'bytes=%d-' % last_byte}
resume_request = requests.get(url, headers=resume_header, stream=True,
verify=True, allow_redirects=True)
for data in resume_request.iter_content(chunk_size):
last_byte += len(data)
if cb and cb.is_cancelled():
return False
if cb:
cb.progress(100 * (last_byte / expected_size))
f.write(data)
resume_request.close()
if cb and cb.is_cancelled():
return False
if cb:
cb.progress(100)
return True
def __init__(self, path=""):
self.configpath = path
self.root = Tk()
self.cameras = Camera()
self.parser = Parser()
self.gui = GUI(root=self.root)
self.callback = Callback(self.root)
self.isImage_active = False
self.isVideo_active = False
self.isCamera_active = False
self.isObject_active = False
def list_mp3(self, cb: Callback = None, verbose: bool = False) -> List[Episode]:
"""
Will fetch the RSS or directory info and return the list of available MP3s
@param cb: Callback object
@param verbose: Outputs more logs
@return: List of MP3 urls
"""
try:
r = requests.get(self._url)
except requests.RequestException as exc:
err_str = 'Failed to connect to URL ({})'.format(exc)
logging.error(err_str)
raise BulkDownloaderException(err_str)
if r.status_code != 200:
err_str = 'Failed to access URL (code {})'.format(r.status_code)
logging.error(err_str)
raise BulkDownloaderException(err_str)
page = r.content
if cb and cb.is_cancelled():
return []
if self._page_is_rss(page):
logging.info('Processing RSS document')
to_download = self._get_episodes_to_download_from_rss(page)
# We trim the list if needed
if 0 < self._last_n < len(to_download):
to_download = to_download[0:self._last_n]
else:
err_str = 'Content is not RSS'
logging.error(err_str)
raise BulkDownloaderException(err_str)
if cb and cb.is_cancelled():
return []
if verbose:
logging.info('{} episodes found in the feed:'.format(len(to_download)))
for elem in to_download:
logging.info(elem)
return to_download
def try_download(url, path, max_try=3, sleep_time=5, cb: Callback = None) -> bool:
"""
Try to download the file multiple times, in case of connection failures
@param url: URL to download
@param path: Local file to be saved
@param max_try: Number of download tries
@param sleep_time: Wait time between tries in second
@param cb: Callback object
@return: True if the file was completely downloaded
"""
count = 0
while count < max_try:
if download_with_resume(url, path, cb):
return True
if cb and cb.is_cancelled():
return False
count += 1
sleep(sleep_time)
logging.error('Download of {} failed after {} tries'.format(url, max_try))
return False
def __init__(self, master):
Frame.__init__(self, master)
master.title('Podcast Bulk Downloader v{}'.format(pbd_version))
# master.geometry('500x800')
style = ttk.Style()
self._style = StringVar()
if 'vista' in style.theme_names():
self._style.set('vista')
else:
self._style.set('default')
style.theme_use(self._style.get())
# Layout configuration
columns = 0
while columns < 10:
master.columnconfigure(columns, weight=1)
columns += 1
rows = 0
while rows < 5:
w = 1 if rows != 3 else 5
master.rowconfigure(rows, weight=w)
rows += 1
# First line
self._label_rss = ttk.Label(master, text='Feed')
self._label_rss.grid(row=0, column=0, padx=2, pady=2, sticky=W + E)
self._entry_rss = ttk.Entry(master)
self._entry_rss.grid(row=0,
column=1,
padx=2,
pady=2,
columnspan=9,
sticky=W + E)
# Second line
self._label_folder = ttk.Label(master, text='Folder')
self._label_folder.grid(row=1, column=0, padx=2, sticky=W + E)
self._entry_folder = ttk.Entry(master)
self._entry_folder.grid(row=1,
column=1,
padx=2,
columnspan=8,
sticky=W + E)
self._btn_nav = ttk.Button(master,
text='...',
command=self.browse_directory)
self._btn_nav.grid(row=1, column=9, padx=2, pady=2, sticky=W + E)
# Third line
self._overwrite = IntVar()
self._cb_overwrite = ttk.Checkbutton(master,
text='Overwrite existing files',
variable=self._overwrite,
onvalue=1,
offvalue=0)
self._cb_overwrite.grid(row=2,
column=0,
columnspan=2,
sticky=W + E,
padx=2,
pady=2)
self._btn_fetch = ttk.Button(master, text='Fetch', command=self.fetch)
self._btn_fetch.grid(row=2,
column=7,
columnspan=1,
sticky=W + E,
padx=2,
pady=2)
self._btn_download = ttk.Button(master,
text='Download',
command=self.download)
self._btn_download.grid(row=2,
column=8,
columnspan=1,
sticky=W + E,
padx=2,
pady=2)
self._btn_cancel = ttk.Button(master,
text='Cancel',
command=self.cancel)
self._btn_cancel.grid(row=2,
column=9,
columnspan=1,
sticky=W + E,
padx=2,
pady=2)
# Fourth line
self._progress_bar = ttk.Progressbar(master,
orient='horizontal',
mode='determinate')
self._progress_bar.grid(row=3,
column=0,
columnspan=10,
sticky=W + E + N + S,
padx=2,
pady=2)
self._progress_bar["maximum"] = 100
# Fifth line
self._text = Text(master)
self._text.grid(row=4,
column=0,
columnspan=10,
sticky=W + E + N + S,
padx=2,
pady=2)
self._text.configure(state=DISABLED)
self._logger = Log2Text(self._text)
logging.getLogger().setLevel(logging.INFO)
logging.getLogger().addHandler(self._logger)
# Utilities
self._dl = BulkDownloader(self._entry_rss.get(),
self._entry_folder.get())
self._dl_thread = Thread(target=None)
self._fetch_thread = Thread(target=None)
self._callback = Callback(self._progress_bar)
# Launch background task
self.reset_buttons()
class PDBApp(Frame):
def __init__(self, master):
Frame.__init__(self, master)
master.title('Podcast Bulk Downloader v{}'.format(pbd_version))
# master.geometry('500x800')
style = ttk.Style()
self._style = StringVar()
if 'vista' in style.theme_names():
self._style.set('vista')
else:
self._style.set('default')
style.theme_use(self._style.get())
# Layout configuration
columns = 0
while columns < 10:
master.columnconfigure(columns, weight=1)
columns += 1
rows = 0
while rows < 5:
w = 1 if rows != 3 else 5
master.rowconfigure(rows, weight=w)
rows += 1
# First line
self._label_rss = ttk.Label(master, text='Feed')
self._label_rss.grid(row=0, column=0, padx=2, pady=2, sticky=W + E)
self._entry_rss = ttk.Entry(master)
self._entry_rss.grid(row=0,
column=1,
padx=2,
pady=2,
columnspan=9,
sticky=W + E)
# Second line
self._label_folder = ttk.Label(master, text='Folder')
self._label_folder.grid(row=1, column=0, padx=2, sticky=W + E)
self._entry_folder = ttk.Entry(master)
self._entry_folder.grid(row=1,
column=1,
padx=2,
columnspan=8,
sticky=W + E)
self._btn_nav = ttk.Button(master,
text='...',
command=self.browse_directory)
self._btn_nav.grid(row=1, column=9, padx=2, pady=2, sticky=W + E)
# Third line
self._overwrite = IntVar()
self._cb_overwrite = ttk.Checkbutton(master,
text='Overwrite existing files',
variable=self._overwrite,
onvalue=1,
offvalue=0)
self._cb_overwrite.grid(row=2,
column=0,
columnspan=2,
sticky=W + E,
padx=2,
pady=2)
self._btn_fetch = ttk.Button(master, text='Fetch', command=self.fetch)
self._btn_fetch.grid(row=2,
column=7,
columnspan=1,
sticky=W + E,
padx=2,
pady=2)
self._btn_download = ttk.Button(master,
text='Download',
command=self.download)
self._btn_download.grid(row=2,
column=8,
columnspan=1,
sticky=W + E,
padx=2,
pady=2)
self._btn_cancel = ttk.Button(master,
text='Cancel',
command=self.cancel)
self._btn_cancel.grid(row=2,
column=9,
columnspan=1,
sticky=W + E,
padx=2,
pady=2)
# Fourth line
self._progress_bar = ttk.Progressbar(master,
orient='horizontal',
mode='determinate')
self._progress_bar.grid(row=3,
column=0,
columnspan=10,
sticky=W + E + N + S,
padx=2,
pady=2)
self._progress_bar["maximum"] = 100
# Fifth line
self._text = Text(master)
self._text.grid(row=4,
column=0,
columnspan=10,
sticky=W + E + N + S,
padx=2,
pady=2)
self._text.configure(state=DISABLED)
self._logger = Log2Text(self._text)
logging.getLogger().setLevel(logging.INFO)
logging.getLogger().addHandler(self._logger)
# Utilities
self._dl = BulkDownloader(self._entry_rss.get(),
self._entry_folder.get())
self._dl_thread = Thread(target=None)
self._fetch_thread = Thread(target=None)
self._callback = Callback(self._progress_bar)
# Launch background task
self.reset_buttons()
def reset_buttons(self):
if not self._fetch_thread.is_alive() and not self._dl_thread.is_alive(
):
self._switch_action(False)
self._callback.reset()
self.after(100, self.reset_buttons)
def browse_directory(self):
cur_dir = self._entry_folder.get()
initial_dir = cur_dir if os.path.exists(
cur_dir) else os.path.expanduser('~')
directory = filedialog.askdirectory(title='Select directory',
initialdir=initial_dir)
if directory:
self._entry_folder.delete(0, END)
self._entry_folder.insert(0, directory)
def _clean_text_box(self):
try:
self._text.configure(state=NORMAL)
self._text.delete('1.0', END)
self._text.configure(state=DISABLED)
except TclError as exc:
logging.warning('Can\'t clean text ({})'.format(exc))
def _update_dl_with_fields(self):
self._dl._url = self._entry_rss.get()
self._dl.folder(self._entry_folder.get())
self._dl.overwrite(self._overwrite.get() == 1)
def _switch_action(self, action: bool):
state_f_dl = DISABLED if action else NORMAL
state_cancel = NORMAL if action else DISABLED
self._btn_download.configure(state=state_f_dl)
self._btn_fetch.configure(state=state_f_dl)
self._btn_cancel.configure(state=state_cancel)
def download(self):
self._clean_text_box()
self._update_dl_with_fields()
logging.info("Start download")
self._dl_thread = Thread(target=self._dl.download_mp3,
kwargs={'cb': self._callback})
self._switch_action(True)
self._dl_thread.start()
def fetch(self):
self._clean_text_box()
self._update_dl_with_fields()
logging.info("Fetch info")
self._fetch_thread = Thread(target=self._dl.list_mp3,
kwargs={
'verbose': True,
'cb': self._callback
})
self._switch_action(True)
self._fetch_thread.start()
def cancel(self):
self._callback.cancel()
logging.info('Action cancelled by user, waiting for threads to end...')
if self._fetch_thread.is_alive():
self._fetch_thread.join()
if self._dl_thread.is_alive():
self._dl_thread.join()
self._callback.reset()
self._switch_action(False)
logging.info('Threads have ended')
def test_list_mp3():
bdl = bd.BulkDownloader('https://feeds.radiokawa.com/podcast_nawak.xml',
'./dl')
cb = Callback()
assert len(bdl.list_mp3(cb, True)) > 0
def test_try_download_cancel(tmp_directory):
cb = Callback()
cb.cancel()
assert not bd.try_download('https://feeds.radiokawa.com/podcast_nawak.xml',
os.path.join(tmp_directory, 't.xml'), 1, 1, cb)
def test_try_download_ok(tmp_directory):
cb = Callback()
assert bd.try_download('http://xerto.free.fr/newban.jpg',
os.path.join(tmp_directory, 'newban.jpg'), 2, 1, cb)
def download_mp3(self, cb: Callback = None, dry_run: bool = False):
"""
Will get the list of MP3s and download them into the specified folder
@param cb: Callback object
@param dry_run: Will not actually download anythin (for test purposes only)
@return: None
"""
if not self.folder():
err_str = 'No folder is defined for the download'
logging.error(err_str)
raise BulkDownloaderException(err_str)
to_download = self.list_mp3(cb)
logging.info('{} files will be downloaded'.format(len(to_download)))
if cb and cb.is_cancelled():
return
if cb:
cb.progress(0)
count = 0
downloads_successful = 0
downloads_skipped = 0
nb_downloads = len(to_download)
step = 100. / nb_downloads
for episode in to_download:
if cb:
if cb.is_cancelled():
continue
cb.progress(count * step)
# Getting the name and path
path = os.path.join(self.folder(), episode.get_filename())
# Check if we should skip the file
if not self.overwrite() and os.path.isfile(path):
logging.info('Skipping {} as the file already exists at {}'
.format(episode.get_filename(), path))
downloads_skipped += 1
count += 1
continue
# Download file
logging.info('Saving {} to {} from {}'.format(episode.get_filename(), path,
episode.url()))
if cb:
cb.set_function(lambda x: (count + x / 100) * step)
if not dry_run and try_download(episode.url(), path, cb=cb):
downloads_successful += 1
if cb:
cb.set_function(lambda x: x)
count += 1
if cb and cb.is_cancelled():
return
if cb:
cb.progress(100)
logging.info('{}/{} episodes were successfully downloaded'.format(downloads_successful,
nb_downloads))
logging.info('{}/{} episodes were skipped because files already existed'
.format(downloads_skipped, nb_downloads))
class Main:
def __init__(self, path=""):
self.configpath = path
self.root = Tk()
self.cameras = Camera()
self.parser = Parser()
self.gui = GUI(root=self.root)
self.callback = Callback(self.root)
self.isImage_active = False
self.isVideo_active = False
self.isCamera_active = False
self.isObject_active = False
#---------- Local callback ----------#
def openwithimage_callback(self, event=None):
if self.gui.panel != None:
self.gui.destroyPanel()
path = self.callback.openwithimage_callback()
if path != None:
self.cameras.readImage(path, flag=None)
self.cameras.frame = self.cameras.resize(self.cameras.getFrame,
(400, 400))
width, height = self.cameras.getSize()
image = self.cameras.convert2tk(self.cameras.getFrame)
self.gui.createPanel(image, width=width, height=height)
self.isImage_active = True
self.isVideo_active = False
self.isCamera_active = False
def openwithvideo_callback(self, event=None):
if self.gui.panel != None:
self.gui.destroyPanel()
path = self.callback.openwithvideo_callback(event=event)
if path != None:
self.cameras.startVideo(path)
startCapture = self.cameras.readCapture()
startCapture = self.cameras.BGR2RGBA(startCapture)
startCapture = self.cameras.convert2tk(startCapture)
width, height = self.cameras.getSize()
self.gui.createPanel(startCapture, width=width, height=height)
self.captureVideo(loop=True)
self.isImage_active = False
self.isVideo_active = True
self.isCamera_active = False
def openwithcamera_callback(self, event=None):
if self.gui.panel != None:
self.gui.destroyPanel()
settings = self.parser.parseCameraSettings()
src = settings["src"]
try:
self.cameras.startVideo(src=src)
startCapture = self.cameras.readCapture()
startCapture = self.cameras.BGR2RGB(startCapture)
startCapture = self.cameras.convert2tk(startCapture)
width, height = self.cameras.getSize()
self.gui.createPanel(startCapture, width=width, height=height)
self.captureVideo(loop=False)
self.isImage_active = False
self.isVideo_active = False
self.isCamera_active = True
except error:
self.gui.message("Failed to open camera with device " + str(src),
2)
# self.gui.destroyPanel()
def detectBarcode_callback(self, event=None):
self.isObject_active = True
if self.isObject_active:
if self.cameras.getFrame is None:
self.gui.message("No object")
else:
if self.isCamera_active is None:
newframe = self.detectBarcode()
newframe = self.cameras.convert2tk(newframe)
self.gui.update(newframe)
#---------- Edit menu ----------#
def clearframe_callback(self, event=None):
self.callback.clearframe_callback(self.gui)
#---------- Capture Video ----------#
def captureVideo(self, loop=False):
try:
captured = self.cameras.readCapture(islopp=loop, gui=self.gui)
captured = self.cameras.BGR2RGB(captured)
self.cameras.frame = captured
if captured is None:
pass
else:
if self.isObject_active:
frame = self.detectBarcode(
customFrame=self.cameras.getFrame)
if frame is None:
frame = self.cameras.getFrame
else:
frame = self.cameras.getFrame
newframe = self.cameras.convert2tk(frame)
self.gui.update(newframe)
self.gui.delay(15, self.captureVideo)
except error:
self.gui.message("Asserting failed, please check your camera!")
self.gui.destroyPanel()
#---------- Detect barcode ----------#
def detectBarcode(self, customFrame=None):
if customFrame is None:
customFrame = self.cameras.getFrame
image = self.cameras.BGR2Gray(self.cameras.getFrame)
barcode = pyzbar.decode(image)
if len(barcode) == 0: return None
for data in barcode:
points = data.polygon
x, y, w, h = data.rect
pts = self.cameras.cv2array(points)
pts = pts.reshape((-1, 1, 2))
newframe = self.cameras.createpolygon(self.cameras.frame, pts)
code = data.data.decode("utf-8")
newframe = self.cameras.setText(newframe,
text=code,
position=(x, y))
return newframe
#---------- Setup menbubars function ----------#
def setupMenubars(self):
menubars = Menu(self.root)
#---------- File bars ----------#
fileBars = Menu(menubars, tearoff=0)
menubars.add_cascade(label="File", menu=fileBars)
fileBars.add_command(label="Open with image file",
command=self.openwithimage_callback)
fileBars.add_command(label="Open with video file",
command=self.openwithvideo_callback)
fileBars.add_command(label="Open with camera",
command=self.openwithcamera_callback)
fileBars.add_separator()
fileBars.add_command(label="Save barcode", command=print)
fileBars.add_separator()
fileBars.add_command(label="Quit", command=self.callback.quit_callback)
#---------- CTRL + O (Open with image file) ----------#
self.root.bind("<Control-o>", self.openwithimage_callback)
self.root.bind("<Control-O>", self.openwithimage_callback)
#---------- CTRL + Shift + O (Open with video file) ----------#
self.root.bind("<Control-Shift-o>", self.openwithvideo_callback)
self.root.bind("<Control-Shift-O>", self.openwithvideo_callback)
#---------- CTRL + Q (Quit) ----------#
self.root.bind("<Control-Q>", self.callback.quit_callback)
self.root.bind("<Control-q>", self.callback.quit_callback)
#---------- Edit bars ----------#
editbars = Menu(menubars, tearoff=0)
menubars.add_cascade(label="Edit", menu=editbars)
editbars.add_command(label="Clear frame",
command=self.clearframe_callback)
editbars.add_command(label="Detect barcode",
command=self.detectBarcode_callback)
#---------- CTRL + Shift + C (Clear Frame) ----------#
self.root.bind("<Control-Shift-c>", self.clearframe_callback)
self.root.bind("<Control-Shift-C>", self.clearframe_callback)
#---------- Settings bar ----------
settingbars = Menu(menubars, tearoff=0)
menubars.add_cascade(label="Settings", menu=settingbars)
settingbars.add_command(label="Settings camera",
command=self.callback.setCameraDevice_callback)
self.root.config(menu=menubars)
#---------- Setup function ----------#
def setup(self):
callback = [
self.openwithimage_callback, self.openwithvideo_callback,
self.openwithcamera_callback, self.clearframe_callback,
self.detectBarcode_callback
]
w, h = self.gui.getWindowSize() # get window size
self.gui.setTitle("QrScanner")
self.gui.setScreensize(size=convert2geometry(w=w, h=h))
self.setupMenubars()
self.gui.createButton(command=callback)
def run(self):
self.root.mainloop()