class BatchProcessor(QObject):
'''
Worker thread
'''
signals = ObjectSignals()
setLightPWM = pyqtSignal(float) # control signal
gotoXY = pyqtSignal(
float, float, bool) # boolean indicate relative to stage origin or not
gotoX = pyqtSignal(float, bool)
gotoY = pyqtSignal(float, bool)
gotoZ = pyqtSignal(float)
findDiaphragm = pyqtSignal()
disableMotors = pyqtSignal()
findWell = pyqtSignal()
# computeSharpnessScore = pyqtSignal()
rSnapshotTaken = pyqtSignal() # repeater signal
# rSharpnessScore = pyqtSignal() # repeater signal
rWellFound = pyqtSignal() # repeat signal
rDiaphragmFound = pyqtSignal() # repeat signal
rClipRecorded = pyqtSignal() # repeat signal
rPositionReached = pyqtSignal() # repeat signal
takeSnapshot = pyqtSignal(str)
recordClip = pyqtSignal(str, int)
setLogFileName = pyqtSignal(str)
stopCamera = pyqtSignal()
startCamera = pyqtSignal()
startAutoFocus = pyqtSignal(float)
focussed = pyqtSignal() # repeater signal
def __init__(self):
super().__init__()
self.settings = QSettings("settings.ini", QSettings.IniFormat)
self.loadSettings()
self.timer = QTimer(self)
self.timer.timeout.connect(self.run)
self.isInterruptionRequested = False
self.foundDiaphragmLocation = None
self.foundWellLocation = None
self.lightLevel = 1.0 # initial value, will be set later by user
# self.sharpnessScore = 0
def loadSettings(self):
self.msg("info;loading settings from {:s}".format(
self.settings.fileName()))
self.resolution = float(
self.settings.value('camera/resolution_in_px_per_mm'))
def loadBatchSettings(self):
self.msg("info;loading batch settings from {:s}".format(
self.batch_settings.fileName()))
# load run info
self.run_id = self.batch_settings.value('run/id')
self.run_note = self.batch_settings.value('run/note')
t = time.strptime(
self.batch_settings.value('run/duration')[1], '%H:%M:%S')
days = int(self.batch_settings.value('run/duration')[0].split('d')[0])
self.run_duration_s = (
(24 * days + t.tm_hour) * 60 + t.tm_min) * 60 + t.tm_sec
t = time.strptime(self.batch_settings.value('run/wait'), '%H:%M:%S')
self.run_wait_s = (t.tm_hour * 60 + t.tm_min) * 60 + t.tm_sec
s = str(self.batch_settings.value('run/shutdown'))
self.shutdown = s.lower() in ['true', '1', 't', 'y', 'yes']
s = str(self.batch_settings.value('run/snapshot'))
self.snapshot = s.lower() in ['true', '1', 't', 'y', 'yes']
s = str(self.batch_settings.value('run/videoclip'))
self.videoclip = s.lower() in ['true', '1', 't', 'y', 'yes']
self.videoclip_length = int(
self.batch_settings.value('run/clip_length'))
# load well-plate dimensions and compute well locations
self.plate_note = self.batch_settings.value('plate/note')
self.nr_of_columns = int(
self.batch_settings.value('plate/nr_of_columns'))
self.nr_of_rows = int(self.batch_settings.value('plate/nr_of_rows'))
self.A1_to_side_offset = float(
self.batch_settings.value('plate/A1_to_side_offset'))
self.column_well_spacing = float(
self.batch_settings.value('plate/column_well_spacing'))
self.A1_to_top_offset = float(
self.batch_settings.value('plate/A1_to_top_offset'))
self.row_well_spacing = float(
self.batch_settings.value('plate/row_well_spacing'))
self.computeWellLocations()
def computeWellLocations(self):
# load the wells to process
nr_of_wells = self.batch_settings.beginReadArray("wells")
self.wells = []
for i in range(0, nr_of_wells):
self.batch_settings.setArrayIndex(i)
well_id = self.batch_settings.value('id')
well_note = self.batch_settings.value('note')
r = re.split('(\d+)', well_id)
row = ord(r[0].lower()) - 96
col = int(r[1])
location_mm = [round(self.A1_to_side_offset + (col-1)*self.column_well_spacing, 2), \
round(self.A1_to_top_offset + (row-1)*self.row_well_spacing, 2), \
0]
self.wells.append(
Well(name=well_id, position=[row, col], location=location_mm))
self.batch_settings.endArray(
) # close array, also required when opening!
def msg(self, text):
if text:
text = self.__class__.__name__ + ";" + str(text)
print(text)
self.signals.message.emit(text)
@pyqtSlot()
def start(self):
# open storage folder
# dlg = QFileDialog()
# self.storage_path = QFileDialog.getExistingDirectory(dlg, 'Open storage folder', '/media/pi/', QFileDialog.ShowDirsOnly | QFileDialog.DontResolveSymlinks)
# if self.storage_path == "" or self.storage_path is None:
# return
self.storage_path = os.path.sep.join(
[os.getcwd(), self.settings.value('temp_folder')])
# clear temporary storage path
os.system('rm {:s}'.format(os.path.sep.join([self.storage_path,
"*.*"])))
# open batch definition file
dlg = QFileDialog()
self.batch_file_name = QFileDialog.getOpenFileName(
dlg, 'Open batch definition file', os.getcwd(),
"Ini file (*.ini)")[0]
if self.batch_file_name == "" or self.batch_file_name is None:
return
self.batch_settings = QSettings(self.batch_file_name,
QSettings.IniFormat)
self.loadBatchSettings()
# copy batch definition file to storage folder
os.system('cp {:s} {:s}'.format(self.batch_file_name,
self.storage_path))
note_file_name = os.path.sep.join(
[self.storage_path, self.run_id + ".log"])
self.setLogFileName.emit(note_file_name)
# copy files to webdav host
conn_settings = QSettings("connections.ini", QSettings.IniFormat)
self.webdav_path = os.path.sep.join(
[conn_settings.value('webdav/storage_path'), self.run_id])
self.webdav_options = {
'webdav_hostname': conn_settings.value('webdav/hostname'),
'webdav_login': conn_settings.value('webdav/login'),
'webdav_password': conn_settings.value('webdav/password')
}
try:
self.webdav_client = Client(self.webdav_options)
self.webdav_client.mkdir(self.webdav_path)
self.webdav_client.push(remote_directory=self.webdav_path,
local_directory=self.storage_path)
except WebDavException as err:
traceback.print_exc()
self.signals.error.emit(
(type(err), err.args, traceback.format_exc()))
# create temporary image storage path
self.image_storage_path = os.path.sep.join([self.storage_path, 'img'])
if not os.path.exists(self.image_storage_path):
os.makedirs(self.image_storage_path)
# start-up recipe
self.msg("info;plate note: {:s}".format(self.plate_note))
self.msg("info;run note: {:s}".format(self.run_note))
self.msg("info;{:d} wells found in {:s}".format(
len(self.wells), self.batch_settings.fileName()))
self.msg("info;{:s} run during {:d}s with {:d}s interleave".format(
self.run_id, self.run_duration_s, self.run_wait_s))
self.setLightPWM.emit(0.2)
self.startCamera.emit()
self.msg("info;goto first well")
x = self.wells[0].location[0]
y = self.wells[0].location[1]
z = 33 #self.wells[0].location[2]
self.gotoXY.emit(x, y, True)
self.wait_signal(self.rPositionReached, 10000)
self.gotoZ.emit(z)
self.wait_signal(self.rPositionReached, 10000)
# Let the user set the x,y,z-location manually by moving to first well and open dialog
dialog = ManualPositioningDialog(x, y, z)
dialog.stageXTranslation.valueChanged.connect(
lambda x: self.gotoX.emit(x, True))
dialog.stageYTranslation.valueChanged.connect(
lambda y: self.gotoY.emit(y, True))
dialog.stageZTranslation.valueChanged.connect(
lambda z: self.gotoZ.emit(z))
dialog.light.valueChanged.connect(
lambda x: self.setLightPWM.emit(x / 100))
dialog.exec_()
self.A1_to_side_offset = round(dialog.stageXTranslation.value(), 3)
self.A1_to_top_offset = round(dialog.stageYTranslation.value(), 3)
z = round(dialog.stageZTranslation.value(), 3)
self.lightLevel = round(dialog.light.value() / 100, 3)
self.msg("info;user set well A1 at (x,y)=({:.3f},{:.3f})".format(
self.A1_to_side_offset, self.A1_to_top_offset))
self.msg("info;user set focus at z={:.3f}".format(z))
self.msg("info;user set light intensity at {:.3f}".format(
self.lightLevel))
# recompute the well locations in our batch
self.computeWellLocations()
for well in self.wells:
well.location[
2] = z # copy z, later on we may do autofocus per well
# start timer
self.prev_note_nr = 0 # for logging
self.start_time_s = time.time()
self.timer.start(0)
def run(self):
''' Timer call back function, als initiates next one-shot
'''
start_run_time_s = time.time()
self.setLightPWM.emit(self.lightLevel)
self.startCamera.emit()
self.msg("info;go home")
self.gotoXY.emit(0, 0, False)
self.wait_signal(self.rPositionReached, 10000)
self.msg("info;goto first well")
self.gotoXY.emit(self.wells[0].location[0], self.wells[0].location[1],
True)
self.wait_signal(self.rPositionReached, 10000)
# self.wait_ms(60000) # wait for camera to adjust to light
for well in self.wells:
self.msg("info;gauging well {:s})".format(well.name))
# split goto xyz command
self.gotoX.emit(well.location[0], True)
self.wait_signal(self.rPositionReached, 10000)
self.gotoY.emit(well.location[1], True)
self.wait_signal(self.rPositionReached, 10000)
self.gotoZ.emit(well.location[2])
self.wait_signal(self.rPositionReached, 10000)
# autofocus
if self.batch_settings.value('run/autofocus', False, type=bool):
self.new_z = 0
self.startAutoFocus.emit(well.location[2])
self.wait_signal(self.focussed, 100000)
if self.new_z != 0:
well.location[2] = self.new_z
else:
# focus failed, so return to initial z position
self.gotoZ.emit(well.location[2])
self.wait_signal(self.rPositionReached, 10000)
try:
# clear temporary storage path
os.system('rm {:s}'.format(
os.path.sep.join([self.image_storage_path, "*.*"])))
except err:
traceback.print_exc()
self.signals.error.emit(
(type(err), err.args, traceback.format_exc()))
# take snapshot or video
self.wait_ms(2000)
prefix = os.path.sep.join([
self.image_storage_path,
str(well.position) + "_" + str(well.location)
])
if self.snapshot:
self.takeSnapshot.emit(prefix)
self.wait_signal(self.rSnapshotTaken) # snapshot taken
if self.videoclip and self.videoclip_length > 0:
self.recordClip.emit(prefix, self.videoclip_length)
self.wait_signal(self.rClipRecorded) # clip recorded
try:
# push data
remote_path = os.path.sep.join([self.webdav_path, well.name])
self.msg(": info; pushing data to {}".format(remote_path))
self.webdav_client.mkdir(remote_path)
self.webdav_client.push(
remote_directory=remote_path,
local_directory=self.image_storage_path)
# push log file
self.webdav_client.push(remote_directory=self.webdav_path,
local_directory=self.storage_path)
except WebDavException as err:
traceback.print_exc()
self.signals.error.emit(
(type(err), err.args, traceback.format_exc()))
# Wrapup current round of acquisition
self.setLightPWM.emit(0.00)
self.stopCamera.emit()
elapsed_total_time_s = time.time() - self.start_time_s
elapsed_run_time_s = time.time() - start_run_time_s
self.msg("info;single run time={:.1f}s".format(elapsed_run_time_s))
self.msg("info;total run time={:.1f}s".format(elapsed_total_time_s))
progress_percentage = int(100 * elapsed_total_time_s /
self.run_duration_s)
self.signals.progress.emit(progress_percentage)
self.msg("info;progress={:d}%".format(progress_percentage))
# send a notification
note_nr = int(progress_percentage / 10)
if note_nr != self.prev_note_nr:
self.prev_note_nr = note_nr
message = """Subject: Progress = {}% \n\n Still {} s left""".format(
progress_percentage,
int(self.run_duration_s - elapsed_total_time_s))
# do something fancy here in future: https://realpython.com/python-send-email/#sending-fancy-emails
self.sendNotification(message)
# check if we still have time to do another round
if elapsed_total_time_s + self.run_wait_s < self.run_duration_s:
self.timer.setInterval(self.run_wait_s * 1000)
self.msg("info;wait for {:.1f} s".format(self.run_wait_s))
else:
self.timer.stop()
self.signals.ready.emit()
self.msg("info;run finalized")
message = """Subject: run finalized"""
# do something fancy here in future: https://realpython.com/python-send-email/#sending-fancy-emails
self.sendNotification(message)
if self.shutdown:
self.msg("info;emitting finished")
self.signals.finished.emit()
def sendNotification(self, message):
conn_settings = QSettings("connections.ini", QSettings.IniFormat)
port = 465 # For SSL
context = ssl.create_default_context() # Create a secure SSL context
try:
with smtplib.SMTP_SSL("smtp.gmail.com", port,
context=context) as server:
login = conn_settings.value('smtp/login')
password = conn_settings.value('smtp/password')
server.login(login, password)
server.sendmail(conn_settings.value('smtp/login'), \
conn_settings.value('subscriber/email'), \
message)
except Exception as err:
traceback.print_exc()
self.signals.error.emit(
(type(err), err.args, traceback.format_exc()))
def requestInterruption(self):
self.isInterruptionRequested = True
@pyqtSlot(np.ndarray)
def diaphragmFound(self, location):
self.foundDiaphragmLocation = location
self.msg("info;diaphragmFound signal received")
self.rDiaphragmFound.emit()
@pyqtSlot(np.ndarray)
def wellFound(self, location):
self.foundWellLocation = location
self.msg("info;wellFound signal received")
self.rWellFound.emit()
# @pyqtSlot(float)
# def setSharpnessScore(self, score):
# self.sharpnessScore = score
# self.msg("info;sharpnessScore signal received")
# self.rSharpnessScore.emit()
@pyqtSlot()
def snapshotTaken(self):
self.msg("info;snapshotTaken signal received")
self.rSnapshotTaken.emit()
@pyqtSlot()
def clipRecorded(self):
self.msg("info;clipRecorded signal received")
self.rClipRecorded.emit()
@pyqtSlot()
def positionReached(self):
self.msg("info;positionReached signal received")
self.rPositionReached.emit()
@pyqtSlot(float)
def focussedSlot(self, val):
self.new_z = val
self.focussed.emit()
@pyqtSlot()
def stop(self):
self.msg("info;stopping")
self.requestInterruption()
if self.timer.isActive():
self.timer.stop()
self.signals.finished.emit()
def wait_ms(self, timeout):
''' Block loop until timeout (ms) elapses.
'''
loop = QEventLoop()
QTimer.singleShot(timeout, loop.exit)
loop.exec_()
def wait_signal(self, signal, timeout=100):
''' Block loop until signal emitted, or timeout (ms) elapses.
'''
loop = QEventLoop()
signal.connect(loop.quit) # only quit is a slot of QEventLoop
QTimer.singleShot(timeout, loop.exit)
loop.exec_()
class WebDAV:
"""Commerce Cloud WebDAV session.
Args:
client (CommerceCloudClientSession): Active client session with Commerce Cloud for a bearer token.
instance (str, optional): Optional commerce cloud instance, useful for opening clients to multiple instances using the same bearer token. Defaults to None.
cert (str, optional): Path to TLS client certificate. Defaults to None.
key ([type], optional): Export key for the TLS certificate. Defaults to None.
verify (bool, optional): Verify TLS certificates, set to false for self signed. Defaults to True.
"""
def __init__(self,
client,
instance=None,
cert=None,
key=None,
verify=True):
self.client = client
self._instance = instance or self.client.instance
self.options = {"webdav_hostname": self._instance.rstrip("/")}
self.verify = verify
self.token = self.client.Token
self.options.update({"webdav_token": self.token["access_token"]})
self.webdav_client = Client(self.options)
self.webdav_client.verify = self.verify
if cert and key:
self.cert = str(Path(cert).resolve())
self.key = str(Path(key).resolve())
self.options.update({"cert_path": self.cert, "key_path": self.key})
def reauth(self):
"""Checks token expiry and re-initialises the Client if a new token is needed.
"""
if self.token["expires_at"] < int(time.time()):
self.client.getToken()
self.options.update({"webdav_token": self.token["access_token"]})
self.webdav_client = Client(self.options)
def reconnect(self):
"""Re-initalise the Client session.
"""
self.webdav_client = Client(self.options)
@property
def hostname(self):
"""Return the hostname the WebDAV client connection is connected to.
Returns:
str: Hostname including prefix eg https://
"""
return self.options["webdav_hostname"]
@property
def netloc(self):
"""Return a urlparse netloc string of the connected hostname.
Returns:
str: netloc of hostname.
"""
url = urlparse(self.options["webdav_hostname"])
return url.netloc
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
def GetInfo(self, remote_filepath: str, headers: dict = None) -> list:
"""Get properties for entity
[extended_summary]
Args:
remote_filepath (str): Path to remote resource.
headers (dict, optional): Additional headers to apply to request. Defaults to None.
Raises:
RetryException: Adds to retries counter on failure.
Returns:
list: WebDAV attribute information.
"""
try:
return self.webdav_client.info(remote_filepath)
except (NoConnection, ConnectionException, WebDavException):
self.reauth()
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
def GetDirectoryList(self,
filepath: str,
get_info: bool = False,
headers: dict = None) -> list:
"""Get list of files and folders in a path from WebDAV endpoint.
[extended_summary]
Args:
filepath (str): Path to get directory listing for.
get_info (bool): returns dictionary of attributes instead of file list.
headers (dict, optional): Additional headers to apply to request. Defaults to None.
Returns:
list: Directory listing.
"""
try:
return self.webdav_client.list(filepath, get_info=get_info)
except (NoConnection, ConnectionException, WebDavException):
self.reauth()
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
def Upload(self, local_filepath: str, remote_filepath: str):
"""Upload file or directory recursively to WebDAV endpoint.
[extended_summary]
Args:
local_filepath (str): Local path to file or directory to upload.
remote_filepath (str): Remote path to upload to.
"""
local_filepath = str(Path(local_filepath).resolve())
try:
self.webdav_client.upload_sync(remote_filepath, local_filepath)
except (NoConnection, ConnectionException, WebDavException):
self.reauth()
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
def StreamUpload(self, payload, remote_path: str, file_name: str):
"""Upload FileIO, StringIO, BytesIO or string to WebDAV
[extended_summary]
Args:
payload: Stream payload
remote_path (str): Remote path relative to host.
file_name (str): Name for the file uploaded.
"""
try:
self.webdav_client.upload_to(payload, f"{remote_path}/{file_name}")
except (NoConnection, ConnectionException, WebDavException):
self.reauth()
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
def MakeDir(self, remote_path: str):
"""Make new directory at path specified.
Args:
remote_path (str): Path of proposed new directory.
"""
try:
self.webdav_client.mkdir(remote_path)
except (NoConnection, ConnectionException, WebDavException):
self.reauth()
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
def Move(self,
remote_path_source: str,
remote_path_dest: str,
overwrite: bool = False):
"""Make new directory at path specified.
Args:
remote_path_source (str): Path of source resource.
remote_path_dest (str): Path of destination resource.
overwrite (bool): Overwrite destination resource. Defaults to False.
"""
try:
self.webdav_client.move(remote_path_source, remote_path_dest,
overwrite)
except (NoConnection, ConnectionException, WebDavException):
self.reauth()
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
def Delete(self, remote_filepath: str):
"""Delete file on remote WebDAV endpoint.
Args:
remote_filepath (str): Location of resource to delete.
"""
try:
self.webdav_client.clean(remote_filepath)
except (NoConnection, ConnectionException, WebDavException):
self.reauth()
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
def Download(self, local_filepath: str, remote_filepath: str):
"""Download file/folder from WebDAV endpoint.
This is a synchronous operation, and the file is downloaded in full to the local_filepath.
Args:
local_filepath (str): Local path to download to, including filename of file saved.
remote_filepath (str): Remote path to file to download.
"""
local_filepath = str(Path(local_filepath).resolve())
try:
self.webdav_client.download_sync(remote_filepath, local_filepath)
except (NoConnection, ConnectionException, WebDavException):
self.reauth()
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
def Pull(self, local_filepath: str, remote_filepath: str):
"""Sync file/folder from WebDAV endpoint to local storage.
This downloads missing or nwer modified files from the remote to local storage.
You can use it to do "resumeable" transfers, but the checks are slow for deeply nested files.
Args:
local_filepath (str): Local path to download to, including filename of file saved.
remote_filepath (str): Remote path to file to download.
"""
local_filepath = str(Path(local_filepath).resolve())
try:
self.webdav_client.pull(remote_filepath, local_filepath)
return True
except (NoConnection, ConnectionException, WebDavException):
self.reauth()
raise RetryException
return False
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=3,
retry_window_after_first_call_in_seconds=10,
)
def Push(self, local_filepath: str, remote_filepath: str):
"""Sync file/folder from local storage to WebDAV endpoint.
This uploads missing or nwer modified files from the local to remote storage.
You can use it to do "resumeable" transfers, but the checks are slow for deeply nested files.
Args:
local_filepath (str): Local path to download to, including filename of file saved.
remote_filepath (str): Remote path to file to download.
"""
local_filepath = str(Path(local_filepath).resolve())
try:
self.webdav_client.push(local_filepath, remote_filepath)
return True
except (NoConnection, ConnectionException, WebDavException):
self.reauth()
raise RetryException
return False
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=10,
retry_window_after_first_call_in_seconds=15,
)
def StreamDownload(self,
remote_filepath: str,
buffer=None,
decode: bool = False):
"""Download a file in chunks to a local file buffer.
You must provide a BytesIO object or one will be created for you.
Args:
remote_filepath (str): Path to remote resource to download.
buffer ([type], optional): Buffer write streamed content to.
decode (bool, optional): Optionally try to decode downloaded file into a string. Defaults to False.
Raises:
RetryException: Adds to retries counter on failure.
Returns:
Bytes: Returns a BytesIO object for further use.
"""
self.reauth()
if buffer is None:
buffer = BytesIO()
try:
self.webdav_client.download_from(buff=buffer,
remote_path=remote_filepath)
if decode is True:
return buffer.getvalue().decode("utf-8")
else:
buffer.seek(0)
return buffer
except (NoConnection, ConnectionException, WebDavException):
raise RetryException
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=10,
retry_window_after_first_call_in_seconds=60,
)
def HashObject(self, remote_filepath: str) -> str:
"""Generate a MD5 hashsum for a remote resource.
This is streamed into memory, hashed and discarded. Optimised for low memory but
high bandwidth environments.
Args:
remote_filepath (str): Path to remote resource.
Raises:
RetryException: Adds to retries counter on failure.
Returns:
str: MDSSUM of the file requested.
"""
self.reauth()
try:
sum = md5(self.StreamDownload(remote_filepath).getbuffer())
return {
"filepath": remote_filepath,
"hashtype": "MD5",
"hashsum": sum.hexdigest(),
}
except (NoConnection, ConnectionException, WebDavException):
self.reconnect()
raise RetryException
def RecursiveFileListing(self, remote_filepath: str) -> str:
"""Recursive filetree walker, returns paths found.
Args:
remote_filepath (str): [description]
Raises:
RetryException: Adds to retries counter on failure.
Yields:
Iterator[str]: Yields resource paths for any files found.
"""
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=10,
retry_window_after_first_call_in_seconds=60,
)
def get_list(self, path):
self.reauth()
try:
return self.webdav_client.list(path, get_info=True)
except (NoConnection, ConnectionException, WebDavException):
self.reconnect()
raise RetryException
def get_files(self, path):
return [x for x in get_list(self, path) if x["isdir"] is False]
def get_dirs(self, path):
return [
x["path"] for x in get_list(self, path) if x["isdir"] is True
]
yield from get_files(self, remote_filepath)
for subdir in get_dirs(self, remote_filepath):
yield from self.RecursiveFileListing(subdir)
def RecursiveFolderListing(self, remote_filepath: str) -> str:
"""Recursive filetree walker, returns paths found.
Args:
remote_filepath (str): [description]
Raises:
RetryException: Adds to retries counter on failure.
Yields:
Iterator[str]: Yields resource paths for any files found.
"""
@retry(
retry_on_exceptions=(RetryException),
max_calls_total=10,
retry_window_after_first_call_in_seconds=60,
)
def get_list(self, path):
self.reauth()
try:
return self.webdav_client.list(path, get_info=True)
except (NoConnection, ConnectionException, WebDavException):
self.reconnect()
raise RetryException
def get_dirs(self, path):
return [
x["path"] for x in get_list(self, path) if x["isdir"] is True
]
dirlist = get_dirs(self, remote_filepath)
yield from dirlist
for subdir in get_dirs(self, remote_filepath):
yield from self.RecursiveFolderListing(subdir)
