def upload(options=options): client = Client(options) if not client.check("/server"): client.mkdir("/server") path="/server/"+str(int(time.time())) client.mkdir(path) client.upload(path,"./dat/") print("Data backup succeeded!")
class WebDavServiceBase: def __init__(self, conf: WebDavConfig): url = f"http{'s' if conf.use_https else ''}://{conf.host}:{conf.port}{conf.path}{conf.root_dir}" options = { 'webdav_hostname': url, 'webdav_login': conf.username, 'webdav_password': conf.password, 'webdav_timeout': 600 } self.client = Client(options) if conf.force_direct: self.client.session.proxies = {} self.sem = threading.Semaphore(5) def dir_exists(self, path: str): try: # rclone doesn't allow you check an existing dir return self.client.check(path) except webdav3.exceptions.MethodNotSupported as e: print(e) return True def ensure_dir(self, tag: str, path: str): service_name = f"{SERVICE_NAME}:{tag}" if ServiceKVStore.exists(service_name, path): return if self.dir_exists(path): ServiceKVStore.put(service_name, path, {}) return self.client.mkdir(path) ServiceKVStore.put(service_name, path, {}) def write_file(self, tag: str, service: ServiceType, dir_name: str, filename: str, buffer: IO): with self.sem: self.ensure_dir(tag, service.value) self.ensure_dir(tag, f"{service.value}/{dir_name}") fp = f"{service.value}/{dir_name}/{filename}" with NamedTemporaryFile() as f: f.write(buffer.read()) f.flush() try: self.client.upload(fp, f.name) except webdav3.exceptions.RemoteParentNotFound as err: print(err) pp = f"{service.value}/{dir_name}" # assert not ServiceKVStore.exists(SERVICE_NAME, pp) # self.client.mkdir(pp) from webdav3.urn import Urn directory_urn = Urn(pp, directory=True) response = self.client.execute_request( action='mkdir', path=directory_urn.quote()) assert response in (200, 201), response ServiceKVStore.put(SERVICE_NAME, pp, {}) self.client.upload(fp, f.name) return fp
class webDavService: def __init__(self): self.conf = config() log = logger() self.path = os.path.dirname(os.path.realpath(sys.argv[0])) self.confwebdav = log.getlogger('webdav') self.options = {} self.client = None def loadBaseInfo(self): url = self.conf.decrypt(self.conf.getOption('webDav', 'url')) if url.endswith("/"): url = url[0:-1] self.options = { 'webdav_hostname': url, 'webdav_login': self.conf.decrypt(self.conf.getOption('webDav', 'username')), 'webdav_password': self.conf.decrypt(self.conf.getOption('webDav', 'password')), # 'webdav_root': '/dav/', 'disable_check': True, } self.client = Client(self.options) def creatDir(self): try: self.client.mkdir("tomato_db") except Exception as e: self.confwebdav.error(e) pass def clean(self): try: self.client.clean("tomato_db/tomato.db") except Exception as e: self.confwebdav.error(e) pass def upload(self, file): try: self.loadBaseInfo() self.creatDir() self.clean() self.client.upload(remote_path="tomato_db/tomato.db", local_path=self.path + file) except Exception as e: self.confwebdav.error(e) pass def download(self, file): try: self.loadBaseInfo() self.client.download(remote_path="tomato_db/tomato.db", local_path=self.path + file) except Exception as e: self.confwebdav.error(e) pass
def fetch(options=options): client = Client(options) if not client.check("/server"): client.mkdir("/server") lst=client.list("/server")[1::] cnt=0 for i in lst: cnt+=1 print("Index:",cnt," Time:",parse_time(i)) index=int(input("Please select the file you want to restore:"))%len(lst) client.download("/server/"+lst[index-1],"./dat/") print(parse_time(lst[index-1]),"has been restored")
def fetch(options=options): client = Client(options) if not client.check("/blocks"): client.mkdir("/blocks") lst = client.list("/blocks")[1::] cnt = 0 for i in lst: cnt += 1 print("Index:", cnt, i) index = int( input("Please select the block you want to install:")) % len(lst) client.download("/blocks/" + lst[index - 1], "./blocks/" + lst[index - 1]) # print(parse_time(lst[index-1]),"has been restored") print(lst[index - 1], "has been installed successfully")
class Dav: """A class to handle webdav request for uploading the mail attachments """ def __init__(self, host, user, password): """initializes a webdav connection Args: host (string): host url user (string): dav user name password (string): dav password """ self.options = { 'webdav_hostname': host, 'webdav_login': user, 'webdav_password': password } self.client=Client(self.options) def uploadAll(self, dir): """Uploads all attachments to the cloud Args: dir (string): existing directory to save the files to Returns: False: if something went wrong """ files = [f for f in listdir("attachments") if isfile(join("attachments", f))] year = datetime.datetime.now().strftime("%Y") date = datetime.datetime.now().strftime("%m-%d") if not year in self.client.list(dir): try: self.client.mkdir(dir+"/"+year) except Exception as e: print(e) return False if not date in self.client.list(dir+"/"+year) and len(files) > 0: try: self.client.mkdir(dir+"/"+year+"/"+date) except Exception as e: print(e) return False for f in files: try: self.client.upload_sync(remote_path=dir+"/"+year+"/"+date+"/"+str(random.randint(10000,100000))+"_"+f, local_path="attachments/"+f) os.remove("attachments/"+f) except Exception as e: print(e)
def upload_webdav(): options = { 'webdav_hostname': parse_config('WEBDAV', 'WEBDAV_UPLOAD_URL'), 'webdav_login': parse_config('WEBDAV', 'WEBDAV_UPLOAD_USER'), 'webdav_password': parse_config('WEBDAV', 'WEBDAV_UPLOAD_PW') } remotePath = parse_config('WEBDAV', 'WEBDAV_UPLOAD_PATH') client = Client(options) client.verify = False today = datetime.date.today() year, week_num, day_of_week = today.isocalendar() path = str(year) + "_cw" + str(week_num) if not client.check(remotePath + "/" + path): client.mkdir(remotePath + "/" + path) kwargs = { 'remote_path': remotePath + "/" + path + "/", 'local_path': targetPath + "/" + path + "/" } client.upload_async(**kwargs)
def sync_to_webdav(url, username, password): client = Client({ "webdav_hostname": url, "webdav_login": username, "webdav_password": password, "webdav_root": ROOT_DIR, }) files = ( "/etc/certs/otp_configuration.json", "/etc/certs/super_gluu_creds.json", ) for file_ in files: try: logger.info(f"Sync {file_} to {url}{ROOT_DIR}{file_}") client.mkdir("/etc") client.mkdir("/etc/certs") client.upload_file(file_, file_) except (RemoteResourceNotFound, NoConnection, RemoteParentNotFound) as exc: logger.warning( f"Unable to sync {file_} to {url}{ROOT_DIR}{file_}; reason={exc}" )
class BatchProcessor(QObject): ''' Worker thread ''' signals = ObjectSignals() setLightPWM = pyqtSignal(float) # control signal gotoXY = pyqtSignal( float, float, bool) # boolean indicate relative to stage origin or not gotoX = pyqtSignal(float, bool) gotoY = pyqtSignal(float, bool) gotoZ = pyqtSignal(float) findDiaphragm = pyqtSignal() disableMotors = pyqtSignal() findWell = pyqtSignal() # computeSharpnessScore = pyqtSignal() rSnapshotTaken = pyqtSignal() # repeater signal # rSharpnessScore = pyqtSignal() # repeater signal rWellFound = pyqtSignal() # repeat signal rDiaphragmFound = pyqtSignal() # repeat signal rClipRecorded = pyqtSignal() # repeat signal rPositionReached = pyqtSignal() # repeat signal takeSnapshot = pyqtSignal(str) recordClip = pyqtSignal(str, int) setLogFileName = pyqtSignal(str) stopCamera = pyqtSignal() startCamera = pyqtSignal() startAutoFocus = pyqtSignal(float) focussed = pyqtSignal() # repeater signal def __init__(self): super().__init__() self.settings = QSettings("settings.ini", QSettings.IniFormat) self.loadSettings() self.timer = QTimer(self) self.timer.timeout.connect(self.run) self.isInterruptionRequested = False self.foundDiaphragmLocation = None self.foundWellLocation = None self.lightLevel = 1.0 # initial value, will be set later by user # self.sharpnessScore = 0 def loadSettings(self): self.msg("info;loading settings from {:s}".format( self.settings.fileName())) self.resolution = float( self.settings.value('camera/resolution_in_px_per_mm')) def loadBatchSettings(self): self.msg("info;loading batch settings from {:s}".format( self.batch_settings.fileName())) # load run info self.run_id = self.batch_settings.value('run/id') self.run_note = self.batch_settings.value('run/note') t = time.strptime( self.batch_settings.value('run/duration')[1], '%H:%M:%S') days = int(self.batch_settings.value('run/duration')[0].split('d')[0]) self.run_duration_s = ( (24 * days + t.tm_hour) * 60 + t.tm_min) * 60 + t.tm_sec t = time.strptime(self.batch_settings.value('run/wait'), '%H:%M:%S') self.run_wait_s = (t.tm_hour * 60 + t.tm_min) * 60 + t.tm_sec s = str(self.batch_settings.value('run/shutdown')) self.shutdown = s.lower() in ['true', '1', 't', 'y', 'yes'] s = str(self.batch_settings.value('run/snapshot')) self.snapshot = s.lower() in ['true', '1', 't', 'y', 'yes'] s = str(self.batch_settings.value('run/videoclip')) self.videoclip = s.lower() in ['true', '1', 't', 'y', 'yes'] self.videoclip_length = int( self.batch_settings.value('run/clip_length')) # load well-plate dimensions and compute well locations self.plate_note = self.batch_settings.value('plate/note') self.nr_of_columns = int( self.batch_settings.value('plate/nr_of_columns')) self.nr_of_rows = int(self.batch_settings.value('plate/nr_of_rows')) self.A1_to_side_offset = float( self.batch_settings.value('plate/A1_to_side_offset')) self.column_well_spacing = float( self.batch_settings.value('plate/column_well_spacing')) self.A1_to_top_offset = float( self.batch_settings.value('plate/A1_to_top_offset')) self.row_well_spacing = float( self.batch_settings.value('plate/row_well_spacing')) self.computeWellLocations() def computeWellLocations(self): # load the wells to process nr_of_wells = self.batch_settings.beginReadArray("wells") self.wells = [] for i in range(0, nr_of_wells): self.batch_settings.setArrayIndex(i) well_id = self.batch_settings.value('id') well_note = self.batch_settings.value('note') r = re.split('(\d+)', well_id) row = ord(r[0].lower()) - 96 col = int(r[1]) location_mm = [round(self.A1_to_side_offset + (col-1)*self.column_well_spacing, 2), \ round(self.A1_to_top_offset + (row-1)*self.row_well_spacing, 2), \ 0] self.wells.append( Well(name=well_id, position=[row, col], location=location_mm)) self.batch_settings.endArray( ) # close array, also required when opening! def msg(self, text): if text: text = self.__class__.__name__ + ";" + str(text) print(text) self.signals.message.emit(text) @pyqtSlot() def start(self): # open storage folder # dlg = QFileDialog() # self.storage_path = QFileDialog.getExistingDirectory(dlg, 'Open storage folder', '/media/pi/', QFileDialog.ShowDirsOnly | QFileDialog.DontResolveSymlinks) # if self.storage_path == "" or self.storage_path is None: # return self.storage_path = os.path.sep.join( [os.getcwd(), self.settings.value('temp_folder')]) # clear temporary storage path os.system('rm {:s}'.format(os.path.sep.join([self.storage_path, "*.*"]))) # open batch definition file dlg = QFileDialog() self.batch_file_name = QFileDialog.getOpenFileName( dlg, 'Open batch definition file', os.getcwd(), "Ini file (*.ini)")[0] if self.batch_file_name == "" or self.batch_file_name is None: return self.batch_settings = QSettings(self.batch_file_name, QSettings.IniFormat) self.loadBatchSettings() # copy batch definition file to storage folder os.system('cp {:s} {:s}'.format(self.batch_file_name, self.storage_path)) note_file_name = os.path.sep.join( [self.storage_path, self.run_id + ".log"]) self.setLogFileName.emit(note_file_name) # copy files to webdav host conn_settings = QSettings("connections.ini", QSettings.IniFormat) self.webdav_path = os.path.sep.join( [conn_settings.value('webdav/storage_path'), self.run_id]) self.webdav_options = { 'webdav_hostname': conn_settings.value('webdav/hostname'), 'webdav_login': conn_settings.value('webdav/login'), 'webdav_password': conn_settings.value('webdav/password') } try: self.webdav_client = Client(self.webdav_options) self.webdav_client.mkdir(self.webdav_path) self.webdav_client.push(remote_directory=self.webdav_path, local_directory=self.storage_path) except WebDavException as err: traceback.print_exc() self.signals.error.emit( (type(err), err.args, traceback.format_exc())) # create temporary image storage path self.image_storage_path = os.path.sep.join([self.storage_path, 'img']) if not os.path.exists(self.image_storage_path): os.makedirs(self.image_storage_path) # start-up recipe self.msg("info;plate note: {:s}".format(self.plate_note)) self.msg("info;run note: {:s}".format(self.run_note)) self.msg("info;{:d} wells found in {:s}".format( len(self.wells), self.batch_settings.fileName())) self.msg("info;{:s} run during {:d}s with {:d}s interleave".format( self.run_id, self.run_duration_s, self.run_wait_s)) self.setLightPWM.emit(0.2) self.startCamera.emit() self.msg("info;goto first well") x = self.wells[0].location[0] y = self.wells[0].location[1] z = 33 #self.wells[0].location[2] self.gotoXY.emit(x, y, True) self.wait_signal(self.rPositionReached, 10000) self.gotoZ.emit(z) self.wait_signal(self.rPositionReached, 10000) # Let the user set the x,y,z-location manually by moving to first well and open dialog dialog = ManualPositioningDialog(x, y, z) dialog.stageXTranslation.valueChanged.connect( lambda x: self.gotoX.emit(x, True)) dialog.stageYTranslation.valueChanged.connect( lambda y: self.gotoY.emit(y, True)) dialog.stageZTranslation.valueChanged.connect( lambda z: self.gotoZ.emit(z)) dialog.light.valueChanged.connect( lambda x: self.setLightPWM.emit(x / 100)) dialog.exec_() self.A1_to_side_offset = round(dialog.stageXTranslation.value(), 3) self.A1_to_top_offset = round(dialog.stageYTranslation.value(), 3) z = round(dialog.stageZTranslation.value(), 3) self.lightLevel = round(dialog.light.value() / 100, 3) self.msg("info;user set well A1 at (x,y)=({:.3f},{:.3f})".format( self.A1_to_side_offset, self.A1_to_top_offset)) self.msg("info;user set focus at z={:.3f}".format(z)) self.msg("info;user set light intensity at {:.3f}".format( self.lightLevel)) # recompute the well locations in our batch self.computeWellLocations() for well in self.wells: well.location[ 2] = z # copy z, later on we may do autofocus per well # start timer self.prev_note_nr = 0 # for logging self.start_time_s = time.time() self.timer.start(0) def run(self): ''' Timer call back function, als initiates next one-shot ''' start_run_time_s = time.time() self.setLightPWM.emit(self.lightLevel) self.startCamera.emit() self.msg("info;go home") self.gotoXY.emit(0, 0, False) self.wait_signal(self.rPositionReached, 10000) self.msg("info;goto first well") self.gotoXY.emit(self.wells[0].location[0], self.wells[0].location[1], True) self.wait_signal(self.rPositionReached, 10000) # self.wait_ms(60000) # wait for camera to adjust to light for well in self.wells: self.msg("info;gauging well {:s})".format(well.name)) # split goto xyz command self.gotoX.emit(well.location[0], True) self.wait_signal(self.rPositionReached, 10000) self.gotoY.emit(well.location[1], True) self.wait_signal(self.rPositionReached, 10000) self.gotoZ.emit(well.location[2]) self.wait_signal(self.rPositionReached, 10000) # autofocus if self.batch_settings.value('run/autofocus', False, type=bool): self.new_z = 0 self.startAutoFocus.emit(well.location[2]) self.wait_signal(self.focussed, 100000) if self.new_z != 0: well.location[2] = self.new_z else: # focus failed, so return to initial z position self.gotoZ.emit(well.location[2]) self.wait_signal(self.rPositionReached, 10000) try: # clear temporary storage path os.system('rm {:s}'.format( os.path.sep.join([self.image_storage_path, "*.*"]))) except err: traceback.print_exc() self.signals.error.emit( (type(err), err.args, traceback.format_exc())) # take snapshot or video self.wait_ms(2000) prefix = os.path.sep.join([ self.image_storage_path, str(well.position) + "_" + str(well.location) ]) if self.snapshot: self.takeSnapshot.emit(prefix) self.wait_signal(self.rSnapshotTaken) # snapshot taken if self.videoclip and self.videoclip_length > 0: self.recordClip.emit(prefix, self.videoclip_length) self.wait_signal(self.rClipRecorded) # clip recorded try: # push data remote_path = os.path.sep.join([self.webdav_path, well.name]) self.msg(": info; pushing data to {}".format(remote_path)) self.webdav_client.mkdir(remote_path) self.webdav_client.push( remote_directory=remote_path, local_directory=self.image_storage_path) # push log file self.webdav_client.push(remote_directory=self.webdav_path, local_directory=self.storage_path) except WebDavException as err: traceback.print_exc() self.signals.error.emit( (type(err), err.args, traceback.format_exc())) # Wrapup current round of acquisition self.setLightPWM.emit(0.00) self.stopCamera.emit() elapsed_total_time_s = time.time() - self.start_time_s elapsed_run_time_s = time.time() - start_run_time_s self.msg("info;single run time={:.1f}s".format(elapsed_run_time_s)) self.msg("info;total run time={:.1f}s".format(elapsed_total_time_s)) progress_percentage = int(100 * elapsed_total_time_s / self.run_duration_s) self.signals.progress.emit(progress_percentage) self.msg("info;progress={:d}%".format(progress_percentage)) # send a notification note_nr = int(progress_percentage / 10) if note_nr != self.prev_note_nr: self.prev_note_nr = note_nr message = """Subject: Progress = {}% \n\n Still {} s left""".format( progress_percentage, int(self.run_duration_s - elapsed_total_time_s)) # do something fancy here in future: https://realpython.com/python-send-email/#sending-fancy-emails self.sendNotification(message) # check if we still have time to do another round if elapsed_total_time_s + self.run_wait_s < self.run_duration_s: self.timer.setInterval(self.run_wait_s * 1000) self.msg("info;wait for {:.1f} s".format(self.run_wait_s)) else: self.timer.stop() self.signals.ready.emit() self.msg("info;run finalized") message = """Subject: run finalized""" # do something fancy here in future: https://realpython.com/python-send-email/#sending-fancy-emails self.sendNotification(message) if self.shutdown: self.msg("info;emitting finished") self.signals.finished.emit() def sendNotification(self, message): conn_settings = QSettings("connections.ini", QSettings.IniFormat) port = 465 # For SSL context = ssl.create_default_context() # Create a secure SSL context try: with smtplib.SMTP_SSL("smtp.gmail.com", port, context=context) as server: login = conn_settings.value('smtp/login') password = conn_settings.value('smtp/password') server.login(login, password) server.sendmail(conn_settings.value('smtp/login'), \ conn_settings.value('subscriber/email'), \ message) except Exception as err: traceback.print_exc() self.signals.error.emit( (type(err), err.args, traceback.format_exc())) def requestInterruption(self): self.isInterruptionRequested = True @pyqtSlot(np.ndarray) def diaphragmFound(self, location): self.foundDiaphragmLocation = location self.msg("info;diaphragmFound signal received") self.rDiaphragmFound.emit() @pyqtSlot(np.ndarray) def wellFound(self, location): self.foundWellLocation = location self.msg("info;wellFound signal received") self.rWellFound.emit() # @pyqtSlot(float) # def setSharpnessScore(self, score): # self.sharpnessScore = score # self.msg("info;sharpnessScore signal received") # self.rSharpnessScore.emit() @pyqtSlot() def snapshotTaken(self): self.msg("info;snapshotTaken signal received") self.rSnapshotTaken.emit() @pyqtSlot() def clipRecorded(self): self.msg("info;clipRecorded signal received") self.rClipRecorded.emit() @pyqtSlot() def positionReached(self): self.msg("info;positionReached signal received") self.rPositionReached.emit() @pyqtSlot(float) def focussedSlot(self, val): self.new_z = val self.focussed.emit() @pyqtSlot() def stop(self): self.msg("info;stopping") self.requestInterruption() if self.timer.isActive(): self.timer.stop() self.signals.finished.emit() def wait_ms(self, timeout): ''' Block loop until timeout (ms) elapses. ''' loop = QEventLoop() QTimer.singleShot(timeout, loop.exit) loop.exec_() def wait_signal(self, signal, timeout=100): ''' Block loop until signal emitted, or timeout (ms) elapses. ''' loop = QEventLoop() signal.connect(loop.quit) # only quit is a slot of QEventLoop QTimer.singleShot(timeout, loop.exit) loop.exec_()
class ClientTestCase(TestCase): remote_path_file = 'test_dir/test.txt' remote_path_file2 = 'test_dir2/test.txt' remote_path_dir = 'test_dir' remote_path_dir2 = 'test_dir2' local_base_dir = 'tests/' local_file = 'test.txt' local_file_path = local_base_dir + 'test.txt' local_path_dir = local_base_dir + 'res/test_dir' def setUp(self): options = { 'webdav_hostname': 'https://webdav.yandex.ru', 'webdav_login': '******', 'webdav_password': '******' } self.client = Client(options) if path.exists(path=self.local_path_dir): shutil.rmtree(path=self.local_path_dir) def tearDown(self): if path.exists(path=self.local_path_dir): shutil.rmtree(path=self.local_path_dir) if self.client.check(remote_path=self.remote_path_dir): self.client.clean(remote_path=self.remote_path_dir) if self.client.check(remote_path=self.remote_path_dir2): self.client.clean(remote_path=self.remote_path_dir2) def test_list(self): self._prepare_for_downloading() file_list = self.client.list() self.assertIsNotNone(file_list, 'List of files should not be None') self.assertGreater(file_list.__len__(), 0, 'Expected that amount of files more then 0') def test_free(self): self.assertGreater( self.client.free(), 0, 'Expected that free space on WebDAV server is more then 0 bytes') def test_check(self): self.assertTrue(self.client.check(), 'Expected that root directory is exist') def test_mkdir(self): if self.client.check(remote_path=self.remote_path_dir): self.client.clean(remote_path=self.remote_path_dir) self.client.mkdir(remote_path=self.remote_path_dir) self.assertTrue(self.client.check(remote_path=self.remote_path_dir), 'Expected the directory is created.') @unittest.skip( "Yandex brakes response for file it contains property resourcetype as collection but it should " "be empty for file") def test_download_to(self): self._prepare_for_downloading() buff = BytesIO() self.client.download_from(buff=buff, remote_path=self.remote_path_file) self.assertEquals(buff.getvalue(), 'test content for testing of webdav client') @unittest.skip( "Yandex brakes response for file it contains property resourcetype as collection but it should " "be empty for file") def test_download(self): self._prepare_for_downloading() self.client.download(local_path=self.local_path_dir, remote_path=self.remote_path_dir) self.assertTrue(path.exists(self.local_path_dir), 'Expected the directory is downloaded.') self.assertTrue(path.isdir(self.local_path_dir), 'Expected this is a directory.') self.assertTrue( path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file is downloaded') self.assertTrue( path.isfile(self.local_path_dir + os.path.sep + self.local_path_file), 'Expected this is a file') @unittest.skip( "Yandex brakes response for file it contains property resourcetype as collection but it should " "be empty for file") def test_download_sync(self): self._prepare_for_downloading() os.mkdir(self.local_path_dir) def callback(): self.assertTrue( path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file is downloaded') self.assertTrue( path.isfile(self.local_path_dir + os.path.sep + self.local_file), 'Expected this is a file') self.client.download_sync(local_path=self.local_path_dir + os.path.sep + self.local_file, remote_path=self.remote_path_file, callback=callback) self.assertTrue( path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file has already been downloaded') @unittest.skip( "Yandex brakes response for file it contains property resourcetype as collection but it should " "be empty for file") def test_download_async(self): self._prepare_for_downloading() os.mkdir(self.local_path_dir) def callback(): self.assertTrue( path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file is downloaded') self.assertTrue( path.isfile(self.local_path_dir + os.path.sep + self.local_file), 'Expected this is a file') self.client.download_async(local_path=self.local_path_dir + os.path.sep + self.local_file, remote_path=self.remote_path_file, callback=callback) self.assertFalse( path.exists(self.local_path_dir + os.path.sep + self.local_file), 'Expected the file has not been downloaded yet') def test_upload_from(self): self._prepare_for_uploading() buff = StringIO(u'test content for testing of webdav client') self.client.upload_to(buff=buff, remote_path=self.remote_path_file) self.assertTrue(self.client.check(self.remote_path_file), 'Expected the file is uploaded.') def test_upload(self): self._prepare_for_uploading() self.client.upload(remote_path=self.remote_path_file, local_path=self.local_path_dir) self.assertTrue(self.client.check(self.remote_path_dir), 'Expected the directory is created.') self.assertTrue(self.client.check(self.remote_path_file), 'Expected the file is uploaded.') def test_upload_file(self): self._prepare_for_uploading() self.client.upload_file(remote_path=self.remote_path_file, local_path=self.local_file_path) self.assertTrue(self.client.check(remote_path=self.remote_path_file), 'Expected the file is uploaded.') def test_upload_sync(self): self._prepare_for_uploading() def callback(): self.assertTrue(self.client.check(self.remote_path_dir), 'Expected the directory is created.') self.assertTrue(self.client.check(self.remote_path_file), 'Expected the file is uploaded.') self.client.upload(remote_path=self.remote_path_file, local_path=self.local_path_dir) def test_upload_async(self): self._prepare_for_uploading() def callback(): self.assertTrue(self.client.check(self.remote_path_dir), 'Expected the directory is created.') self.assertTrue(self.client.check(self.remote_path_file), 'Expected the file is uploaded.') self.client.upload(remote_path=self.remote_path_file, local_path=self.local_path_dir) def test_copy(self): self._prepare_for_downloading() self.client.mkdir(remote_path=self.remote_path_dir2) self.client.copy(remote_path_from=self.remote_path_file, remote_path_to=self.remote_path_file2) self.assertTrue(self.client.check(remote_path=self.remote_path_file2)) def test_move(self): self._prepare_for_downloading() self.client.mkdir(remote_path=self.remote_path_dir2) self.client.move(remote_path_from=self.remote_path_file, remote_path_to=self.remote_path_file2) self.assertFalse(self.client.check(remote_path=self.remote_path_file)) self.assertTrue(self.client.check(remote_path=self.remote_path_file2)) def test_clean(self): self._prepare_for_downloading() self.client.clean(remote_path=self.remote_path_dir) self.assertFalse(self.client.check(remote_path=self.remote_path_file)) self.assertFalse(self.client.check(remote_path=self.remote_path_dir)) def test_info(self): self._prepare_for_downloading() result = self.client.info(remote_path=self.remote_path_file) self.assertEquals(result['name'], 'test.txt') self.assertEquals(result['size'], '41') self.assertTrue('created' in result) self.assertTrue('modified' in result) def test_directory_is_dir(self): self._prepare_for_downloading() self.assertTrue(self.client.is_dir(self.remote_path_dir), 'Should return True for directory') def test_file_is_not_dir(self): self._prepare_for_downloading() self.assertFalse(self.client.is_dir(self.remote_path_file), 'Should return False for file') def test_get_property_of_non_exist(self): self._prepare_for_downloading() result = self.client.get_property(remote_path=self.remote_path_file, option={'name': 'aProperty'}) self.assertEquals( result, None, 'For not found property should return value as None') def test_set_property(self): self._prepare_for_downloading() self.client.set_property(remote_path=self.remote_path_file, option={ 'namespace': 'test', 'name': 'aProperty', 'value': 'aValue' }) result = self.client.get_property(remote_path=self.remote_path_file, option={ 'namespace': 'test', 'name': 'aProperty' }) self.assertEquals(result, 'aValue', 'Property value should be set') def test_set_property_batch(self): self._prepare_for_downloading() self.client.set_property_batch(remote_path=self.remote_path_file, option=[{ 'namespace': 'test', 'name': 'aProperty', 'value': 'aValue' }, { 'namespace': 'test', 'name': 'aProperty2', 'value': 'aValue2' }]) result = self.client.get_property(remote_path=self.remote_path_file, option={ 'namespace': 'test', 'name': 'aProperty' }) self.assertEquals(result, 'aValue', 'First property value should be set') result = self.client.get_property(remote_path=self.remote_path_file, option={ 'namespace': 'test', 'name': 'aProperty2' }) self.assertEquals(result, 'aValue2', 'Second property value should be set') def _prepare_for_downloading(self): if not self.client.check(remote_path=self.remote_path_dir): self.client.mkdir(remote_path=self.remote_path_dir) if not self.client.check(remote_path=self.remote_path_file): self.client.upload_file(remote_path=self.remote_path_file, local_path=self.local_file_path) if not path.exists(self.local_path_dir): os.makedirs(self.local_path_dir) def _prepare_for_uploading(self): if not self.client.check(remote_path=self.remote_path_dir): self.client.mkdir(remote_path=self.remote_path_dir) if not path.exists(path=self.local_path_dir): os.makedirs(self.local_path_dir) if not path.exists(path=self.local_path_dir + os.sep + self.local_file): shutil.copy(src=self.local_file_path, dst=self.local_path_dir + os.sep + self.local_file)
class WebDev(object): def __init__(self, options ): # self.client = wc.Client(options) self.client = Client(options) def read(self, name): with BytesIO() as buf: self.client.download_from(buf, name) return buf.getvalue() def write(self,content, target): data = self.read(target) with BytesIO() as buf: buf.write(data) buf.write(content.encode()) self.client.upload_to(buf.getvalue(), target) def mkdir(self, dir_name): self.client.mkdir(dir_name) def list(self): data = self.client.list() print(data) def clear(self, target): self.client.clean(target) def check(self,remote_path): data = self.client.check(remote_path) if data: print('{} is exists'.format(remote_path)) else: print('{} is not exists'.format(remote_path)) def upload(self,remote_path, local_path): self.client.upload(remote_path, local_path) def help_dev(self): print(dir(self.client)) def mkfile(self, remote_path): with BytesIO() as buf: buf.write(b'') self.client.upload_to(buf.getvalue(), remote_path) def download_file(self, remote_path): self.client.download_file(remote_path, remote_path) def rename(self, old_name, new_name): self.client.copy(old_name, old_name) self.client.clean(old_name) def set_property(self,remote_path, option): self.client.set_property(remote_path, option=[option])
class FileBrowserWebdavStorage(StorageMixin, Storage): def __init__(self, base_url='/', url_as_download=True, simple_listdir=False, webdav_root='/'): self.base_url = base_url self.url_as_download = url_as_download self.simple_listdir = simple_listdir webdav_client_options = { 'webdav_hostname': settings.CONTRAX_FILE_STORAGE_WEBDAV_ROOT_URL.rstrip('/'), 'webdav_login': settings.CONTRAX_FILE_STORAGE_WEBDAV_USERNAME, 'webdav_password': settings.CONTRAX_FILE_STORAGE_WEBDAV_PASSWORD, } self.client = Client(webdav_client_options) try: self.client.mkdir('/media') self.client.mkdir('/media/photo') except: pass self.client.webdav.root = webdav_root self.client.root = webdav_root def path(self, name): """ Return a local filesystem path where the file can be retrieved using Python's built-in open() function. Storage systems that can't be accessed using open() should *not* implement this method. """ # FIXME: this would be useful with self.location != '' # in this case use this notation: # 1. define self.location in __init__ # 2. rewrite path() method to be like # return os.oath.join(self.location, name) # 3. everywhere in other sel.methods use self.path(name) instead of name attr return name def isdir(self, path): """ Returns true if name exists and is a directory. """ return self.client.check(path) and self.client.is_dir(path) def isfile(self, path): """ Returns true if name exists and is a regular file. """ return self.client.check(path) and not self.client.is_dir(path) def move(self, old_file_name, new_file_name, allow_overwrite=False): """ Moves safely a file from one location to another. If allow_ovewrite==False and new_file_name exists, raises an exception. """ return self.client.move(remote_path_from=old_file_name, remote_path_to=new_file_name, overwrite=allow_overwrite) def makedirs(self, path): """ Creates all missing directories specified by name. Analogue to os.mkdirs(). """ return self.client.mkdir(path) def rmtree(self, path): """ Deletes a directory and everything it contains. Analogue to shutil.rmtree(). """ return self.client.clean(path) def setpermission(self, path): """ Sets file permission """ pass def _open(self, path, mode='rb'): tmp = io.BytesIO() self.client.download_from(tmp, path) tmp.seek(0) return File(tmp) def _save(self, path, content): res = self.client.resource(path) res.read_from(content) return path def get_valid_name(self, name): """ Return a filename, based on the provided filename, that's suitable for use in the target storage system. """ return get_valid_filename(name) def delete(self, path): """ Delete the specified file from the storage system. """ if self.exists(path): self.client.clean(path) def exists(self, path): """ Return True if a file referenced by the given name already exists in the storage system, or False if the name is available for a new file. """ return self.client.check(path) def listdir(self, path): """ List the contents of the specified path. Return a 2-tuple of lists: the first item being directories, the second item being files. """ _list = self.client.list(path) # for API: iterating over big directory take too much time if self.simple_listdir: return _list # for filebrowser directories, files = [], [] for entry in _list: entry_path = os.path.join(path, entry) if self.isdir(entry_path): directories.append(entry.rstrip('/')) else: files.append(entry) return directories, files def size(self, path): """ Return the total size, in bytes, of the file specified by name. """ return self.client.info(path)['size'] def url(self, path): """ Return an absolute URL where the file's contents can be accessed directly by a Web browser. """ url = filepath_to_uri(path) if url is not None: url = url.lstrip('/') url = urljoin(self.base_url, url) if self.url_as_download and self.isfile(path): url += '?action=download' return url @staticmethod def _datetime_from_timestamp(ts, fmt): """ If timezone support is enabled, make an aware datetime object in UTC; otherwise make a naive one in the local timezone. """ dt = datetime.strptime(ts, fmt) if settings.USE_TZ: # Safe to use .replace() because UTC doesn't have DST return dt.replace(tzinfo=timezone.utc) else: return dt def get_accessed_time(self, path): """ Return the last accessed time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ pass def get_created_time(self, path): """ Return the creation time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ return self._datetime_from_timestamp(self.client.info(path)['created'], fmt='%Y-%m-%dT%H:%M:%SZ') def get_modified_time(self, path): """ Return the last modified time (as a datetime) of the file specified by name. The datetime will be timezone-aware if USE_TZ=True. """ return self._datetime_from_timestamp( self.client.info(path)['modified'], fmt='%a, %d %b %Y %H:%M:%S %Z')
def upload_dav(): try: options = { 'webdav_hostname': "https://192.168.1.6/remote.php/dav/files/clouduser/", 'webdav_login': "******", 'webdav_password': "******", 'verbose':True } conn=False client = Client(options) client.verify = False try: if(client.list()): print('\n' * 4) print("Connection Successfull") conn=True print('\n' * 3) except: print('\n' * 4) print("Connection error. Check credentials") if conn: print("##############################################################") opt=int(input("Choose action to be performed:\n1:Show all files\n2:Make Directory\n3:Delete\n4:Download file\n5:Upload File\n")) if opt==1: files=client.list() print(files) elif opt==2: d=input("Enter directory name\n") if client.mkdir(d): print("Created Successfully") elif opt==3: d=input("Enter directory or file name\n") client.clean(d) print("Deleted") elif opt==4: file=input("Enter file to download with public key and signature\n") path_1=file+'.encrypted' path_2=file+'.sign' path_3='public_key.pem' s_time=time.time() client.download_sync(path_1, path_1) client.download_sync(path_2, path_2) client.download_sync(path_3, path_3) e_time=time.time() t=e_time-s_time print("Downloaded, Time taken is",e_time-s_time) elif opt==5: file=input("Enter file to upload with public key and signature\n") path_1=file+'.encrypted' path_2=file+'.sign' path_3='public_key.pem' s_time=time.time() client.upload_sync(path_1, path_1) client.upload_sync(path_2, path_2) client.upload_sync(path_3, path_3) e_time=time.time() print("Uplaoded, Time taken is",e_time-s_time) else: None except WebDavException as exception: print("\n\n",exception,"\n\n")
class BaseClientTestCase(unittest.TestCase): remote_path_file = 'test_dir/test.txt' remote_compressed_path_file = 'test_dir/compressed.txt' remote_path_file2 = 'test_dir2/test.txt' remote_inner_path_file = 'test_dir/inner/test.txt' remote_path_dir = 'test_dir' remote_path_dir2 = 'test_dir2' remote_inner_path_dir = 'test_dir/inner' inner_dir_name = 'inner' local_base_dir = 'tests/' local_file = 'test.txt' local_file_path = local_base_dir + 'test.txt' local_compressed_file_path = local_base_dir + 'compressed.txt' local_path_dir = local_base_dir + 'res/test_dir' options = { 'webdav_hostname': 'http://localhost:8585', 'webdav_login': '******', 'webdav_password': '******', 'webdav_timeout': 10, 'webdav_override_methods': { 'check': 'GET' } } # options = { # 'webdav_hostname': 'https://demo1.nextcloud.com/remote.php/dav/files/RCw8Y9XXFnzkLJbN/', # 'webdav_login': '******', # 'webdav_password': '******', # 'webdav_override_methods': { # 'check': 'GET' # } # } # options = { # 'webdav_hostname': 'https://webdav.yandex.ru', # 'webdav_login': '******', # 'webdav_password': '******' # } def setUp(self): self.client = Client(self.options) self.clean_local_dir(self.local_path_dir) def tearDown(self): self.clean_local_dir(self.local_path_dir) self.clean_remote_dir(self.remote_path_dir) self.clean_remote_dir(self.remote_path_dir2) def clean_remote_dir(self, remote_path_dir): if self.client.check(remote_path=remote_path_dir): self.client.clean(remote_path=remote_path_dir) @staticmethod def clean_local_dir(local_path_dir): if path.exists(path=local_path_dir): shutil.rmtree(path=local_path_dir) def _prepare_for_downloading(self, inner_dir=False, base_path=''): if base_path: self._create_remote_dir_if_needed(base_path) self._prepare_dir_for_downloading(base_path + self.remote_path_dir, base_path + self.remote_path_file, self.local_file_path) if not path.exists(self.local_path_dir): os.makedirs(self.local_path_dir) if inner_dir: self._prepare_dir_for_downloading( base_path + self.remote_inner_path_dir, base_path + self.remote_inner_path_file, self.local_file_path) def _prepare_dir_for_downloading(self, remote_path_dir, remote_path_file, local_file_path): self._create_remote_dir_if_needed(remote_path_dir) if not self.client.check(remote_path=remote_path_file): self.client.upload_file(remote_path=remote_path_file, local_path=local_file_path) def _create_remote_dir_if_needed(self, remote_dir): if not self.client.check(remote_path=remote_dir): self.client.mkdir(remote_path=remote_dir) def _prepare_for_uploading(self): if not self.client.check(remote_path=self.remote_path_dir): self.client.mkdir(remote_path=self.remote_path_dir) if not path.exists(path=self.local_path_dir): os.makedirs(self.local_path_dir) if not path.exists(path=self.local_path_dir + os.sep + self.local_file): shutil.copy(src=self.local_file_path, dst=self.local_path_dir + os.sep + self.local_file)
class WebDAV: """Commerce Cloud WebDAV session. Args: client (CommerceCloudClientSession): Active client session with Commerce Cloud for a bearer token. instance (str, optional): Optional commerce cloud instance, useful for opening clients to multiple instances using the same bearer token. Defaults to None. cert (str, optional): Path to TLS client certificate. Defaults to None. key ([type], optional): Export key for the TLS certificate. Defaults to None. verify (bool, optional): Verify TLS certificates, set to false for self signed. Defaults to True. """ def __init__(self, client, instance=None, cert=None, key=None, verify=True): self.client = client self._instance = instance or self.client.instance self.options = {"webdav_hostname": self._instance.rstrip("/")} self.verify = verify self.token = self.client.Token self.options.update({"webdav_token": self.token["access_token"]}) self.webdav_client = Client(self.options) self.webdav_client.verify = self.verify if cert and key: self.cert = str(Path(cert).resolve()) self.key = str(Path(key).resolve()) self.options.update({"cert_path": self.cert, "key_path": self.key}) def reauth(self): """Checks token expiry and re-initialises the Client if a new token is needed. """ if self.token["expires_at"] < int(time.time()): self.client.getToken() self.options.update({"webdav_token": self.token["access_token"]}) self.webdav_client = Client(self.options) def reconnect(self): """Re-initalise the Client session. """ self.webdav_client = Client(self.options) @property def hostname(self): """Return the hostname the WebDAV client connection is connected to. Returns: str: Hostname including prefix eg https:// """ return self.options["webdav_hostname"] @property def netloc(self): """Return a urlparse netloc string of the connected hostname. Returns: str: netloc of hostname. """ url = urlparse(self.options["webdav_hostname"]) return url.netloc @retry( retry_on_exceptions=(RetryException), max_calls_total=3, retry_window_after_first_call_in_seconds=10, ) def GetInfo(self, remote_filepath: str, headers: dict = None) -> list: """Get properties for entity [extended_summary] Args: remote_filepath (str): Path to remote resource. headers (dict, optional): Additional headers to apply to request. Defaults to None. Raises: RetryException: Adds to retries counter on failure. Returns: list: WebDAV attribute information. """ try: return self.webdav_client.info(remote_filepath) except (NoConnection, ConnectionException, WebDavException): self.reauth() raise RetryException @retry( retry_on_exceptions=(RetryException), max_calls_total=3, retry_window_after_first_call_in_seconds=10, ) def GetDirectoryList(self, filepath: str, get_info: bool = False, headers: dict = None) -> list: """Get list of files and folders in a path from WebDAV endpoint. [extended_summary] Args: filepath (str): Path to get directory listing for. get_info (bool): returns dictionary of attributes instead of file list. headers (dict, optional): Additional headers to apply to request. Defaults to None. Returns: list: Directory listing. """ try: return self.webdav_client.list(filepath, get_info=get_info) except (NoConnection, ConnectionException, WebDavException): self.reauth() raise RetryException @retry( retry_on_exceptions=(RetryException), max_calls_total=3, retry_window_after_first_call_in_seconds=10, ) def Upload(self, local_filepath: str, remote_filepath: str): """Upload file or directory recursively to WebDAV endpoint. [extended_summary] Args: local_filepath (str): Local path to file or directory to upload. remote_filepath (str): Remote path to upload to. """ local_filepath = str(Path(local_filepath).resolve()) try: self.webdav_client.upload_sync(remote_filepath, local_filepath) except (NoConnection, ConnectionException, WebDavException): self.reauth() raise RetryException @retry( retry_on_exceptions=(RetryException), max_calls_total=3, retry_window_after_first_call_in_seconds=10, ) def StreamUpload(self, payload, remote_path: str, file_name: str): """Upload FileIO, StringIO, BytesIO or string to WebDAV [extended_summary] Args: payload: Stream payload remote_path (str): Remote path relative to host. file_name (str): Name for the file uploaded. """ try: self.webdav_client.upload_to(payload, f"{remote_path}/{file_name}") except (NoConnection, ConnectionException, WebDavException): self.reauth() raise RetryException @retry( retry_on_exceptions=(RetryException), max_calls_total=3, retry_window_after_first_call_in_seconds=10, ) def MakeDir(self, remote_path: str): """Make new directory at path specified. Args: remote_path (str): Path of proposed new directory. """ try: self.webdav_client.mkdir(remote_path) except (NoConnection, ConnectionException, WebDavException): self.reauth() raise RetryException @retry( retry_on_exceptions=(RetryException), max_calls_total=3, retry_window_after_first_call_in_seconds=10, ) def Move(self, remote_path_source: str, remote_path_dest: str, overwrite: bool = False): """Make new directory at path specified. Args: remote_path_source (str): Path of source resource. remote_path_dest (str): Path of destination resource. overwrite (bool): Overwrite destination resource. Defaults to False. """ try: self.webdav_client.move(remote_path_source, remote_path_dest, overwrite) except (NoConnection, ConnectionException, WebDavException): self.reauth() raise RetryException @retry( retry_on_exceptions=(RetryException), max_calls_total=3, retry_window_after_first_call_in_seconds=10, ) def Delete(self, remote_filepath: str): """Delete file on remote WebDAV endpoint. Args: remote_filepath (str): Location of resource to delete. """ try: self.webdav_client.clean(remote_filepath) except (NoConnection, ConnectionException, WebDavException): self.reauth() raise RetryException @retry( retry_on_exceptions=(RetryException), max_calls_total=3, retry_window_after_first_call_in_seconds=10, ) def Download(self, local_filepath: str, remote_filepath: str): """Download file/folder from WebDAV endpoint. This is a synchronous operation, and the file is downloaded in full to the local_filepath. Args: local_filepath (str): Local path to download to, including filename of file saved. remote_filepath (str): Remote path to file to download. """ local_filepath = str(Path(local_filepath).resolve()) try: self.webdav_client.download_sync(remote_filepath, local_filepath) except (NoConnection, ConnectionException, WebDavException): self.reauth() raise RetryException @retry( retry_on_exceptions=(RetryException), max_calls_total=3, retry_window_after_first_call_in_seconds=10, ) def Pull(self, local_filepath: str, remote_filepath: str): """Sync file/folder from WebDAV endpoint to local storage. This downloads missing or nwer modified files from the remote to local storage. You can use it to do "resumeable" transfers, but the checks are slow for deeply nested files. Args: local_filepath (str): Local path to download to, including filename of file saved. remote_filepath (str): Remote path to file to download. """ local_filepath = str(Path(local_filepath).resolve()) try: self.webdav_client.pull(remote_filepath, local_filepath) return True except (NoConnection, ConnectionException, WebDavException): self.reauth() raise RetryException return False @retry( retry_on_exceptions=(RetryException), max_calls_total=3, retry_window_after_first_call_in_seconds=10, ) def Push(self, local_filepath: str, remote_filepath: str): """Sync file/folder from local storage to WebDAV endpoint. This uploads missing or nwer modified files from the local to remote storage. You can use it to do "resumeable" transfers, but the checks are slow for deeply nested files. Args: local_filepath (str): Local path to download to, including filename of file saved. remote_filepath (str): Remote path to file to download. """ local_filepath = str(Path(local_filepath).resolve()) try: self.webdav_client.push(local_filepath, remote_filepath) return True except (NoConnection, ConnectionException, WebDavException): self.reauth() raise RetryException return False @retry( retry_on_exceptions=(RetryException), max_calls_total=10, retry_window_after_first_call_in_seconds=15, ) def StreamDownload(self, remote_filepath: str, buffer=None, decode: bool = False): """Download a file in chunks to a local file buffer. You must provide a BytesIO object or one will be created for you. Args: remote_filepath (str): Path to remote resource to download. buffer ([type], optional): Buffer write streamed content to. decode (bool, optional): Optionally try to decode downloaded file into a string. Defaults to False. Raises: RetryException: Adds to retries counter on failure. Returns: Bytes: Returns a BytesIO object for further use. """ self.reauth() if buffer is None: buffer = BytesIO() try: self.webdav_client.download_from(buff=buffer, remote_path=remote_filepath) if decode is True: return buffer.getvalue().decode("utf-8") else: buffer.seek(0) return buffer except (NoConnection, ConnectionException, WebDavException): raise RetryException @retry( retry_on_exceptions=(RetryException), max_calls_total=10, retry_window_after_first_call_in_seconds=60, ) def HashObject(self, remote_filepath: str) -> str: """Generate a MD5 hashsum for a remote resource. This is streamed into memory, hashed and discarded. Optimised for low memory but high bandwidth environments. Args: remote_filepath (str): Path to remote resource. Raises: RetryException: Adds to retries counter on failure. Returns: str: MDSSUM of the file requested. """ self.reauth() try: sum = md5(self.StreamDownload(remote_filepath).getbuffer()) return { "filepath": remote_filepath, "hashtype": "MD5", "hashsum": sum.hexdigest(), } except (NoConnection, ConnectionException, WebDavException): self.reconnect() raise RetryException def RecursiveFileListing(self, remote_filepath: str) -> str: """Recursive filetree walker, returns paths found. Args: remote_filepath (str): [description] Raises: RetryException: Adds to retries counter on failure. Yields: Iterator[str]: Yields resource paths for any files found. """ @retry( retry_on_exceptions=(RetryException), max_calls_total=10, retry_window_after_first_call_in_seconds=60, ) def get_list(self, path): self.reauth() try: return self.webdav_client.list(path, get_info=True) except (NoConnection, ConnectionException, WebDavException): self.reconnect() raise RetryException def get_files(self, path): return [x for x in get_list(self, path) if x["isdir"] is False] def get_dirs(self, path): return [ x["path"] for x in get_list(self, path) if x["isdir"] is True ] yield from get_files(self, remote_filepath) for subdir in get_dirs(self, remote_filepath): yield from self.RecursiveFileListing(subdir) def RecursiveFolderListing(self, remote_filepath: str) -> str: """Recursive filetree walker, returns paths found. Args: remote_filepath (str): [description] Raises: RetryException: Adds to retries counter on failure. Yields: Iterator[str]: Yields resource paths for any files found. """ @retry( retry_on_exceptions=(RetryException), max_calls_total=10, retry_window_after_first_call_in_seconds=60, ) def get_list(self, path): self.reauth() try: return self.webdav_client.list(path, get_info=True) except (NoConnection, ConnectionException, WebDavException): self.reconnect() raise RetryException def get_dirs(self, path): return [ x["path"] for x in get_list(self, path) if x["isdir"] is True ] dirlist = get_dirs(self, remote_filepath) yield from dirlist for subdir in get_dirs(self, remote_filepath): yield from self.RecursiveFolderListing(subdir)
class BaseClientTestCase(unittest.TestCase): remote_path_file = 'test_dir/test.txt' remote_path_file2 = 'test_dir2/test.txt' remote_inner_path_file = 'test_dir/inner/test.txt' remote_path_dir = 'test_dir' remote_path_dir2 = 'test_dir2' remote_inner_path_dir = 'test_dir/inner' local_base_dir = 'tests/' local_file = 'test.txt' local_file_path = local_base_dir + 'test.txt' local_path_dir = local_base_dir + 'res/test_dir' options = { 'webdav_hostname': 'http://localhost:8585', 'webdav_login': '******', 'webdav_password': '******' } # options = { # 'webdav_hostname': 'https://webdav.yandex.ru', # 'webdav_login': '******', # 'webdav_password': '******' # } def setUp(self): self.client = Client(self.options) if path.exists(path=self.local_path_dir): shutil.rmtree(path=self.local_path_dir) def tearDown(self): if path.exists(path=self.local_path_dir): shutil.rmtree(path=self.local_path_dir) if self.client.check(remote_path=self.remote_path_dir): self.client.clean(remote_path=self.remote_path_dir) if self.client.check(remote_path=self.remote_path_dir2): self.client.clean(remote_path=self.remote_path_dir2) def _prepare_for_downloading(self, inner_dir=False): if not self.client.check(remote_path=self.remote_path_dir): self.client.mkdir(remote_path=self.remote_path_dir) if not self.client.check(remote_path=self.remote_path_file): self.client.upload_file(remote_path=self.remote_path_file, local_path=self.local_file_path) if not path.exists(self.local_path_dir): os.makedirs(self.local_path_dir) if inner_dir: if not self.client.check(remote_path=self.remote_inner_path_dir): self.client.mkdir(remote_path=self.remote_inner_path_dir) if not self.client.check(remote_path=self.remote_inner_path_file): self.client.upload_file( remote_path=self.remote_inner_path_file, local_path=self.local_file_path) def _prepare_for_uploading(self): if not self.client.check(remote_path=self.remote_path_dir): self.client.mkdir(remote_path=self.remote_path_dir) if not path.exists(path=self.local_path_dir): os.makedirs(self.local_path_dir) if not path.exists(path=self.local_path_dir + os.sep + self.local_file): shutil.copy(src=self.local_file_path, dst=self.local_path_dir + os.sep + self.local_file)
else: print('\r' + ' ' * l + '\r', end='') k = 0 print(s * (l - k) + '] ' + ' = 100%') options = { 'webdav_hostname': token, 'webdav_login': login, 'webdav_password': password, } client = Client(options) rem_path = 'tmp' if not client.check(rem_path): # если нет, то создаём client.mkdir(rem_path) mymes('Directory [' + rem_path + '] created', 0) client.info(rem_path) video_path = './videos' files = os.listdir(video_path) free_space() for file in files: if re.fullmatch(r'Video\d\.\w{,5}', file): client.upload_async(remote_path=rem_path + '/' + file, local_path=video_path + '/' + file, callback=callback) upload_progress(file, '#', 0.3) callback_flag = False mymes('Uploading of all files is finished', 0)