def verify_directory_deletion(configuration_file=config_file,
to_delete="dummy.txt"):
mgmt = folder_management.ADLSFolderManagement(configuration_file)
print("Removing directory", to_delete)
rc = mgmt.delete_directory(to_delete)
rc["reference"] = "delete_directory for >" + to_delete + "<."
print("delete returned", rc)
def verify_directory_deletion(
configuration_file="resources/connection_config.json",
to_delete=folder):
mgmt = folder_management.ADLSFolderManagement(configuration_file)
print("Removing directory", folder)
rc = mgmt.delete_directory(folder)
rc["reference"] = "delete_directory for >" + to_delete + "<."
print("delete returned", rc)
def test_list_files(self):
handler = interface_file_handling.InterfaceFileHandling(
configuration_file="tests/resources/connection_config.json")
mgmt = folder_management.ADLSFolderManagement(
configuration_file="tests/resources/connection_config.json")
folder = "test-list-files"
mgmt.create_directory(folder)
result, files, filenames = handler.list_files(folder, "*")
# assert files is not None
assert result["code"] == "OK"
mgmt.delete_directory(folder)
def __init__(self, configuration_file, run_id=None):
self.run_id = None
self.run_id, self.time_id = self.determine_run_id()
print("run_id is >%s< with time_stamp >%s<" %
(self.run_id, self.time_id))
self.result = messages.message["ok"]
self.file_handler = interface_file_handling.InterfaceFileHandling(
configuration_file=configuration_file)
self.mgmt = folder_management.ADLSFolderManagement(
configuration_file=configuration_file)
self.container_handler = self.mgmt.container_client
self.oldest_name = None
def create_directory(configuration_file=config_file, to_create=None):
"""
Configuration and access verification
create and delete folder "verification"
"""
print(__name__)
mgmt = folder_management.ADLSFolderManagement(configuration_file)
print("Creating directory:", to_create)
rc = mgmt.create_directory(to_create)
rc["reference"] = "create_directory for >" + to_create + "<."
print("create returned", rc)
return rc
def test_copy_files(self):
self.handler = interface_file_handling.InterfaceFileHandling(
configuration_file="tests/resources/connection_config.json")
mgmt = folder_management.ADLSFolderManagement(
configuration_file="tests/resources/connection_config.json")
source_folder = "test-source"
mgmt.create_directory(source_folder)
target_folder = "test-target"
mgmt.create_directory(target_folder)
result = self.handler.copy_files(from_location=source_folder,
to_location=target_folder,
file_pattern="*")
assert result["code"] == "OK"
def verify_directory_creation(
configuration_file="resources/connection_config.json",
to_create=folder):
"""
Configuration and access verification
create and delete folder "verification"
"""
print(__name__)
mgmt = folder_management.ADLSFolderManagement(configuration_file)
print("Creating directory:", to_create)
rc = mgmt.create_directory(to_create)
rc["reference"] = "create_directory for >" + to_create + "<."
print("create returned", rc)
def __init__(self, configuration_file):
# use provided connection config file
self.settings = settings.GenericSettings(configuration_file=configuration_file)
self.settings.get_config()
self.service_client, self.file_system_client, self.blob_service_client, self.container_client, self.sas_token \
= connection.ConnectionManagement(self.settings).create_connection(
storage_account_name=self.settings.storage_account_name
, storage_account_key=self.settings.storage_account_key
, container=self.settings.storage_container
)
self.mgmt = folder_management.ADLSFolderManagement(configuration_file=configuration_file)
self.max_wait_in_sec = 60
self.recheck = 10
self.tgt = None
def test_check_files(self):
self.handler = interface_file_handling.InterfaceFileHandling(
configuration_file="tests/resources/connection_config.json")
mgmt = folder_management.ADLSFolderManagement(
configuration_file="tests/resources/connection_config.json")
source_folder = str(uuid.uuid4()) + "test-source"
mgmt.create_directory(source_folder)
target_folder = str(uuid.uuid4()) + "test-target"
mgmt.create_directory(target_folder)
# both will be empty, so equal
result = self.handler.check_files(source_location=source_folder,
target_location=target_folder,
file_pattern="*")
assert result["code"] == "OK"
mgmt.delete_directory(source_folder)
mgmt.delete_directory(target_folder)
def main():
batch_handler = from_to_phase.FilesBatchPhaseFromTo(
configuration_file=config_file, run_id=None)
handler = interface_file_handling.InterfaceFileHandling(
configuration_file=config_file)
mgmt = folder_management.ADLSFolderManagement(
configuration_file=config_file)
create_directory(configuration_file=config_file,
to_create=mgmt.settings.incoming)
create_directory(configuration_file=config_file,
to_create=mgmt.settings.todo)
result = prepare_incoming(handler, mgmt)
print("result prepare_incoming: ", result)
result = batch_handler.from_incoming2todo()
print("result from_incoming2todo:", result)
result = batch_handler.from_todo2busy()
print("result from_todo2busy", result)
result = batch_handler.from_busy2done()
print("result from_busy2done", result)
return result
def verify_copy_files(configuration_file="resources/connection_config.json"):
print(__name__)
print("Overview of the verification steps:")
print("1. create two verification directories")
print("2. verify these are the same (empty)")
print("3. upload a file")
print("4. copy the file")
print(
"5. verify the two directories contain the same files (the one that was uploaded)"
)
print("=== Executing verification ===")
handler = interface_file_handling.InterfaceFileHandling(
configuration_file="resources/connection_config.json")
mgmt = folder_management.ADLSFolderManagement(configuration_file)
print("1. create two verification directories")
print("\tsource_folder: " + source_folder)
mgmt.create_directory(source_folder)
print("\ttarget_folder: " + target_folder)
mgmt.create_directory(target_folder)
print("3. upload a file")
handler.upload_file(source_folder, dummy_file)
print("4. copy the file(s)")
result = handler.copy_files(from_location=source_folder,
to_location=target_folder,
file_pattern="*")
print(result)
print(
"5. verify the two directories contain the same files (the one that was uploaded)"
)
result = handler.check_files(source_location=source_folder,
target_location=target_folder,
file_pattern="*")
print(result)
print("=== Cleanup ===")
print("\tremoving " + source_folder)
mgmt.delete_directory(source_folder)
print("\tremoving " + target_folder)
mgmt.delete_directory(target_folder)
def test_create_folder():
folder_management.ADLSFolderManagement(
"tests/resources/connection_config.json").create_directory("test")