def _naive_relative_to(path: PosixPath, root: PosixPath) -> PurePosixPath:
"""
Compute relative PurePosixPath, result may include '..' parts.
Both arguments must be absolute PurePosixPath's and lack '..' parts.
Possibility of symlinks is ignored, i. e. arguments are interpreted
as resolved paths.
"""
if not path.is_absolute():
raise ValueError(path)
if not root.is_absolute():
raise ValueError(root)
if '..' in path.parts:
raise ValueError(path)
if '..' in root.parts:
raise ValueError(root)
upstairs = 0
while root not in path.parents:
parent = root.parent
assert parent != root
root = parent
upstairs += 1
return PurePosixPath(
* (('..',) * upstairs),
path.relative_to(root) )
def __init__(self,
root_directory,
token,
file_size_limit=FILE_SIZE_LIMIT,
min_chunk_size=MINIMUM_CHUNK_SIZE,
max_chunk_size=MAXIMUM_CHUNK_SIZE):
""" Construct a :py:class:`Server` instance.
Long description.
:param root_directory: foobar.
:param token: foobar.
:param file_size_limit: foobar.
:param min_chunk_size: foobar.
:param max_chunk_size: foobar.
"""
root_directory = PosixPath(root_directory)
if not root_directory.is_absolute():
root_directory = PosixPath(os.getcwd(), root_directory)
if not root_directory.exists() or root_directory.is_file():
raise NotADirectoryError(
"The root directory must be an existing directory")
self.root_directory = root_directory
self.token = bytes(token, 'utf-8')
if file_size_limit <= 0:
raise ValueError("The file size limit must be greater than 0")
self.file_size_limit = file_size_limit
if min_chunk_size <= 0:
raise ValueError("The minimum chunk size must be greater than 0")
self.min_chunk_size = min_chunk_size
self.max_chunk_size = max_chunk_size
self.state = ServerState.IDLE
# server UPLOAD and DOWNLOAD state related attributes
self.chunk_size = 0
self.temporary_file = None # file open in 'wb' mode OR file open in 'rb' mode
self.file_description = None
self.remaining_bytes = 0
self.file_destination = None # upload only attribute
# socket relate attributes
self.router = None
self.dealer = None
self.socket = None
self.is_running = False
def look_for_file(filename, paths):
'''
Tries to smartly find the absolute path of a config file.
If the given path is absolute and exists return it unmodified, otherwise do usual leaf based lookup
If the given path contains only a file name check for existence in _SEARCH_DIRS returning if found
If the given path contains a relative filepath check for existence in _SEARCH_DIRS joining each with the fragement
'''
filename = PosixPath(filename)
if filename.is_absolute():
if filename.exists():
return filename
return None
for confDir in paths:
if confDir.joinpath(filename).exists():
return confDir.joinpath(filename).resolve()
return None
def infer_settings(opt_root, opt_pattern="**/optimizer.py"):
opt_root = PosixPath(opt_root)
assert opt_root.is_dir(), "Opt root directory doesn't exist: %s" % opt_root
assert opt_root.is_absolute(), "Only absolute path should have even gotten this far."
# Always sort for reproducibility
source_files = sorted(opt_root.glob(opt_pattern))
source_files = [ss.relative_to(opt_root) for ss in source_files]
settings = {_cleanup(str(ss.parent)): [str(ss), {}] for ss in source_files}
assert all(joinable(kk) for kk in settings), "Something went wrong in name sanitization."
assert len(settings) == len(source_files), "Name collision after sanitization of %s" % repr(source_files)
assert len(set(CONFIG.keys()) & set(settings.keys())) == 0, "Name collision with builtin optimizers."
return settings
def download_directory(self,
source,
destination,
name=None,
chunk_size=512,
process_chunk=None,
timeout=None):
""" Download a directory from the remote directory.
This method downloads an entire directory from a given directory
in the remote directory.
The **source** parameter refers to the remote directory to be
transfered from the remote directory and must to be a
:term:`path-like object`. It must be an absolute path or it will
raise the ValueError exception. If the source directory can't be
found or is not a directory, the SourceNotFound exception is
raised.
The **destination** parameter refers to **an existing** local
directory in which the directory must be transfered to. It must
be a :term:`path-like object` and if it's a relative path, it's
treated like relative to the current working directory. If the
destination directory can't be found or is not a directory, the
DestinationNotFound exception is raised.
The name parameter can be used to rename the source directory
while downloading it (the content is guaranteed to be the same).
It must be a string of a :term:`valid file name` and must not
conflict with an existing directory (or file) in the destination
directory. By default, it reads the name from the source to
leave it unchanged. If the name isn't valid, a
:py:exc:`InvalidFileName` is raised and if the file is
conflicting, a :py:exc:`FileExistsError` exception is raised.
Additionally, you can adjust the chunk size value which defines
how fragmented files have to be received from the server and/or
pass a callback that process each fragment **before** it's
written to the local file. Usually, the chunk value is
between 512 and 8192.
The callback is called with various parameters and in a
specific order; the chunk data, the remaining bytes, the file
size and the file name. The chunk data is a bytes string of the
actual data just received from the server. The remaining bytes
is an integer indicating the number of bytes left to be received
(and this includes the current chunk of data). The file size is
a fixed integer telling how large the file is, and the file name
is the file name currently being processed.
For instance, it can be used to display a progress indicator.
Here is an example. ::
def display_progress(chunk_data, remaining_bytes, file_size, file_name):
chunk_size = 512
progress = (file_size - (remaining_bytes - len(chunk_data))) / file_size * 100
sys.stdout.write("\r{0:0.2f}% | {1}".format(progress, file_name))
sys.stdout.flush()
if remaining_bytes <= chunk_size:
sys.stdout.write('\n')
return True
If the operation takes longer than the given timeout, a
:py:exc:`TimeoutError` exception is raised.
:param source: Foobar.
:param destination: Foobar.
:param name: Foobar.
:param chunk_size: Foobar.
:param process_chunk: Foobar.
:param timeout: Foobar.
:raises ValueError: If the source directory isn't an absolute path.
:raises SourceNotFound: If the source file doesn't exist or isn't a file.
:raises DestinationNotFound: If the destination directory doesn't exist or isn't a directory.
:raises FileExistsError: If the source file conflicts with an existing file or directory.
:raises InvalidFileName: If the source file doesn't have a valid name.
:raises TimeoutError: If it takes more than the timeout value to receive a response.
"""
# ensure we work with posix paths
source = PurePosixPath(source)
destination = PosixPath(destination)
# normalize the destination to work with an absolute path
if not destination.is_absolute():
destination = PosixPath(os.getcwd(), destination)
# compute the name from the source if not specified (file name
# unchanged)
if not name:
name = source.name
# raise ValueError exception if source directory is not an
# absolute path
if not source.is_absolute():
raise ValueError("Source must be an absolute path")
# raise SourceNotFound exception if the source directory doesn't
# exist or is not a directory
if str(source) != source.root:
try:
files = self.list_files(source.parent, timeout)
except Exception:
raise NotImplementedError # catch and treat relevant exceptions
if source.name not in files or files[source.name][1] == True:
raise SourceNotFound("Source directory could not be found")
# check if the destination directory exists and raises
# DestinationNotFound exception if it doesn't exist or is not
# a directory (a root is always a valid destination)
if not destination.exists() or not destination.is_dir():
raise DestinationNotFound(
"Destination directory could not be found")
# check if the file name doesn't conflict with an existing file
# (or directory) in the destination directory
if name in os.listdir(destination):
raise FileExistsError
# the following code is a workaround! it should let the server
# refuse the chunk size instead, but if we do that, the
# first directory is created first and left undeleted after the
# first file is denied from being downloaded
if chunk_size == 0 or chunk_size > 8192:
raise ValueError("Chunk size value is invalid")
# foobars
self._download_directory(source, destination, name, chunk_size,
process_chunk, timeout)
# Config
config_dest = "./configurations"
config_index = "./config_index.csv"
# Read file list
config_list = pd.read_csv(config_index)
# For all config files
for index, row in config_list.iterrows():
# Get row data
path = PosixPath(row['path'])
private = row['is_private'] == 1
# Destination resolving
expendad_path = path.expanduser()
if path.is_absolute():
dest = config_dest + "/" + str(path)
elif path.parts[0] == "~":
dest = config_dest + "/HOME/" + str(path.relative_to("~"))
else:
dest = config_dest + expendad_path
dest_dir = os.path.dirname(dest)
# Backup
if expendad_path.exists():
# Make parent dir if necessary
os.makedirs(dest_dir, exist_ok=True)
# Make dummy placeholders for private files
if private:
dest = dest + "_DUMMY"
if expendad_path.is_file():
print("DUMMY backup file : " + str(path) + ", to : " + dest)
