def sync(self, source, destination):
"""Synchronise two locations."""
start_time = time.time()
self.source = normalise_url(source)
self.destination = normalise_url(destination)
# Instantiate the transports.
try:
self.source_transport = self.transports[url_split(
self.source).scheme]()
except KeyError:
log.error("Protocol not supported: %s." %
url_split(self.source).scheme)
return
try:
self.destination_transport = self.transports[url_split(
self.destination).scheme]()
except KeyError:
log.error("Protocol not supported: %s." %
url_split(self.destination).scheme)
return
# Give the transports a chance to connect to their servers.
try:
self.source_transport.connect(self.source, self.config)
except:
log.error("Connection to source failed, exiting...")
self.exit(1)
try:
self.destination_transport.connect(self.destination, self.config)
except:
log.error("Connection to destination failed, exiting...")
self.exit(1)
# These are the most attributes we can expect from getattr calls in these two protocols.
self.max_attributes = (self.source_transport.getattr_attributes
& self.destination_transport.getattr_attributes)
self.max_evaluation_attributes = (
self.source_transport.evaluation_attributes
& self.destination_transport.evaluation_attributes)
if not self.check_locations():
self.exit(1)
# Begin the actual synchronisation.
self.recurse()
self.source_transport.disconnect()
self.destination_transport.disconnect()
total_time = time.time() - start_time
locale.setlocale(locale.LC_NUMERIC, '')
try:
bps = locale.format("%d", int(self.bytes_total / total_time), True)
except ZeroDivisionError:
bps = "inf"
log.info("Copied %s files (%s bytes) in %s sec (%s Bps)." %
(locale.format("%d", self.file_counter, True),
locale.format("%d", self.bytes_total, True),
locale.format("%.2f", total_time, True), bps))
def recurse(self):
"""Recursively synchronise everything."""
source_dir_list = self.source_transport.listdir(self.source)
dest = FileObject(self.destination_transport, self.destination)
# If the source is a file, rather than a directory, just copy it. We know for sure that
# it exists from the checks we did before, so the "False" return value can't be because
# of that.
if not source_dir_list:
# If the destination ends in a slash or is an actual directory:
if self.destination.endswith("/") or dest.isdir:
if not dest.isdir:
self.destination_transport.mkdir(dest.url)
# Splice the source filename onto the destination URL.
dest_url = url_split(dest.url)
dest_url.file = url_split(self.source,
uses_hostname=self.source_transport.uses_hostname,
split_filename=True).file
dest_url = url_join(dest_url)
else:
dest_url = self.destination
self.compare_and_copy(
FileObject(self.source_transport, self.source, {"isdir": False}),
FileObject(self.destination_transport, dest_url, {"isdir": False}),
)
return
# If source is a directory...
directory_stack = [FileObject(self.source_transport, self.source, {"isdir": True})]
# Depth-first tree traversal.
while directory_stack:
# TODO: Rethink the assumption that a file cannot have the same name as a directory.
item = directory_stack.pop()
log.debug("URL %s is %sa directory." % \
(item.url, not item.isdir and "not " or ""))
if item.isdir:
# Don't skip the first directory.
if not self.config.recursive and item.url != self.source:
log.info("Skipping directory %s..." % item)
continue
# Obtain a directory list.
new_dir_list = []
for new_file in reversed(self.source_transport.listdir(item.url)):
if self.include_file(new_file):
new_dir_list.append(new_file)
else:
log.debug("Skipping %s..." % (new_file))
dest = url_splice(self.source, item.url, self.destination)
dest = FileObject(self.destination_transport, dest)
log.debug("Comparing directories %s and %s..." % (item.url, dest.url))
self.compare_directories(item, new_dir_list, dest.url)
directory_stack.extend(new_dir_list)
else:
dest_url = url_splice(self.source, item.url, self.destination)
log.debug("Destination URL is %s." % dest_url)
dest = FileObject(self.destination_transport, dest_url)
self.compare_and_copy(item, dest)
def sync(self, source, destination):
"""Synchronise two locations."""
start_time = time.time()
self.source = normalise_url(source)
self.destination = normalise_url(destination)
# Instantiate the transports.
try:
self.source_transport = self.transports[url_split(self.source).scheme]()
except KeyError:
log.error("Protocol not supported: %s." % url_split(self.source).scheme)
return
try:
self.destination_transport = self.transports[url_split(self.destination).scheme]()
except KeyError:
log.error("Protocol not supported: %s." % url_split(self.destination).scheme)
return
# Give the transports a chance to connect to their servers.
try:
self.source_transport.connect(self.source, self.config)
except:
log.error("Connection to source failed, exiting...")
self.exit(1)
try:
self.destination_transport.connect(self.destination, self.config)
except:
log.error("Connection to destination failed, exiting...")
self.exit(1)
# These are the most attributes we can expect from getattr calls in these two protocols.
self.max_attributes = (self.source_transport.getattr_attributes &
self.destination_transport.getattr_attributes)
self.max_evaluation_attributes = (self.source_transport.evaluation_attributes &
self.destination_transport.evaluation_attributes)
if not self.check_locations():
self.exit(1)
# Begin the actual synchronisation.
self.recurse()
self.source_transport.disconnect()
self.destination_transport.disconnect()
total_time = time.time() - start_time
locale.setlocale(locale.LC_NUMERIC, '')
try:
bps = locale.format("%d", int(self.bytes_total / total_time), True)
except ZeroDivisionError:
bps = "inf"
log.info("Copied %s files (%s bytes) in %s sec (%s Bps)." % (
locale.format("%d", self.file_counter, True),
locale.format("%d", self.bytes_total, True),
locale.format("%.2f", total_time, True),
bps))
def check_locations(self):
"""Check that the two locations are suitable for synchronisation."""
if url_split(self.source).get_dict().keys == ["scheme"]:
log.error(
"You need to specify something more than that for the source.")
return False
elif url_split(self.source).get_dict().keys == ["scheme"]:
log.error(
"You need to specify more information than that for the destination."
)
return False
elif not self.source_transport.exists(self.source):
log.error("The source location \"%s\" does not exist, aborting." %
self.source)
return False
# Check if both locations are of the same type.
source_isdir = self.source_transport.isdir(self.source)
leave = False
if self.source.startswith(self.destination) and source_isdir:
log.error(
"The destination directory is a parent of the source directory."
)
leave = True
elif not hasattr(self.source_transport, "read"):
log.error("The source protocol is write-only.")
leave = True
elif not hasattr(self.destination_transport, "write"):
log.error("The destination protocol is read-only.")
leave = True
elif not hasattr(self.destination_transport,
"remove") and self.config.delete:
log.error(
"The destination protocol does not support file deletion.")
leave = True
elif self.config.requested_attributes - self.source_transport.getattr_attributes:
log.error("Requested attributes cannot be read: %s." %
", ".join(x for x in self.config.requested_attributes - \
self.source_transport.getattr_attributes)
)
leave = True
elif self.config.requested_attributes - self.destination_transport.setattr_attributes:
log.error("Requested attributes cannot be set: %s." %
", ".join(x for x in self.config.requested_attributes - \
self.destination_transport.setattr_attributes)
)
leave = True
if leave:
return False
else:
return True
def connect(self, url, config):
"""Initiate a connection to the remote host."""
options = config.full_options
# Make the import global.
global paramiko
try:
# We import paramiko only when we need it because its import is really slow.
import paramiko
except ImportError:
print "SFTP: You will need to install the paramiko library to have sftp support."
raise
url = urlfunctions.url_split(url)
if not url.port:
url.port = 22
self._transport = paramiko.Transport((url.hostname, url.port))
username = url.username
if not url.username:
if hasattr(options, "username"):
username = options.username
else:
url.username = getpass.getuser()
password = url.password
if not url.password:
if hasattr(options, "password"):
password = options.password
else:
password = getpass.getpass(
"SFTP: Please enter the password for %s@%s:" % (url.username, url.hostname)
)
self._transport.connect(username=username, password=password)
self._connection = paramiko.SFTPClient.from_transport(self._transport)
def _get_filename(self, url):
"""Retrieve the local filename from a given URL."""
split_url = urlfunctions.url_split(url, uses_hostname=self.uses_hostname)
# paths are relative unless they start with two //
path = split_url.path
if len(path) > 1 and path.startswith("/"):
path = path[1:]
return path
def check_locations(self):
"""Check that the two locations are suitable for synchronisation."""
if url_split(self.source).get_dict().keys == ["scheme"]:
log.error("You need to specify something more than that for the source.")
return False
elif url_split(self.source).get_dict().keys == ["scheme"]:
log.error("You need to specify more information than that for the destination.")
return False
elif not self.source_transport.exists(self.source):
log.error("The source location \"%s\" does not exist, aborting." %
self.source)
return False
# Check if both locations are of the same type.
source_isdir = self.source_transport.isdir(self.source)
leave = False
if self.source.startswith(self.destination) and source_isdir:
log.error("The destination directory is a parent of the source directory.")
leave = True
elif not hasattr(self.source_transport, "read"):
log.error("The source protocol is write-only.")
leave = True
elif not hasattr(self.destination_transport, "write"):
log.error("The destination protocol is read-only.")
leave = True
elif not hasattr(self.destination_transport, "remove") and self.config.delete:
log.error("The destination protocol does not support file deletion.")
leave = True
elif self.config.requested_attributes - self.source_transport.getattr_attributes:
log.error("Requested attributes cannot be read: %s." %
", ".join(x for x in self.config.requested_attributes - \
self.source_transport.getattr_attributes)
)
leave = True
elif self.config.requested_attributes - self.destination_transport.setattr_attributes:
log.error("Requested attributes cannot be set: %s." %
", ".join(x for x in self.config.requested_attributes - \
self.destination_transport.setattr_attributes)
)
leave = True
if leave:
return False
else:
return True
def _get_filename(self, url, remove_slash=True):
"""Retrieve the local filename from a given URL."""
# Remove the trailing slash as a convention unless specified otherwise.
if remove_slash:
urlfunctions.append_slash(url, False)
filename = urlfunctions.url_split(url).path
if filename == "":
filename = "/"
return filename
def _get_filename(self, url, remove_slash=True):
"""Retrieve the local filename from a given URL."""
# Remove the trailing slash as a convention unless specified otherwise.
if remove_slash:
urlfunctions.append_slash(url, False)
filename = urlfunctions.url_split(url).path
if filename == "":
filename = "/"
return filename
def connect(self, url, config):
"""Unpickle the filesystem dictionary."""
self._storage = urlfunctions.url_split(url).hostname
# If the storage is in-memory only, don't do anything.
if self._storage == "memory":
return
try:
pickled_file = open(self._storage, "rb")
except IOError:
return
self._filesystem = pickle.load(pickled_file)
pickled_file.close()
def connect(self, url, config):
"""Unpickle the filesystem dictionary."""
self._storage = urlfunctions.url_split(url).hostname
# If the storage is in-memory only, don't do anything.
if self._storage == "memory":
return
try:
pickled_file = open(self._storage, "rb")
except IOError:
return
self._filesystem = pickle.load(pickled_file)
pickled_file.close()
def connect(self, url, config):
"""Initiate a connection to the remote host."""
options = config.full_options
# Make the import global.
global paramiko
try:
# We import paramiko only when we need it because its import is really slow.
import paramiko
except ImportError:
print "SFTP: You will need to install the paramiko library to have sftp support."
raise
url = urlfunctions.url_split(url)
if not url.port:
url.port = 22
self._transport = paramiko.Transport((url.hostname, url.port))
topts = dict()
topts['username'] = url.username
if not url.username:
if hasattr(options, "username"):
topts['username'] = options.username
else:
topts['username'] = getpass.getuser()
if hasattr(options, "sshkey"):
keyfilename = options.sshkey
keydata = open(keyfilename).read()
if "BEGIN DSA" in keydata or "BEGIN DSS" in keydata:
topts['pkey'] = paramiko.dsskey.DSSKey.from_private_key(StringIO(keydata))
elif "BEGIN RSA" in keydata:
topts['pkey'] = paramiko.rsakey.RSAKey.from_private_key(StringIO(keydata))
else:
raise BadAuthenticationInformation("Key type in file %s cannot be identified" % (keyfilename))
else:
password = url.password
if not url.password:
if hasattr(options, "password"):
topts['password'] = options.password
else:
topts['password'] = getpass.getpass(
"SFTP: Please enter the password for %s@%s:" % (url.username, url.hostname)
)
self._transport.connect(**topts)
self._connection = paramiko.SFTPClient.from_transport(self._transport)
def test_url_join(self):
"""Test url_join."""
tests = (
("http://*****:*****@myhost:80/some/path/file;things?myhost=hi#lala", True, True),
("http://*****:*****@myhost:80/some/path/;things?myhost=hi#lala", True, True),
("http://user@myhost/file;things?myhost=hi#lala", True, True),
("http://myhost/;things?myhost=hi#lala", True, True),
("http://*****:*****@myhost:80/?myhost=hi#lala", True, True),
("myhost/", True, True),
("user:pass@myhost:80/", True, True),
("user:pass@myhost/some#lala", True, True),
("http://myhost:80/;things?myhost=hi#lala", True, True),
("http://myhost/#lala", True, True),
("file://path", False, True),
("file://path/file", False, True),
("file:///path", False, True),
("file:///path/file", False, True),
("file:///path/file?something=else", False, True),
)
for test in tests:
self.assertEqual(urlfunctions.url_join(urlfunctions.url_split(*test)), test[0])
def test_url_join(self):
"""Test url_join."""
tests = (
("http://*****:*****@myhost:80/some/path/file;things?myhost=hi#lala",
True, True),
("http://*****:*****@myhost:80/some/path/;things?myhost=hi#lala",
True, True),
("http://user@myhost/file;things?myhost=hi#lala", True, True),
("http://myhost/;things?myhost=hi#lala", True, True),
("http://*****:*****@myhost:80/?myhost=hi#lala", True, True),
("myhost/", True, True),
("user:pass@myhost:80/", True, True),
("user:pass@myhost/some#lala", True, True),
("http://myhost:80/;things?myhost=hi#lala", True, True),
("http://myhost/#lala", True, True),
("file://path", False, True),
("file://path/file", False, True),
("file:///path", False, True),
("file:///path/file", False, True),
("file:///path/file?something=else", False, True),
)
for test in tests:
self.assertEqual(
urlfunctions.url_join(urlfunctions.url_split(*test)), test[0])
def _get_filename(self, url):
"""Retrieve the local filename from a given URL."""
split_url = urlfunctions.url_split(url, uses_hostname=self.uses_hostname)
return split_url.path
def recurse(self):
"""Recursively synchronise everything."""
source_dir_list = self.source_transport.listdir(self.source)
dest = FileObject(self.destination_transport, self.destination)
# If the source is a file, rather than a directory, just copy it. We know for sure that
# it exists from the checks we did before, so the "False" return value can't be because
# of that.
if not source_dir_list:
# If the destination ends in a slash or is an actual directory:
if self.destination.endswith("/") or dest.isdir:
if not dest.isdir:
self.destination_transport.mkdir(dest.url)
# Splice the source filename onto the destination URL.
dest_url = url_split(dest.url)
dest_url.file = url_split(
self.source,
uses_hostname=self.source_transport.uses_hostname,
split_filename=True).file
dest_url = url_join(dest_url)
else:
dest_url = self.destination
self.compare_and_copy(
FileObject(self.source_transport, self.source,
{"isdir": False}),
FileObject(self.destination_transport, dest_url,
{"isdir": False}),
)
return
# If source is a directory...
directory_stack = [
FileObject(self.source_transport, self.source, {"isdir": True})
]
# Depth-first tree traversal.
while directory_stack:
# TODO: Rethink the assumption that a file cannot have the same name as a directory.
item = directory_stack.pop()
log.debug("URL %s is %sa directory." % \
(item.url, not item.isdir and "not " or ""))
if item.isdir:
# Don't skip the first directory.
if not self.config.recursive and item.url != self.source:
log.info("Skipping directory %s..." % item)
continue
# Obtain a directory list.
new_dir_list = []
for new_file in reversed(
self.source_transport.listdir(item.url)):
if self.include_file(new_file):
new_dir_list.append(new_file)
else:
log.debug("Skipping %s..." % (new_file))
dest = url_splice(self.source, item.url, self.destination)
dest = FileObject(self.destination_transport, dest)
log.debug("Comparing directories %s and %s..." %
(item.url, dest.url))
self.compare_directories(item, new_dir_list, dest.url)
directory_stack.extend(new_dir_list)
else:
dest_url = url_splice(self.source, item.url, self.destination)
log.debug("Destination URL is %s." % dest_url)
dest = FileObject(self.destination_transport, dest_url)
self.compare_and_copy(item, dest)
def compare_directories(self, source, source_dir_list, dest_dir_url):
"""Compare the source's directory list with the destination's and perform any actions
necessary, such as deleting files or creating directories."""
dest_dir_list = self.destination_transport.listdir(dest_dir_url)
if not dest_dir_list:
if not self.config.dry_run:
self.destination_transport.mkdir(dest_dir_url)
# Populate the item's attributes for the remote directory so we can set them.
attribute_set = self.max_evaluation_attributes & \
self.destination_transport.setattr_attributes
attribute_set = attribute_set | self.config.requested_attributes
attribute_set = attribute_set ^ self.config.exclude_attributes
source.populate_attributes(attribute_set)
self.set_destination_attributes(dest_dir_url,
source.attributes)
dest_dir_list = []
# Construct a dictionary of {filename: FileObject} items.
dest_paths = dict([(url_split(append_slash(x.url, False),
self.destination_transport.uses_hostname,
True).file, x) for x in dest_dir_list])
create_dirs = []
for item in source_dir_list:
# Remove slashes so the splitter can get the filename.
url = url_split(append_slash(item.url, False),
self.source_transport.uses_hostname, True).file
# If the file exists and both the source and destination are of the same type...
if url in dest_paths and dest_paths[url].isdir == item.isdir:
# ...if it's a directory, set its attributes as well...
if dest_paths[url].isdir:
log.info("Setting attributes for %s..." % url)
item.populate_attributes(
self.max_evaluation_attributes
| self.config.requested_attributes)
self.set_destination_attributes(dest_paths[url].url,
item.attributes)
# ...and remove it from the list.
del dest_paths[url]
else:
# If an item is in the source but not the destination tree...
if item.isdir and self.config.recursive:
# ...create it if it's a directory.
create_dirs.append(item)
if self.config.delete:
for item in dest_paths.values():
if item.isdir:
if self.config.recursive:
log.info("Deleting destination directory %s..." % item)
self.recursively_delete(item)
else:
log.info("Deleting destination file %s..." % item)
self.destination_transport.remove(item.url)
if self.config.dry_run:
return
# Create directories after we've deleted everything else because sometimes a directory in
# the source might have the same name as a file, so we need to delete files first.
for item in create_dirs:
dest_url = url_splice(self.source, item.url, self.destination)
self.destination_transport.mkdir(dest_url)
item.populate_attributes(self.max_evaluation_attributes
| self.config.requested_attributes)
self.set_destination_attributes(dest_url, item.attributes)
def test_url_split(self):
"""Test url_split."""
tests = (
(("http://*****:*****@myhost:80/some/path/file;things?myhost=hi#lala",
True, False), {
"scheme": "http",
"netloc": "user:pass@myhost:80",
"username": "******",
"password": "******",
"hostname": "myhost",
"port": 80,
"path": "/some/path/file",
"file": "",
"params": "things",
"query": "myhost=hi",
"anchor": "lala"
}),
(("http://myhost/some/path/file;things?myhost=hi#lala", True,
False), {
"scheme": "http",
"netloc": "myhost",
"username": "",
"password": "",
"hostname": "myhost",
"port": 0,
"path": "/some/path/file",
"file": "",
"params": "things",
"query": "myhost=hi",
"anchor": "lala"
}),
(("http://user@myhost/some/path/", True, False), {
"scheme": "http",
"netloc": "user@myhost",
"username": "******",
"password": "",
"hostname": "myhost",
"port": 0,
"path": "/some/path/",
"file": "",
"params": "",
"query": "",
"anchor": ""
}),
(("http://myhost", True, False), {
"scheme": "http",
"netloc": "myhost",
"username": "",
"password": "",
"hostname": "myhost",
"port": 0,
"path": "",
"file": "",
"params": "",
"query": "",
"anchor": ""
}),
(("file://some/directory", False, False), {
"scheme": "file",
"path": "some/directory",
"file": "",
"params": "",
"query": "",
"anchor": ""
}),
(("file://some/directory", True, True), {
"scheme": "file",
"netloc": "some",
"username": "",
"password": "",
"hostname": "some",
"port": 0,
"path": "/",
"file": "directory",
"params": "",
"query": "",
"anchor": ""
}),
(("host", True, True), {
"scheme": "",
"netloc": "host",
"username": "",
"password": "",
"hostname": "host",
"port": 0,
"path": "",
"file": "",
"params": "",
"query": "",
"anchor": ""
}),
(("http://*****:*****@myhost:80/some/path/file;things?arg=hi#lala",
True, True), {
"scheme": "http",
"netloc": "user:pass@myhost:80",
"username": "******",
"password": "******",
"hostname": "myhost",
"port": 80,
"path": "/some/path/",
"file": "file",
"params": "things",
"query": "arg=hi",
"anchor": "lala"
}),
(("http://*****:*****@myhost:80/some/path/file#lala", True, False), {
"scheme": "http",
"netloc": "user:pass@myhost:80",
"username": "******",
"password": "******",
"hostname": "myhost",
"port": 80,
"path": "/some/path/file",
"file": "",
"params": "",
"query": "",
"anchor": "lala"
}),
(("file://I:/some/path/file", False, True), {
"scheme": "file",
"path": "I:/some/path/",
"file": "file",
"params": "",
"query": "",
"anchor": ""
}),
(("file://file", False, True), {
"scheme": "file",
"path": "",
"file": "file",
"params": "",
"query": "",
"anchor": ""
}),
)
for test, expected_output in tests:
result = urlfunctions.url_split(*test)
for key in expected_output.keys():
self.assertEqual(getattr(result, key), expected_output[key])
def compare_directories(self, source, source_dir_list, dest_dir_url):
"""Compare the source's directory list with the destination's and perform any actions
necessary, such as deleting files or creating directories."""
dest_dir_list = self.destination_transport.listdir(dest_dir_url)
if not dest_dir_list:
if not self.config.dry_run:
self.destination_transport.mkdir(dest_dir_url)
# Populate the item's attributes for the remote directory so we can set them.
attribute_set = self.max_evaluation_attributes & \
self.destination_transport.setattr_attributes
attribute_set = attribute_set | self.config.requested_attributes
attribute_set = attribute_set ^ self.config.exclude_attributes
source.populate_attributes(attribute_set)
self.set_destination_attributes(dest_dir_url, source.attributes)
dest_dir_list = []
# Construct a dictionary of {filename: FileObject} items.
dest_paths = dict([(url_split(append_slash(x.url, False),
self.destination_transport.uses_hostname,
True).file, x) for x in dest_dir_list])
create_dirs = []
for item in source_dir_list:
# Remove slashes so the splitter can get the filename.
url = url_split(append_slash(item.url, False),
self.source_transport.uses_hostname,
True).file
# If the file exists and both the source and destination are of the same type...
if url in dest_paths and dest_paths[url].isdir == item.isdir:
# ...if it's a directory, set its attributes as well...
if dest_paths[url].isdir:
log.info("Setting attributes for %s..." % url)
item.populate_attributes(self.max_evaluation_attributes |
self.config.requested_attributes)
self.set_destination_attributes(dest_paths[url].url, item.attributes)
# ...and remove it from the list.
del dest_paths[url]
else:
# If an item is in the source but not the destination tree...
if item.isdir and self.config.recursive:
# ...create it if it's a directory.
create_dirs.append(item)
if self.config.delete:
for item in dest_paths.values():
if item.isdir:
if self.config.recursive:
log.info("Deleting destination directory %s..." % item)
self.recursively_delete(item)
else:
log.info("Deleting destination file %s..." % item)
self.destination_transport.remove(item.url)
if self.config.dry_run:
return
# Create directories after we've deleted everything else because sometimes a directory in
# the source might have the same name as a file, so we need to delete files first.
for item in create_dirs:
dest_url = url_splice(self.source, item.url, self.destination)
self.destination_transport.mkdir(dest_url)
item.populate_attributes(self.max_evaluation_attributes |
self.config.requested_attributes)
self.set_destination_attributes(dest_url, item.attributes)
def _get_filename(self, url):
"""Retrieve the local filename from a given URL."""
split_url = urlfunctions.url_split(url, uses_hostname=self.uses_hostname)
return split_url.path
def test_url_split(self):
"""Test url_split."""
tests = (
(("http://*****:*****@myhost:80/some/path/file;things?myhost=hi#lala", True, False),
{"scheme": "http",
"netloc": "user:pass@myhost:80",
"username": "******",
"password": "******",
"hostname": "myhost",
"port": 80,
"path": "/some/path/file",
"file": "",
"params": "things",
"query": "myhost=hi",
"anchor": "lala"}),
(("http://myhost/some/path/file;things?myhost=hi#lala", True, False),
{"scheme": "http",
"netloc": "myhost",
"username": "",
"password": "",
"hostname": "myhost",
"port": 0,
"path": "/some/path/file",
"file": "",
"params": "things",
"query": "myhost=hi",
"anchor": "lala"}),
(("http://user@myhost/some/path/", True, False),
{"scheme": "http",
"netloc": "user@myhost",
"username": "******",
"password": "",
"hostname": "myhost",
"port": 0,
"path": "/some/path/",
"file": "",
"params": "",
"query": "",
"anchor": ""}),
(("http://myhost", True, False),
{"scheme": "http",
"netloc": "myhost",
"username": "",
"password": "",
"hostname": "myhost",
"port": 0,
"path": "",
"file": "",
"params": "",
"query": "",
"anchor": ""}),
(("file://some/directory", False, False),
{"scheme": "file",
"path": "some/directory",
"file": "",
"params": "",
"query": "",
"anchor": ""}),
(("file://some/directory", True, True),
{"scheme": "file",
"netloc": "some",
"username": "",
"password": "",
"hostname": "some",
"port": 0,
"path": "/",
"file": "directory",
"params": "",
"query": "",
"anchor": ""}),
(("host", True, True),
{"scheme": "",
"netloc": "host",
"username": "",
"password": "",
"hostname": "host",
"port": 0,
"path": "",
"file": "",
"params": "",
"query": "",
"anchor": ""}),
(("http://*****:*****@myhost:80/some/path/file;things?arg=hi#lala", True, True),
{"scheme": "http",
"netloc": "user:pass@myhost:80",
"username": "******",
"password": "******",
"hostname": "myhost",
"port": 80,
"path": "/some/path/",
"file": "file",
"params": "things",
"query": "arg=hi",
"anchor": "lala"}),
(("http://*****:*****@myhost:80/some/path/file#lala", True, False),
{"scheme": "http",
"netloc": "user:pass@myhost:80",
"username": "******",
"password": "******",
"hostname": "myhost",
"port": 80,
"path": "/some/path/file",
"file": "",
"params": "",
"query": "",
"anchor": "lala"}),
(("file://I:/some/path/file", False, True),
{"scheme": "file",
"path": "I:/some/path/",
"file": "file",
"params": "",
"query": "",
"anchor": ""}),
(("file://file", False, True),
{"scheme": "file",
"path": "",
"file": "file",
"params": "",
"query": "",
"anchor": ""}),
)
for test, expected_output in tests:
result = urlfunctions.url_split(*test)
for key in expected_output.keys():
self.assertEqual(getattr(result, key), expected_output[key])
