def run(self):
parser = self.get_optparse()
options, args = parser.parse_args()
self.options = options
if options.listopeners:
self.list_openers()
return 0
ilocals = {}
if options.fs:
for import_opener in options.fs:
module_name, opener_class = import_opener.rsplit('.', 1)
try:
opener_module = __import__(module_name, globals(), ilocals,
[opener_class], -1)
except ImportError:
self.error("Unable to import opener %s\n" % import_opener)
return 0
new_opener = getattr(opener_module, opener_class)
try:
if not issubclass(new_opener, Opener):
self.error('%s is not an fs.opener.Opener\n' %
import_opener)
return 0
except TypeError:
self.error('%s is not an opener class\n' % import_opener)
return 0
if options.verbose:
self.output('Imported opener %s\n' % import_opener)
opener.add(new_opener)
if not six.PY3:
args = [str(arg, sys.getfilesystemencoding()) for arg in args]
self.verbose = options.verbose
try:
return self.do_run(options, args) or 0
except FSError as e:
self.error(self.wrap_error(str(e)) + '\n')
if options.debug:
raise
return 1
except KeyboardInterrupt:
if self.is_terminal():
self.output("\n")
return 0
except SystemExit:
return 0
except Exception as e:
self.error(self.wrap_error('Error - %s\n' % str(e)))
if options.debug:
raise
return 1
def init_about_filesystem():
opener.add(AboutOpener)
for name, text in about.iteritems():
url = "about://%s" % name
fh = opener.open(url, "wb")
if hasattr(text, "create_bitmap"):
fhb = opener.open(text.absolute_path, "rb")
text = fhb.read()
fhb.close()
fh.write(text)
fh.close()
log.debug("Created %s" % url)
def run(self):
parser = self.get_optparse()
options, args = parser.parse_args()
self.check_args(parser, options)
self.options = options
if options.listopeners:
self.list_openers()
return 0
ilocals = {}
if options.fs:
for import_opener in options.fs:
module_name, opener_class = import_opener.rsplit('.', 1)
try:
opener_module = __import__(module_name, globals(), ilocals, [opener_class], -1)
except ImportError:
self.error("Unable to import opener %s\n" % import_opener)
return 0
new_opener = getattr(opener_module, opener_class)
try:
if not issubclass(new_opener, Opener):
self.error('%s is not an fs.opener.Opener\n' % import_opener)
return 0
except TypeError:
self.error('%s is not an opener class\n' % import_opener)
return 0
if options.verbose:
self.output('Imported opener %s\n' % import_opener)
opener.add(new_opener)
if not six.PY3:
args = [unicode(arg, sys.getfilesystemencoding()) for arg in args]
self.verbose = options.verbose
try:
return self.do_run(options, args) or 0
except FSError, e:
self.error(self.wrap_error(unicode(e)) + '\n')
if options.debug:
raise
return 1
def init_about_filesystem(task_factories):
opener.add(AboutOpener)
opener.add(TemplateOpener)
opener.add(BlankOpener)
for name, text in about.iteritems():
save_to_about_filesystem(name, text)
for factory in task_factories:
for name, text in factory.factory.about_filesystem.iteritems():
save_to_about_filesystem(name, text)
* https://www.example.org/index.html"""
@classmethod
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable,
create_dir):
from fs.httpfs import HTTPFS
if '/' in fs_path:
dirname, resourcename = fs_path.rsplit('/', 1)
else:
dirname = fs_path
resourcename = ''
fs = HTTPFS('https://' + dirname)
return fs, resourcename
opener.add(HTTPSOpener)
def patch_match_hostname():
"""Fixes https://github.com/boto/boto/issues/2836"""
_old_match_hostname = ssl.match_hostname
def _new_match_hostname(cert, hostname):
if hostname.endswith('.s3.amazonaws.com'):
pos = hostname.find('.s3.amazonaws.com')
hostname = hostname[:pos].replace('.', '') + hostname[pos:]
return _old_match_hostname(cert, hostname)
ssl.match_hostname = _new_match_hostname
:param fs_name: The name of the opener, as extracted from the protocol
part of the url.
:param fs_name_params: Reserved for future use.
:param fs_path: Path part of the url.
:param writeable: If True, then ``get_fs`` must return an FS that can
be written to.
:param create_dir: If True then ``get_fs`` should attempt to silently
create the directory referenced in the path.
"""
fs_url = "{0}://{1}".format(fs_name, fs_path)
root_url, path, query = spliturl(fs_url)
dirpath, resourcepath = pathsplit(path)
fs = XRootDPyFS(root_url + dirpath + query)
if create_dir and path:
fs.makedir(path, recursive=True, allow_recreate=True)
if dirpath:
fs = fs.opendir(dirpath)
if not resourcepath:
return fs, None
else:
return fs, resourcepath
opener.add(XRootDPyOpener)
desc = """HTTPS file opener. HTTPS only supports reading files, and not much else.
example:
* https://www.example.org/index.html"""
@classmethod
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir):
from fs.httpfs import HTTPFS
if '/' in fs_path:
dirname, resourcename = fs_path.rsplit('/', 1)
else:
dirname = fs_path
resourcename = ''
fs = HTTPFS('https://' + dirname)
return fs, resourcename
opener.add(HTTPSOpener)
def patch_match_hostname():
"""Fixes https://github.com/boto/boto/issues/2836"""
_old_match_hostname = ssl.match_hostname
def _new_match_hostname(cert, hostname):
if hostname.endswith('.s3.amazonaws.com'):
pos = hostname.find('.s3.amazonaws.com')
hostname = hostname[:pos].replace('.', '') + hostname[pos:]
return _old_match_hostname(cert, hostname)
ssl.match_hostname = _new_match_hostname
:param create_dir: if True then get_fs should attempt to silently
create the directory references in path
"""
fs_url = "{0}://{1}".format(fs_name, fs_path)
if not is_valid_url(fs_url):
raise OpenerError('Invalid XRootD URL.')
scheme, netloc, path, params, query, fragment = urlparse(fs_url)
root_url = "{scheme}://{netloc}?{query}".format(scheme=scheme,
netloc=netloc,
query=query)
dirpath, resourcepath = pathsplit(path)
fs = XRootDFS(root_url)
if create_dir and path:
fs.makedir(path, recursive=True, allow_recreate=True)
if dirpath:
fs = fs.opendir(dirpath)
if not resourcepath:
return fs, None
else:
return fs, resourcepath
opener.add(XRootDOpener)
written to
:param create_dir: if True then get_fs should attempt to silently
create the directory references in path
"""
fs_url = "{0}://{1}".format(fs_name, fs_path)
if not is_valid_url(fs_url):
raise OpenerError('Invalid XRootD URL.')
scheme, netloc, path, params, query, fragment = urlparse(fs_url)
root_url = "{scheme}://{netloc}?{query}".format(
scheme=scheme, netloc=netloc, query=query
)
dirpath, resourcepath = pathsplit(path)
fs = XRootDFS(root_url)
if create_dir and path:
fs.makedir(path, recursive=True, allow_recreate=True)
if dirpath:
fs = fs.opendir(dirpath)
if not resourcepath:
return fs, None
else:
return fs, resourcepath
opener.add(XRootDOpener)
def register_openers():
opener.add(CuckooDropboxOpener)
:param fs_name: The name of the opener, as extracted from the protocol
part of the url.
:param fs_name_params: Reserved for future use.
:param fs_path: Path part of the url.
:param writeable: If True, then ``get_fs`` must return an FS that can
be written to.
:param create_dir: If True then ``get_fs`` should attempt to silently
create the directory referenced in the path.
"""
fs_url = "{0}://{1}".format(fs_name, fs_path)
root_url, path, query = spliturl(fs_url)
dirpath, resourcepath = pathsplit(path)
fs = XRootDPyFS(root_url + dirpath + query)
if create_dir and path:
fs.makedir(path, recursive=True, allow_recreate=True)
if dirpath:
fs = fs.opendir(dirpath)
if not resourcepath:
return fs, None
else:
return fs, resourcepath
opener.add(XRootDPyOpener)
def register_openers():
opener.add(CuckooDropboxOpener)
