def vcat():
usage = "%prog [options] vos:VOSpace/node_name"
description = "Writes the content of vos:VOSpace/node_name to stdout."
parser = CommonParser(usage, description=description)
parser.add_option("-q", help="run quietly, exit on error without message", action="store_true")
(opt, args) = parser.parse_args()
parser.process_informational_options()
if not len(args) > 0:
parser.error("no argument given")
logger = logging.getLogger()
exit_code = 0
for uri in args:
try:
_cat(uri, cert_filename=opt.certfile)
except Exception as e:
exit_code = getattr(e, 'errno', -1)
if not opt.q:
logger.error(str(e))
sys.exit(exit_code)
def test_all(self):
"""Test CommonParser."""
commonParser = CommonParser()
# hijack the arguments
sys.argv = ['myapp', '-w']
commonParser.process_informational_options()
self.assertEquals(logging.WARNING, commonParser.log_level)
self.assertEquals(version, commonParser.version)
self.assertEquals(logging.WARNING, logging.getLogger().level)
# Remove all handlers associated with the root logger object.
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
commonParser = CommonParser()
sys.argv = ['myapp', '-d']
commonParser.process_informational_options()
self.assertEquals(logging.DEBUG, commonParser.log_level)
self.assertEquals(version, commonParser.version)
sys.argv = ['myapp', '--version']
with patch('vos.commonparser.sys.exit', Mock()):
commonParser.process_informational_options()
def vlock():
signal.signal(signal.SIGINT, signal_handler)
parser = CommonParser()
parser.add_option("--lock", action="store_true", help="Lock the node")
parser.add_option("--unlock", action="store_true", help="unLock the node")
(opt, args) = parser.parse_args()
parser.process_informational_options()
logger = logging.getLogger()
logger.setLevel(parser.log_level)
logger.addHandler(logging.StreamHandler())
lock = None
if opt.lock:
lock = 'true'
elif opt.unlock:
lock = 'false'
exit_code = 0
try:
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
node = client.get_node(args[0])
if opt.lock or opt.unlock:
lock = not opt.unlock and opt.lock
node.is_locked = lock
client.update(node)
else:
print(node.is_locked)
if not node.is_locked:
exit_code = -1
except KeyboardInterrupt:
exit_code = -1
except Exception as e:
logger.error(str(e))
exit_code = -1
sys.exit(exit_code)
def test_all(self):
"""Test CommonParser."""
common_parser = CommonParser()
# hijack the arguments
sys.argv = ['myapp', '-w']
args = common_parser.parse_args()
set_logging_level_from_args(args)
self.assertEqual(version, common_parser.version)
self.assertEqual(logging.WARNING, logging.getLogger('root').level)
# Remove all handlers associated with the root logger object.
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
common_parser = CommonParser()
sys.argv = ['myapp', '-d']
args = common_parser.parse_args()
set_logging_level_from_args(args)
self.assertEqual(logging.DEBUG, logging.getLogger().level)
self.assertEqual(version, common_parser.version)
common_parser = CommonParser()
sys.argv = ['myapp', '--version']
with patch('vos.commonparser.sys.exit', Mock()):
common_parser.parse_args()
def vrm():
usage="""
vrm vos:/root/node -- deletes a data node
Version: %s """ % (version.version)
parser=CommonParser(usage)
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt, args)=parser.parse_args()
parser.process_informational_options()
logger = logging.getLogger()
logger.setLevel(parser.log_level)
try:
client=vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token)
except Exception as e:
logger.error("Connection failed: %s" % (str(e)))
exit_code = getattr(e, 'errno', -1)
sys.exit(exit_code)
for arg in args:
if arg[0:4]!="vos:":
logger.error("%s is not a valid VOSpace handle" % (arg))
try:
if client.isfile(arg) or client.get_node(arg).islink():
logger.info("deleting %s" %(arg))
client.delete(arg)
elif client.isdir(arg):
logger.error("%s is a directory" % (arg))
elif client.access(arg):
logger.info("deleting link %s" %(arg))
client.delete(arg)
else:
logger.error("%s file not found" % (arg))
sys.exit(-1)
except Exception as e:
import re
if re.search('NodeLocked',str(e)) != None:
logger.error("Use vlock to unlock %s before deleting." % (arg))
exit_code = getattr(e, 'errno', -1)
logger.error("Failed trying to delete {0}: {1}".format( arg, str(e)))
sys.exit(exit_code)
def test_all(self):
"""Test CommonParser."""
commonParser = CommonParser()
# hijack the arguments
sys.argv = ['myapp', '-w']
commonParser.process_informational_options()
self.assertEquals(logging.WARNING, commonParser.log_level)
self.assertEquals(version, commonParser.version)
self.assertEquals(logging.WARNING, logging.getLogger().level)
# Remove all handlers associated with the root logger object.
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
commonParser = CommonParser()
sys.argv = ['myapp', '-d']
commonParser.process_informational_options()
self.assertEquals(logging.DEBUG, commonParser.log_level)
self.assertEquals(version, commonParser.version)
sys.argv = ['myapp', '--version']
with patch('vos.commonparser.sys.exit', Mock()):
commonParser.process_informational_options()
def vrmdir():
usage="""
vrmdir vos:/root/node -- deletes a container node
Version: %s """ % (version.version)
parser = CommonParser(usage)
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt, args) = parser.parse_args()
parser.process_informational_options()
try:
client=vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
except Exception as e:
logging.error("Connection failed: %s" % (str(e)))
sys.exit(e.__getattribute__('errno', -1))
try:
for arg in args:
if arg[0:4]!="vos:":
logging.error("%s is not a valid VOSpace handle" % (arg))
sys.exit(-1)
if client.isdir(arg):
logging.info("deleting %s" %(arg))
client.delete(arg)
elif client.isfile(arg):
logging.error("%s is a file" % (arg))
sys.exit(-1)
else:
logging.error("%s file not found" % (arg))
sys.exit(-1)
except Exception as e:
import re
if re.search('NodeLocked', str(e)) != None:
logging.error("Use vlock to unlock %s before removing." %(arg))
logging.error("Connection failed: %s" % (str(e)))
sys.exit(-1)
def test_all(self):
"""Test CommonParser."""
common_parser = CommonParser()
# hijack the arguments
sys.argv = ['myapp', '-w']
args = common_parser.parse_args()
set_logging_level_from_args(args)
self.assertEqual(version, common_parser.version)
self.assertEqual(logging.WARNING, logging.getLogger('root').level)
# Remove all handlers associated with the root logger object.
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
common_parser = CommonParser()
sys.argv = ['myapp', '-d']
args = common_parser.parse_args()
set_logging_level_from_args(args)
self.assertEqual(logging.DEBUG, logging.getLogger().level)
self.assertEqual(version, common_parser.version)
common_parser = CommonParser()
sys.argv = ['myapp', '--version']
with patch('vos.commonparser.sys.exit', Mock()):
common_parser.parse_args()
def vmkdir():
usage="""
vmkdir vos:/root/node -- creates a new directory (ContainerNode) called node at vospace root
Version: %s """ % (version.version)
parser=CommonParser(usage)
parser.add_option("-p",action="store_true",help="Create intermediate directories as required.")
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt,args)=parser.parse_args()
parser.process_informational_options()
if len(args)>1:
parser.error("Only one directory can be built per call")
logging.info("Creating ContainerNode (directory) %s" % ( args[0]))
try:
client=vos.Client(vospace_certfile=opt.certfile,vospace_token=opt.token)
dirNames=[]
thisDir = args[0]
if opt.p:
while not client.access(thisDir):
dirNames.append(os.path.basename(thisDir))
thisDir = os.path.dirname(thisDir)
while len(dirNames) > 0:
thisDir = os.path.join(thisDir,dirNames.pop())
client.mkdir(thisDir)
else:
client.mkdir(thisDir)
except Exception as e:
logging.error(str(e))
sys.exit(getattr(e, 'errno', -1))
sys.exit(0)
def vsync():
global_md5_cache = None
def signal_handler(h_stream, h_frame):
logging.debug("{} {}".format(h_stream, h_frame))
logging.critical("Interrupt\n")
sys.exit(-1)
# handle interrupts nicely
signal.signal(signal.SIGINT, signal_handler)
start_time = time.time()
parser = CommonParser(description=DESCRIPTION)
parser.add_option('files', nargs='+', help='Files to copy to VOSpace')
parser.add_option('destination', help='VOSpace location to sync files to')
parser.add_option('--ignore-checksum', action="store_true",
help='dont check MD5 sum, use size and time instead')
parser.add_option('--cache_nodes', action='store_true',
help='cache node MD5 sum in an sqllite db')
parser.add_option('--cache_filename',
help="Name of file to use for node cache",
default="{}/.config/vos/node_cache.db".format(HOME))
parser.add_option('--recursive', '-r', help="Do a recursive sync",
action="store_true")
parser.add_option('--nstreams', '-n', type=int,
help="Number of streams to run (MAX: 30)", default=5)
parser.add_option(
'--exclude',
help="ignore directories or files containing this pattern",
default=None)
parser.add_option('--include',
help="only include files matching this pattern",
default=None)
parser.add_option(
'--overwrite',
help=("overwrite copy on server regardless of modification/size/md5 "
"checks"),
action="store_true")
opt = parser.parse_args()
set_logging_level_from_args(opt)
if opt.nstreams > 30:
parser.error("Maximum of 30 streams exceeded")
if opt.cache_nodes:
global_md5_cache = md5_cache.MD5Cache(cache_db=opt.cache_filename)
destination = opt.destination
if destination[0:4] != "vos:":
parser.error("Only allows sync FROM local copy TO VOSpace")
# Currently we don't create nodes in sync and we don't sync onto files
logging.info("Connecting to VOSpace")
client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token)
logging.info("Confirming Destination is a directory")
dest_is_dir = client.isdir(destination)
queue = JoinableQueue(maxsize=10 * opt.nstreams)
good_dirs = []
node_dict = {}
def compute_md5(this_filename, block_size=None):
"""
Read through a file and compute that files MD5 checksum.
:param this_filename: name of the file on disk
:param block_size: number of bytes to read into memory,
defaults to 2**19 bytes
:return: md5 as a hexadecimal string
"""
block_size = block_size is None and 2 ** 19 or block_size
return md5_cache.MD5Cache.compute_md5(this_filename,
block_size=block_size)
def file_md5(this_filename):
import os
md5 = None
if global_md5_cache is not None:
md5 = global_md5_cache.get(this_filename)
if md5 is None or md5[2] < os.stat(this_filename).st_mtime:
md5 = compute_md5(this_filename)
if global_md5_cache is not None:
stat = os.stat(this_filename)
global_md5_cache.update(this_filename, md5, stat.st_size,
stat.st_mtime)
else:
md5 = md5[0]
return md5
class ThreadCopy(Process):
def __init__(self, this_queue, this_client):
super(ThreadCopy, self).__init__()
self.client = this_client
self.queue = this_queue
self.filesSent = 0
self.filesSkipped = 0
self.bytesSent = 0
self.bytesSkipped = 0
self.filesErrored = 0
def run(self):
while True:
(current_source, current_destination) = self.queue.get()
requeue = (current_source, current_destination)
src_md5 = None
stat = os.stat(current_source)
if not opt.ignore_checksum and not opt.overwrite:
src_md5 = file_md5(current_source)
if not opt.overwrite:
# Check if the file is the same
try:
node_info = None
if opt.cache_nodes:
node_info = global_md5_cache.get(
current_destination)
if node_info is None:
logging.debug("Getting node info from VOSpace")
logging.debug(str(node_dict.keys()))
logging.debug(str(current_destination))
node = self.client.get_node(current_destination,
limit=None)
current_destination_md5 = node.props.get(
'MD5', 'd41d8cd98f00b204e9800998ecf8427e')
current_destination_length = node.attr['st_size']
current_destination_time = node.attr['st_ctime']
if opt.cache_nodes:
global_md5_cache.update(
current_destination,
current_destination_md5,
current_destination_length,
current_destination_time)
else:
current_destination_md5 = node_info[0]
current_destination_length = node_info[1]
current_destination_time = node_info[2]
logging.debug("Destination MD5: {}".format(
current_destination_md5))
if ((not opt.ignore_checksum and src_md5 ==
current_destination_md5) or
(opt.ignore_checksum and
current_destination_time >= stat.st_mtime and
current_destination_length == stat.st_size)):
logging.info("skipping: %s matches %s" % (
current_source, current_destination))
self.filesSkipped += 1
self.bytesSkipped += current_destination_length
self.queue.task_done()
continue
except (transfer_exceptions.AlreadyExistsException,
transfer_exceptions.NotFoundException):
pass
logging.info(
"%s -> %s" % (current_source, current_destination))
try:
self.client.copy(current_source, current_destination,
send_md5=True)
node = self.client.get_node(current_destination,
limit=None)
current_destination_md5 = node.props.get(
'MD5', 'd41d8cd98f00b204e9800998ecf8427e')
current_destination_length = node.attr['st_size']
current_destination_time = node.attr['st_ctime']
if opt.cache_nodes:
global_md5_cache.update(current_destination,
current_destination_md5,
current_destination_length,
current_destination_time)
self.filesSent += 1
self.bytesSent += stat.st_size
except (IOError, OSError) as exc:
logging.error(
"Error writing {} to server, skipping".format(
current_source))
logging.error(str(exc))
import re
if re.search('NodeLocked', str(exc)) is not None:
logging.error(
("Use vlock to unlock the node before syncing "
"to {}").format(current_destination))
try:
if exc.errno == 104:
self.queue.put(requeue)
except Exception as e2:
logging.error("Error during requeue")
logging.error(str(e2))
pass
self.filesErrored += 1
pass
self.queue.task_done()
def mkdirs(directory):
"""Recursively make all nodes in the path to directory.
:param directory: str, vospace location of ContainerNode (directory)
to make
:return:
"""
logging.debug("%s %s" % (directory, str(good_dirs)))
# if we've seen this before skip it.
if directory in good_dirs:
return
# try and make a new directory and return
# failure indicates we should see if subdirectories exist
try:
client.mkdir(directory)
logging.info("Made directory {}".format(directory))
good_dirs.append(directory)
return
except transfer_exceptions.AlreadyExistsException:
pass
# OK, must already have existed, add to list
good_dirs.append(directory)
return
def copy(current_source, current_destination):
"""
Copy current_source from local file system to current_destination.
:param current_source: name of local file
:param current_destination: name of localtion on VOSpace to copy file
to (includes filename part)
:return: None
"""
# strip down current_destination until we find a part that exists
# and then build up the path.
if os.path.islink(current_source):
logging.error("{} is a link, skipping".format(current_source))
return
if not os.access(current_source, os.R_OK):
logging.error(
"Failed to open file {}, skipping".format(current_source))
return
import re
if re.match(r'^[A-Za-z0-9._\-();:&*$@!+=/]*$', current_source) is None:
logging.error(
"filename %s contains illegal characters, skipping" %
current_source)
return
dirname = os.path.dirname(current_destination)
mkdirs(dirname)
if opt.include is not None and not re.search(opt.include,
current_source):
return
queue.put((current_source, current_destination), timeout=3600)
def start_streams(no_streams, vospace_client):
list_of_streams = []
for i in range(no_streams):
logging.info("Launching VOSpace connection stream %d" % i)
t = ThreadCopy(queue, this_client=vospace_client)
t.daemon = True
t.start()
list_of_streams.append(t)
return list_of_streams
def build_file_list(base_path, destination_root='', recursive=False,
ignore=None):
"""Build a list of files that should be copied into VOSpace"""
spinner = ['-', '\\', '|', '/', '-', '\\', '|', '/']
count = 0
for (root, dirs, filenames) in os.walk(base_path):
for this_dirname in dirs:
if not recursive:
continue
this_dirname = os.path.join(root, this_dirname)
skip = False
if ignore is not None:
for thisIgnore in ignore.split(','):
if not this_dirname.find(thisIgnore) < 0:
logging.info("excluding: %s " % this_dirname)
skip = True
continue
if skip:
continue
cprefix = os.path.commonprefix((base_path, this_dirname))
this_dirname = os.path.normpath(
destination_root + "/" + this_dirname[len(cprefix):])
mkdirs(this_dirname)
for thisfilename in filenames:
srcfilename = os.path.normpath(
os.path.join(root, thisfilename))
skip = False
if ignore is not None:
for thisIgnore in ignore.split(','):
if not srcfilename.find(thisIgnore) < 0:
logging.info("excluding: %s " % srcfilename)
skip = True
continue
if skip:
continue
cprefix = os.path.commonprefix((base_path, srcfilename))
destfilename = os.path.normpath(
destination_root + "/" + srcfilename[len(cprefix):])
this_dirname = os.path.dirname(destfilename)
mkdirs(this_dirname)
count += 1
if opt.verbose:
sys.stderr.write(
"Building list of files to transfer %s\r" % (
spinner[count % len(spinner)]))
copy(srcfilename, destfilename)
if not recursive:
return
return
streams = start_streams(opt.nstreams, vospace_client=client)
# build a complete file list given all the things on the command line
for filename in opt.files:
filename = os.path.abspath(filename)
this_root = destination
if os.path.isdir(filename):
if filename[-1] != "/":
if os.path.basename(filename) != os.path.basename(destination):
this_root = os.path.join(destination,
os.path.basename(filename))
mkdirs(this_root)
node_dict[this_root] = client.get_node(this_root, limit=None)
try:
build_file_list(filename, destination_root=this_root,
recursive=opt.recursive, ignore=opt.exclude)
except Exception as e:
logging.error(str(e))
logging.error("ignoring error")
elif os.path.isfile(filename):
if dest_is_dir:
this_root = os.path.join(destination,
os.path.basename(filename))
copy(filename, this_root)
else:
logging.error("%s: No such file or directory." % filename)
logging.info(
("Waiting for transfers to complete "
r"******** CTRL-\ to interrupt ********"))
queue.join()
end_time = time.time()
bytes_sent = 0
files_sent = 0
bytes_skipped = 0
files_skipped = 0
files_erred = 0
for stream in streams:
bytes_sent += stream.bytesSent
bytes_skipped += stream.bytesSkipped
files_sent += stream.filesSent
files_skipped += stream.filesSkipped
files_erred += stream.filesErrored
logging.info("==== TRANSFER REPORT ====")
if bytes_sent > 0:
rate = bytes_sent / (end_time - start_time) / 1024.0
logging.info("Sent %d files (%8.1f kbytes @ %8.3f kBytes/s)" % (
files_sent, bytes_sent / 1024.0, rate))
speed_up = (bytes_skipped + bytes_sent) / bytes_sent
logging.info(
"Speedup: %f (skipped %d files)" % (speed_up, files_skipped))
if bytes_sent == 0:
logging.info("No files needed sending ")
if files_erred > 0:
logging.info(
"Error transferring %d files, please try again" % files_erred)
def mountvofs():
parser = CommonParser(description='mount vospace as a filesystem.')
# mountvofs specific options
parser.add_option("--vospace", help="the VOSpace to mount", default="vos:")
parser.add_option("--mountpoint",
help="the mountpoint on the local filesystem",
default="/tmp/vospace")
parser.add_option(
"-f",
"--foreground",
action="store_true",
help="Mount the filesystem as a foreground opperation and " +
"produce copious amounts of debuging information")
parser.add_option("--log",
action="store",
help="File to store debug log to",
default="/tmp/vos.err")
parser.add_option(
"--cache_limit",
action="store",
type=int,
help=
"upper limit on local diskspace to use for file caching (in MBytes)",
default=50 * 2**(10 + 10 + 10))
parser.add_option("--cache_dir",
action="store",
help="local directory to use for file caching",
default=None)
parser.add_option("--readonly",
action="store_true",
help="mount vofs readonly",
default=False)
parser.add_option(
"--cache_nodes",
action="store_true",
default=False,
help="cache dataNode properties, containerNodes are not cached")
parser.add_option("--allow_other",
action="store_true",
default=False,
help="Allow all users access to this mountpoint")
parser.add_option("--max_flush_threads",
action="store",
type=int,
help="upper limit on number of flush (upload) threads",
default=10)
parser.add_option(
"--secure_get",
action="store_true",
default=False,
help="Ensure HTTPS instead of HTTP is used to retrieve data (slower)")
parser.add_option(
"--nothreads",
help="Only run in a single thread, causes some blocking.",
action="store_true")
(opt, args) = parser.parse_args()
parser.process_informational_options()
log_format = ("%(asctime)s %(thread)d vos-" + str(version) +
" %(module)s.%(funcName)s.%(lineno)d %(message)s")
lf = logging.Formatter(fmt=log_format)
fh = logging.FileHandler(filename=os.path.abspath('/tmp/vos.exceptions'))
fh.formatter = lf
# send the 'logException' statements to a seperate log file.
logger = logging.getLogger('exceptions')
logger.handlers = []
logger.setLevel(logging.ERROR)
logger.addHandler(fh)
fh = logging.FileHandler(filename=os.path.abspath(opt.log))
fh.formatter = lf
logger = logging.getLogger('vofs')
logger.handlers = []
logger.setLevel(parser.log_level)
logger.addHandler(fh)
vos_logger = logging.getLogger('vos')
vos_logger.handlers = []
vos_logger.setLevel(logging.ERROR)
vos_logger.addHandler(fh)
logger.debug("Checking connection to VOSpace ")
if not os.access(opt.certfile, os.F_OK):
# setting this to 'blank' instead of None since 'None' implies use the default.
certfile = ""
else:
certfile = os.path.abspath(opt.certfile)
conn = vos.Connection(vospace_certfile=certfile, vospace_token=opt.token)
if opt.token:
readonly = opt.readonly
logger.debug("Got a token, connections should work")
elif conn.ws_client.subject.anon:
readonly = True
logger.debug("No certificate/token, anonymous connections should work")
else:
readonly = opt.readonly
logger.debug("Got authentication, connections should work")
root = opt.vospace
mount = os.path.abspath(opt.mountpoint)
if opt.cache_dir is None:
opt.cache_dir = os.path.normpath(
os.path.join(os.getenv('HOME', '.'), root.replace(":", "_")))
if not os.access(mount, os.F_OK):
os.makedirs(mount)
if platform == "darwin":
fuse = MyFuse(VOFS(root,
opt.cache_dir,
opt,
conn=conn,
cache_limit=opt.cache_limit,
cache_nodes=opt.cache_nodes,
cache_max_flush_threads=opt.max_flush_threads,
secure_get=opt.secure_get),
mount,
fsname=root,
volname=root,
nothreads=opt.nothreads,
defer_permissions=True,
daemon_timeout=DAEMON_TIMEOUT,
readonly=readonly,
user_allow_other=opt.allow_other,
noapplexattr=True,
noappledouble=True,
debug=opt.foreground,
foreground=opt.foreground)
else:
fuse = MyFuse(VOFS(root,
opt.cache_dir,
opt,
conn=conn,
cache_limit=opt.cache_limit,
cache_nodes=opt.cache_nodes,
cache_max_flush_threads=opt.max_flush_threads,
secure_get=opt.secure_get),
mount,
fsname=root,
nothreads=opt.nothreads,
readonly=readonly,
user_allow_other=opt.allow_other,
foreground=opt.foreground)
def vtag():
usage = """
vtag [options] node [key[=value] [key[=value] ...]]
Version: %s """ % (version.version)
signal.signal(signal.SIGINT, signal_handler)
parser = CommonParser(usage)
parser.add_option('--remove',
action="store_true",
help='remove the listed property')
# Not yet supported
# parser.add_option("-R", "--recursive", action='store_const', const=True,
# help="Recursive set read/write properties")
(opt, args) = parser.parse_args()
parser.process_informational_options()
if len(args) == 0:
parser.error("no vospace supplied")
logger = logging.getLogger()
logger.setLevel(parser.log_level)
logger.addHandler(logging.StreamHandler())
## the node should be the first argument, the rest should contain the key/val pairs
node = args.pop(0)
props = []
if opt.remove:
## remove signified by blank value in key=value listing
for prop in args:
if '=' not in prop:
prop += "="
props.append(prop)
else:
props = args
try:
"""Lists/sets values of node properties based on context (length of props).
node: vos.vos.Node object
props: dictionary of properties to set. If dict is zero length, list them all.
"""
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
node = client.get_node(node)
if len(props) == 0:
# print all properties
for key in node.props:
print(key, node.props[key])
return pprint.pprint(node.props)
changed = False
for prop in props:
prop = prop.split('=')
if len(prop) == 1:
# get one property
logger.debug("just print this one out %s" % (prop[0]))
pprint.pprint(node.props.get(prop[0], None))
elif len(prop) == 2:
# update properties
key, value = prop
logger.debug("%s %s" % (len(key), len(value)))
if len(value) == 0:
value = None
logger.debug("%s: %s -> %s" %
(key, node.props.get(key, None), value))
if value != node.props.get(key, None):
node.props[key] = value
changed = True
else:
raise ValueError(
"Illigal keyword of value character ('=') used: %s" %
('='.join(prop)))
if changed:
client.add_props(node)
return 0
except KeyboardInterrupt:
logger.debug("Received keyboard interrupt. Execution aborted...\n")
pass
except Exception as e:
logger.error(str(e))
sys.exit(-1)
def mountvofs():
parser = CommonParser(description='mount vospace as a filesystem.')
# mountvofs specific options
parser.add_option("--vospace", help="the VOSpace to mount", default="vos:")
parser.add_option("--mountpoint",
help="the mountpoint on the local filesystem",
default="/tmp/vospace")
parser.add_option(
"-f", "--foreground", action="store_true",
help="Mount the filesystem as a foreground opperation and " +
"produce copious amounts of debuging information")
parser.add_option("--log", action="store",
help="File to store debug log to",
default="/tmp/vos.err")
parser.add_option(
"--cache_limit", action="store", type=int,
help=("upper limit on local diskspace to "
"use for file caching (in MBytes)"),
default=50 * 2 ** (10 + 10 + 10))
parser.add_option("--cache_dir", action="store",
help="local directory to use for file caching",
default=None)
parser.add_option("--readonly", action="store_true",
help="mount vofs readonly", default=False)
parser.add_option(
"--cache_nodes", action="store_true", default=False,
help="cache dataNode properties, containerNodes are not cached")
parser.add_option("--allow_other", action="store_true", default=False,
help="Allow all users access to this mountpoint")
parser.add_option("--max_flush_threads", action="store", type=int,
help="upper limit on number of flush (upload) threads",
default=10)
parser.add_option(
"--secure_get", action="store_true", default=False,
help="Ensure HTTPS instead of HTTP is used to retrieve data (slower)")
parser.add_option(
"--nothreads",
help="Only run in a single thread, causes some blocking.",
action="store_true")
opt = parser.parse_args()
set_logging_level_from_args(opt)
log_format = ("%(asctime)s %(thread)d vos-" + str(version) +
" %(module)s.%(funcName)s.%(lineno)d %(message)s")
username = getpass.getuser() # not to be used for access control
lf = logging.Formatter(fmt=log_format)
fh = logging.FileHandler(
filename=os.path.abspath('/tmp/vos.{}.exceptions'.format(username)))
fh.formatter = lf
# send the 'logException' statements to a seperate log file.
logger = logging.getLogger('exceptions')
logger.handlers = []
logger.setLevel(logging.ERROR)
logger.addHandler(fh)
fh = logging.FileHandler(filename=os.path.abspath(opt.log))
fh.formatter = lf
logger = logging.getLogger('vofs')
logger.handlers = []
logger.setLevel(opt.log_level)
logger.addHandler(fh)
vos_logger = logging.getLogger('vos')
vos_logger.handlers = []
vos_logger.setLevel(logging.ERROR)
vos_logger.addHandler(fh)
logger.debug("Checking connection to VOSpace ")
if not os.access(opt.certfile, os.F_OK):
# setting this to 'blank' instead of None since 'None' implies use
# the default.
certfile = ""
else:
certfile = os.path.abspath(opt.certfile)
conn = vos.Connection(vospace_certfile=certfile, vospace_token=opt.token)
if opt.token:
readonly = opt.readonly
logger.debug("Got a token, connections should work")
elif conn.ws_client.subject.anon:
readonly = True
logger.debug("No certificate/token, anonymous connections should work")
else:
readonly = opt.readonly
logger.debug("Got authentication, connections should work")
root = opt.vospace
mount = os.path.abspath(opt.mountpoint)
if opt.cache_dir is None:
opt.cache_dir = os.path.normpath(os.path.join(
os.getenv('HOME', '.'), root.replace(":", "_")))
if not os.access(mount, os.F_OK):
os.makedirs(mount)
try:
if platform == "darwin":
MyFuse(VOFS(root, opt.cache_dir, opt, conn=conn,
cache_limit=opt.cache_limit,
cache_nodes=opt.cache_nodes,
cache_max_flush_threads=opt.max_flush_threads,
secure_get=opt.secure_get),
mount,
fsname=root,
volname=root,
nothreads=opt.nothreads,
defer_permissions=True,
daemon_timeout=DAEMON_TIMEOUT,
readonly=readonly,
user_allow_other=opt.allow_other,
noapplexattr=True,
noappledouble=True,
debug=opt.foreground,
foreground=opt.foreground)
else:
MyFuse(VOFS(root, opt.cache_dir, opt, conn=conn,
cache_limit=opt.cache_limit,
cache_nodes=opt.cache_nodes,
cache_max_flush_threads=opt.max_flush_threads,
secure_get=opt.secure_get),
mount,
fsname=root,
nothreads=opt.nothreads,
readonly=readonly,
user_allow_other=opt.allow_other,
foreground=opt.foreground)
except Exception as ex:
logger.error(str(ex))
def vsync():
def signal_handler(signal, frame):
logger.critical("Interupt\n")
sys.exit(-1)
usage = """
vsync [options] files vos:Destination/
Version: %s """ % (version.version)
# handle interupts nicely
signal.signal(signal.SIGINT, signal_handler)
startTime = time.time()
parser = CommonParser(usage)
parser.add_option('--ignore-checksum',
action="store_true",
help='dont check MD5 sum, use size and time instead')
parser.add_option('--cache_nodes',
action='store_true',
help='cache node MD5 sum in an sqllite db')
parser.add_option('--recursive',
'-r',
help="Do a recursive sync",
action="store_true")
parser.add_option('--nstreams',
'-n',
type=int,
help="Number of streams to run (MAX: 30)",
default=1)
parser.add_option(
'--exclude',
help="ignore directories or files containing this pattern",
default=None)
parser.add_option('--include',
help="only include files matching this pattern",
default=None)
parser.add_option(
'--overwrite',
help=
"overwrite copy on server regardless of modification/size/md5 checks",
action="store_true")
parser.add_option(
'--load_test',
action="store_true",
help=
"Used to stress test the VOServer, also set --nstreams to a large value"
)
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt, args) = parser.parse_args()
parser.process_informational_options()
logger = logging.getLogger()
logger.setLevel(parser.log_level)
logger.addHandler(logging.StreamHandler())
if len(args) < 2:
parser.error(
"requires one or more source files and a single destination directory"
)
if opt.nstreams > 30 and not opt.load_test:
parser.error("Maximum of 30 streams exceeded")
if opt.cache_nodes:
from vos import md5_cache
md5Cache = md5_cache.MD5_Cache()
dest = args.pop()
if dest[0:4] != "vos:":
parser.error("Only allows sync FROM local copy TO VOSpace")
## Currently we don't create nodes in sync and we don't sync onto files
logger.info("Connecting to VOSpace")
client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token)
logger.info("Confirming Destination is a directory")
destIsDir = client.isdir(dest)
queue = JoinableQueue(maxsize=10 * opt.nstreams)
goodDirs = []
nodeDict = {}
def computeMD5(filename, block_size=None):
"""
Read through a file and compute that files MD5 checksum.
:param filename: name of the file on disk
:param block_size: number of bytes to read into memory, defaults to 2**19 bytes
:return: md5 as a hexadecimal string
"""
block_size = block_size is None and 2**19 or block_size
md5 = hashlib.md5()
r = open(filename, 'r')
while True:
buf = r.read(block_size)
if len(buf) == 0:
break
md5.update(buf)
r.close()
return md5.hexdigest()
def fileMD5(filename):
import os
md5 = None
if opt.cache_nodes:
md5 = md5Cache.get(filename)
if md5 is None or md5[2] < os.stat(filename).st_mtime:
md5 = computeMD5(filename)
if opt.cache_nodes:
stat = os.stat(filename)
md5Cache.update(filename, md5, stat.st_size, stat.st_mtime)
else:
md5 = md5[0]
return md5
class ThreadCopy(Process):
def __init__(self, queue, client):
super(ThreadCopy, self).__init__()
self.client = client
self.queue = queue
self.filesSent = 0
self.filesSkipped = 0
self.bytesSent = 0
self.bytesSkipped = 0
self.filesErrored = 0
def run(self):
while True:
(src, dest) = self.queue.get()
requeue = (src, dest)
srcMD5 = None
stat = os.stat(src)
if not opt.ignore_checksum and not opt.overwrite:
srcMD5 = fileMD5(src)
if not opt.overwrite:
# Check if the file is the same
try:
nodeInfo = None
if opt.cache_nodes:
nodeInfo = md5Cache.get(dest)
if nodeInfo is None:
logger.debug("Getting node info from VOSpace")
logger.debug(str(nodeDict.keys()))
logger.debug(str(dest))
node = self.client.get_node(dest, limit=None)
destMD5 = node.props.get(
'MD5', 'd41d8cd98f00b204e9800998ecf8427e')
destLength = node.attr['st_size']
destTime = node.attr['st_ctime']
if opt.cache_nodes:
md5Cache.update(dest, destMD5, destLength,
destTime)
else:
destMD5 = nodeInfo[0]
destLength = nodeInfo[1]
destTime = nodeInfo[2]
logger.debug("Dest MD5: %s " % (destMD5))
if (not opt.ignore_checksum and srcMD5 == destMD5) or (
opt.ignore_checksum
and destTime >= stat.st_mtime
and destLength == stat.st_size):
logger.info("skipping: %s matches %s" %
(src, dest))
self.filesSkipped += 1
self.bytesSkipped += destLength
self.queue.task_done()
continue
except (IOError, OSError) as node_error:
"""Ignore the erorr"""
logger.debug(str(node_error))
pass
logger.info("%s -> %s" % (src, dest))
try:
self.client.copy(src, dest, send_md5=True)
node = self.client.get_node(dest, limit=None)
destMD5 = node.props.get(
'MD5', 'd41d8cd98f00b204e9800998ecf8427e')
destLength = node.attr['st_size']
destTime = node.attr['st_ctime']
if opt.cache_nodes:
md5Cache.update(dest, destMD5, destLength, destTime)
self.filesSent += 1
self.bytesSent += stat.st_size
except (IOError, OSError) as e:
logger.error("Error writing %s to server, skipping" %
(src))
logger.error(str(e))
import re
if re.search('NodeLocked', str(e)) != None:
logger.error(
"Use vlock to unlock the node before syncing to %s."
% (dest))
try:
if e.errno == 104:
self.queue.put(requeue)
except Exception as e2:
logger.error("Error during requeue")
logger.error(str(e2))
pass
self.filesErrored += 1
pass
self.queue.task_done()
def mkdirs(dirs):
logger.debug("%s %s" % (dirs, str(goodDirs)))
## if we've seen this before skip it.
if dirs in goodDirs:
return
## try and make a new directory and return
## failure indicates we should see if subdirs exist
try:
client.mkdir(dirs)
logger.info("Made directory %s " % (dirs))
goodDirs.append(dirs)
return
except OSError as e:
exit_code = getattr(e, 'errno', -1)
if exit_code != errno.EEXIST:
raise e
## OK, must already have existed, add to list
goodDirs.append(dirs)
return
def copy(source, dest):
## strip down dest until we find a part that exists
## and then build up the path. Dest should include the filename
if os.path.islink(source):
logger.error("%s is a link, skipping" % (source))
return
if not os.access(source, os.R_OK):
logger.error("Failed to open file %s, skipping" % (source))
return
import re
if re.match('^[A-Za-z0-9\\._\\-\\(\\);:&\\*\\$@!+=\\/]*$',
source) is None:
logger.error("filename %s contains illegal characters, skipping" %
(source))
return
dirname = os.path.dirname(dest)
mkdirs(dirname)
if opt.include is not None and not re.search(opt.include, source):
return
queue.put((source, dest), timeout=3600)
def startStreams(nstreams, vospace_client):
streams = []
for i in range(nstreams):
logger.info("Launching vospace connection stream %d" % (i))
t = ThreadCopy(queue, client=vospace_client)
t.daemon = True
t.start()
streams.append(t)
return streams
def buildFileList(basePath, destRoot='', recursive=False, ignore=None):
"""Build a list of files that should be copied into VOSpace"""
import string
spinner = ['-', '\\', '|', '/', '-', '\\', '|', '/']
count = 0
import re
for (root, dirs, filenames) in os.walk(basePath):
for thisDirname in dirs:
if not recursive:
continue
thisDirname = os.path.join(root, thisDirname)
skip = False
if ignore is not None:
for thisIgnore in ignore.split(','):
if not thisDirname.find(thisIgnore) < 0:
logger.info("excluding: %s " % (thisDirname))
skip = True
continue
if skip:
continue
cprefix = os.path.commonprefix((basePath, thisDirname))
thisDirname = os.path.normpath(destRoot + "/" +
thisDirname[len(cprefix):])
mkdirs(thisDirname)
for thisfilename in filenames:
srcfilename = os.path.normpath(os.path.join(
root, thisfilename))
skip = False
if ignore is not None:
for thisIgnore in ignore.split(','):
if not srcfilename.find(thisIgnore) < 0:
logger.info("excluding: %s " % (srcfilename))
skip = True
continue
if skip:
continue
cprefix = os.path.commonprefix((basePath, srcfilename))
destfilename = os.path.normpath(destRoot + "/" +
srcfilename[len(cprefix):])
thisDirname = os.path.dirname(destfilename)
mkdirs(thisDirname)
count += 1
if opt.verbose:
sys.stderr.write(
"Building list of files to transfer %s\r" %
(spinner[count % len(spinner)]))
copy(srcfilename, destfilename)
if not recursive:
return
return
streams = startStreams(opt.nstreams, vospace_client=client)
### build a complete file list given all the things on the command line
for filename in args:
filename = os.path.abspath(filename)
thisRoot = dest
if os.path.isdir(filename):
if filename[-1] != "/":
if os.path.basename(filename) != os.path.basename(dest):
thisRoot = os.path.join(dest, os.path.basename(filename))
mkdirs(thisRoot)
nodeDict[thisRoot] = client.get_node(thisRoot, limit=None)
try:
buildFileList(filename,
destRoot=thisRoot,
recursive=opt.recursive,
ignore=opt.exclude)
except Exception as e:
logger.error(str(e))
logger.error("ignoring error")
elif os.path.isfile(filename):
if destIsDir:
thisRoot = os.path.join(dest, os.path.basename(filename))
copy(filename, thisRoot)
else:
logger.error("%s: No such file or directory." % (filename))
logger.info("\nWaiting for transfers to complete.\nCTRL-\ to interrupt\n")
queue.join()
endTime = time.time()
bytesSent = 0
filesSent = 0
bytesSkipped = 0
filesSkipped = 0
filesErrored = 0
for stream in streams:
bytesSent += stream.bytesSent
bytesSkipped += stream.bytesSkipped
filesSent += stream.filesSent
filesSkipped += stream.filesSkipped
filesErrored += stream.filesErrored
logger.info("\n\n==== TRANSFER REPORT ====\n\n")
if bytesSent > 0:
rate = bytesSent / (endTime - startTime) / 1024.0
logger.info("Sent %d files (%8.1f kbytes @ %8.3f kBytes/s)" %
(filesSent, bytesSent / 1024.0, rate))
speedUp = (bytesSkipped + bytesSent) / bytesSent
logger.info("Speedup: %f (skipped %d files)" %
(speedUp, filesSkipped))
if bytesSent == 0:
logger.info("No files needed sending ")
if filesErrored > 0:
logger.info("Error transferring %d files, please try again" %
(filesErrored))
def vchmod():
signal.signal(signal.SIGINT, signal_handler)
usage = """vchmod mode node [read/write group names in quotes - maximum of 4 each]
Accepted modes: (og|go|o|g)[+-=](rw|wr|r\w)
Sets read/write properties of a node.
Version: %s """ % version.version
parser = CommonParser(usage)
parser.add_option("-R", "--recursive", action='store_const', const=True,
help="Recursive set read/write properties")
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt, args) = parser.parse_args()
parser.process_informational_options()
logger = logging.getLogger()
if len(args) < 2:
parser.error("Requires mode and node arguments")
cmdArgs = dict(zip(['mode', 'node'], args[0:2]))
groupNames = args[2:]
import re
mode = re.match(r"(?P<who>^og|^go|^o|^g)(?P<op>[\+\-=])(?P<what>rw$|wr$|r$|w$)", cmdArgs['mode'])
if not mode:
parser.print_help()
logger.error("\n\nAccepted modes: (og|go|o|g)[+-=](rw|wr|r\w)\n\n")
sys.exit(-1)
props = {}
if 'o' in mode.group('who'):
if not mode.group('what') == 'r': # read only
parser.print_help()
logger.error("\n\nPublic access is readonly, no public write available \n\n")
sys.exit(-1)
if mode.group('op') == '-':
props['ispublic'] = 'false'
else:
props['ispublic'] = 'true'
if 'g' in mode.group('who'):
if '-' == mode.group('op'):
if not len(groupNames) == 0:
parser.print_help()
logger.error("\n\nNames of groups not required with remove permission\n\n")
sys.exit(-1)
if 'r' in mode.group('what'):
props['readgroup'] = None
if "w" in mode.group('what'):
props['writegroup'] = None
else:
if not len(groupNames) == len(mode.group('what')):
parser.print_help()
logger.error("\n\n%s group names required for %s\n\n" %
(len(mode.group('what')), mode.group('what')))
sys.exit(-1)
if mode.group('what').find('r') > -1:
# remove duplicate whitespaces
rgroups = " ".join(groupNames[mode.group('what').find('r')].split())
props['readgroup'] = (vos.CADC_GMS_PREFIX +
rgroups.replace(" ", " " + vos.CADC_GMS_PREFIX))
if mode.group('what').find('w') > -1:
wgroups = " ".join(groupNames[mode.group('what').find('w')].split())
props['writegroup'] = (vos.CADC_GMS_PREFIX +
wgroups.replace(" ", " " + vos.CADC_GMS_PREFIX))
logger.debug("Properties: %s" % (str(props)))
logger.debug("Node: %s" % cmdArgs['node'])
returnCode = 0
try:
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
node = client.get_node(cmdArgs['node'])
if opt.recursive:
node.props.clear()
node.clear_properties()
# del node.node.findall(vos.Node.PROPERTIES)[0:]
if 'readgroup' in props:
node.chrgrp(props['readgroup'])
if 'writegroup' in props:
node.chwgrp(props['writegroup'])
if 'ispublic' in props:
node.set_public(props['ispublic'])
logger.debug("Node: {0}".format(node))
returnCode = client.update(node, opt.recursive)
except KeyboardInterrupt as ke:
logger.error("Received keyboard interrupt. Execution aborted...\n")
sys.exit(getattr(ke, 'errno', -1))
except Exception as e:
logger.error('Error {}: '.format(str(e)))
sys.exit(getattr(e, 'errno', -1))
sys.exit(returnCode)
def vls():
parser = CommonParser(usage, description=description, add_help_option=False)
parser.add_option("--help", action="store_true")
parser.add_option("-l", "--long", action="store_true", help="verbose listing sorted by name")
parser.add_option("-g", "--group", action="store_true", help="display group read/write information")
parser.add_option("-h", "--human", action="store_true", help="make sizes human readable", default=False)
parser.add_option("-S", "--Size", action="store_true", help="sort files by size", default=False)
parser.add_option("-r", "--reverse", action="store_true", help="reverse the sort order", default=False)
parser.add_option("-t", "--time", action="store_true", help="sort by time copied to VOSpace")
(opt, args) = parser.parse_args()
# We disabled -h/--help so vls can have a -h option.
# Here we re-enable it but only with --help
if opt.help:
parser.print_help()
sys.exit(0)
parser.process_informational_options()
if not len(args) > 0:
parser.error("missing VOSpace argument")
logger = logging.getLogger()
logger.setLevel(parser.log_level)
sortKey = (opt.time and "date") or (opt.Size and "size") or False
try:
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
except Exception as e:
logger.error("Connection failed: %s" % (str(e)))
sys.exit(-1)
columns = ['permissions']
if opt.long:
columns.extend(['creator'])
columns.extend(['readGroup', 'writeGroup', 'isLocked', 'size', 'date'])
def get_terminal_size():
"""Get the size of the terminal
@return: tuple(int, int) giving the row/column of the terminal screen.
"""
def ioctl_gwinsz(fd):
"""
@param fd: A file descriptor that points at the Screen Parameters
@return: the termios screeen structure unpacked into an array.
"""
try:
import fcntl
import termios
import struct
this_cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
except:
return None
return this_cr
cr = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2)
if not cr:
try:
td = os.open(os.ctermid(), os.O_RDONLY)
cr = ioctl_gwinsz(td)
td.close()
except:
pass
if not cr:
try:
cr = (os.environ['LINES'], os.environ['COLUMNS'])
except:
cr = (25, 80)
return int(cr[1]), int(cr[0])
def size_format(size):
"""Format a size value for listing"""
try:
size = float(size)
except Exception as ex:
logger.debug(str(ex))
size = 0.0
if opt.human:
size_unit = ['B', 'K', 'M', 'G', 'T']
try:
length = float(size)
scale = int(math.log(length) / math.log(1024))
length = "%.0f%s" % (length / (1024.0 ** scale), size_unit[scale])
except:
length = str(int(size))
else:
length = str(int(size))
return "%12s " % length
def date_format(epoch):
"""given a time object return a unix-ls like formatted string"""
time_tuple = time.localtime(epoch)
if time.localtime().tm_year != time_tuple.tm_year:
return time.strftime('%b %d %Y ', time_tuple)
return time.strftime('%b %d %H:%S ', time_tuple)
formats = {'permissions': lambda value: "{:<11}".format(value),
'creator': lambda value: " {:<20}".format(value),
'readGroup': lambda value: " {:<15}".format(value.replace(CADC_GMS_PREFIX, "")),
'writeGroup': lambda value: " {:<15}".format(value.replace(CADC_GMS_PREFIX, "")),
'isLocked': lambda value: " {:<8}".format("", "LOCKED")[value == "true"],
'size': size_format,
'date': date_format}
for node in args:
try:
if not node[0:4] == "vos:" :
raise OSError(errno.EBADF, "Invalid node name", node)
logger.debug("getting listing of: %s" % str(node))
infoList = client.get_info_list(node)
except SSLError as error:
logger.error(str(error))
sys.exit(errno.EPERM)
except Exception as e:
logger.error(str(e))
err_no = getattr(e, 'errno', None)
err_no = err_no is not None and err_no or errno.ENOMSG
sys.exit(err_no)
if sortKey:
try:
sorted_list = sorted(infoList, key=lambda name: name[1][sortKey], reverse=not opt.reverse)
except:
sorted_list = infoList
finally:
infoList = sorted_list
for item in infoList:
name_string = item[0]
if opt.long or opt.group:
for col in columns:
value = item[1].get(col, None)
value = value is not None and value or ""
if col in formats:
sys.stdout.write(formats[col](value))
if item[1]["permissions"][0] == 'l':
name_string = "%s -> %s" % (name_string, item[1]['target'])
sys.stdout.write("%s\n" % name_string)
def vln():
usage = """
vln vos:VOSpaceSource vos:VOSpaceTarget
examples:
vln vos:vospace/junk.txt vos:vospace/linkToJunk.txt
vln vos:vospace/directory vos:vospace/linkToDirectory
vln http://external.data.source vos:vospace/linkToExternalDataSource
"""
parser = CommonParser(usage)
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt, args) = parser.parse_args()
parser.process_informational_options()
logger = logging.getLogger()
logger.setLevel(parser.log_level)
logger.addHandler(logging.StreamHandler())
if len(args) != 2:
parser.error("You must specify a source file and a target file")
sys.exit(-1)
if args[1][0:4] != "vos:":
parser.error("The target to source must be in vospace")
sys.exit(-1)
logger.debug("Connecting to vospace using certificate %s" % opt.certfile)
try:
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
except Exception as e:
logger.error("VOS connection failed: {0}".format(e))
sys.exit(-1)
try:
client.link(args[0], args[1])
except Exception as e:
logger.error("Failed to make link from %s to %s" % (args[0], args[1]))
logger.error(getattr(e, 'strerror', 'Unknown Error'))
sys.exit(-1)
def vsync():
global_md5_cache = None
def signal_handler(h_stream, h_frame):
logging.debug("{} {}".format(h_stream, h_frame))
logging.critical("Interrupt\n")
sys.exit(-1)
# handle interrupts nicely
signal.signal(signal.SIGINT, signal_handler)
start_time = time.time()
parser = CommonParser(description=DESCRIPTION)
parser.add_option('files', nargs='+', help='Files to copy to VOSpace')
parser.add_option('destination', help='VOSpace location to sync files to')
parser.add_option('--ignore-checksum', action="store_true",
help='dont check MD5 sum, use size and time instead')
parser.add_option('--cache_nodes', action='store_true',
help='cache node MD5 sum in an sqllite db')
parser.add_option('--cache_filename',
help="Name of file to use for node cache",
default="{}/.config/vos/node_cache.db".format(HOME))
parser.add_option('--recursive', '-r', help="Do a recursive sync",
action="store_true")
parser.add_option('--nstreams', '-n', type=int,
help="Number of streams to run (MAX: 30)", default=5)
parser.add_option(
'--exclude',
help="ignore directories or files containing this pattern",
default=None)
parser.add_option('--include',
help="only include files matching this pattern",
default=None)
parser.add_option(
'--overwrite',
help=("overwrite copy on server regardless of modification/size/md5 "
"checks"),
action="store_true")
opt = parser.parse_args()
set_logging_level_from_args(opt)
if opt.nstreams > 30:
parser.error("Maximum of 30 streams exceeded")
if opt.cache_nodes:
global_md5_cache = md5_cache.MD5Cache(cache_db=opt.cache_filename)
destination = opt.destination
if not vos.is_remote_file(destination):
parser.error("Only allows sync FROM local copy TO VOSpace")
# Currently we don't create nodes in sync and we don't sync onto files
logging.info("Connecting to VOSpace")
client = vos.Client(
vospace_certfile=opt.certfile, vospace_token=opt.token)
logging.info("Confirming Destination is a directory")
dest_is_dir = client.isdir(destination)
queue = JoinableQueue(maxsize=10 * opt.nstreams)
good_dirs = []
node_dict = {}
def compute_md5(this_filename, block_size=None):
"""
Read through a file and compute that files MD5 checksum.
:param this_filename: name of the file on disk
:param block_size: number of bytes to read into memory,
defaults to 2**19 bytes
:return: md5 as a hexadecimal string
"""
block_size = block_size is None and 2 ** 19 or block_size
return md5_cache.MD5Cache.compute_md5(this_filename,
block_size=block_size)
def file_md5(this_filename):
import os
md5 = None
if global_md5_cache is not None:
md5 = global_md5_cache.get(this_filename)
if md5 is None or md5[2] < os.stat(this_filename).st_mtime:
md5 = compute_md5(this_filename)
if global_md5_cache is not None:
stat = os.stat(this_filename)
global_md5_cache.update(this_filename, md5, stat.st_size,
stat.st_mtime)
else:
md5 = md5[0]
return md5
class ThreadCopy(Process):
def __init__(self, this_queue):
super(ThreadCopy, self).__init__()
self.client = vos.Client(
vospace_certfile=opt.certfile,
vospace_token=opt.token)
self.queue = this_queue
self.filesSent = 0
self.filesSkipped = 0
self.bytesSent = 0
self.bytesSkipped = 0
self.filesErrored = 0
def run(self):
while True:
(current_source, current_destination) = self.queue.get()
requeue = (current_source, current_destination)
src_md5 = None
stat = os.stat(current_source)
if not opt.ignore_checksum and not opt.overwrite:
src_md5 = file_md5(current_source)
if not opt.overwrite:
# Check if the file is the same
try:
node_info = None
if opt.cache_nodes:
node_info = global_md5_cache.get(
current_destination)
if node_info is None:
logging.debug("Getting node info from VOSpace")
logging.debug(str(node_dict.keys()))
logging.debug(str(current_destination))
node = self.client.get_node(current_destination,
limit=None)
current_destination_md5 = node.props.get(
'MD5', 'd41d8cd98f00b204e9800998ecf8427e')
current_destination_length = node.attr['st_size']
current_destination_time = node.attr['st_ctime']
if opt.cache_nodes:
global_md5_cache.update(
current_destination,
current_destination_md5,
current_destination_length,
current_destination_time)
else:
current_destination_md5 = node_info[0]
current_destination_length = node_info[1]
current_destination_time = node_info[2]
logging.debug("Destination MD5: {}".format(
current_destination_md5))
if ((not opt.ignore_checksum and src_md5 ==
current_destination_md5) or
(opt.ignore_checksum and
current_destination_time >= stat.st_mtime and
current_destination_length == stat.st_size)):
logging.info("skipping: %s matches %s" % (
current_source, current_destination))
self.filesSkipped += 1
self.bytesSkipped += current_destination_length
self.queue.task_done()
continue
except (transfer_exceptions.AlreadyExistsException,
transfer_exceptions.NotFoundException):
pass
logging.info(
"%s -> %s" % (current_source, current_destination))
try:
self.client.copy(current_source, current_destination,
send_md5=True)
node = self.client.get_node(current_destination,
limit=None)
current_destination_md5 = node.props.get(
'MD5', 'd41d8cd98f00b204e9800998ecf8427e')
current_destination_length = node.attr['st_size']
current_destination_time = node.attr['st_ctime']
if opt.cache_nodes:
global_md5_cache.update(current_destination,
current_destination_md5,
current_destination_length,
current_destination_time)
self.filesSent += 1
self.bytesSent += stat.st_size
except (IOError, OSError) as exc:
logging.error(
"Error writing {} to server, skipping".format(
current_source))
logging.error(str(exc))
import re
if re.search('NodeLocked', str(exc)) is not None:
logging.error(
("Use vlock to unlock the node before syncing "
"to {}").format(current_destination))
try:
if exc.errno == 104:
self.queue.put(requeue)
except Exception as e2:
logging.error("Error during requeue")
logging.error(str(e2))
pass
self.filesErrored += 1
pass
self.queue.task_done()
def mkdirs(directory):
"""Recursively make all nodes in the path to directory.
:param directory: str, vospace location of ContainerNode (directory)
to make
:return:
"""
logging.debug("%s %s" % (directory, str(good_dirs)))
# if we've seen this before skip it.
if directory in good_dirs:
return
# try and make a new directory and return
# failure indicates we should see if subdirectories exist
try:
client.mkdir(directory)
logging.info("Made directory {}".format(directory))
good_dirs.append(directory)
return
except transfer_exceptions.AlreadyExistsException:
pass
# OK, must already have existed, add to list
good_dirs.append(directory)
return
def copy(current_source, current_destination):
"""
Copy current_source from local file system to current_destination.
:param current_source: name of local file
:param current_destination: name of localtion on VOSpace to copy file
to (includes filename part)
:return: None
"""
# strip down current_destination until we find a part that exists
# and then build up the path.
if os.path.islink(current_source):
logging.error("{} is a link, skipping".format(current_source))
return
if not os.access(current_source, os.R_OK):
logging.error(
"Failed to open file {}, skipping".format(current_source))
return
import re
if re.match(r'^[A-Za-z0-9._\-();:&*$@!+=/]*$', current_source) is None:
logging.error(
"filename %s contains illegal characters, skipping" %
current_source)
return
dirname = os.path.dirname(current_destination)
mkdirs(dirname)
if opt.include is not None and not re.search(opt.include,
current_source):
return
queue.put((current_source, current_destination), timeout=3600)
def start_streams(no_streams):
list_of_streams = []
for i in range(no_streams):
logging.info("Launching VOSpace connection stream %d" % i)
t = ThreadCopy(queue)
t.daemon = True
t.start()
list_of_streams.append(t)
return list_of_streams
def build_file_list(base_path, destination_root='', recursive=False,
ignore=None):
"""Build a list of files that should be copied into VOSpace"""
spinner = ['-', '\\', '|', '/', '-', '\\', '|', '/']
count = 0
for (root, dirs, filenames) in os.walk(base_path):
for this_dirname in dirs:
if not recursive:
continue
this_dirname = os.path.join(root, this_dirname)
skip = False
if ignore is not None:
for thisIgnore in ignore.split(','):
if not this_dirname.find(thisIgnore) < 0:
logging.info("excluding: %s " % this_dirname)
skip = True
continue
if skip:
continue
cprefix = os.path.commonprefix((base_path, this_dirname))
this_dirname = os.path.normpath(
destination_root + "/" + this_dirname[len(cprefix):])
mkdirs(this_dirname)
for thisfilename in filenames:
srcfilename = os.path.normpath(
os.path.join(root, thisfilename))
skip = False
if ignore is not None:
for thisIgnore in ignore.split(','):
if not srcfilename.find(thisIgnore) < 0:
logging.info("excluding: %s " % srcfilename)
skip = True
continue
if skip:
continue
cprefix = os.path.commonprefix((base_path, srcfilename))
destfilename = os.path.normpath(
destination_root + "/" + srcfilename[len(cprefix):])
this_dirname = os.path.dirname(destfilename)
mkdirs(this_dirname)
count += 1
if opt.verbose:
sys.stderr.write(
"Building list of files to transfer %s\r" % (
spinner[count % len(spinner)]))
copy(srcfilename, destfilename)
if not recursive:
return
return
streams = start_streams(opt.nstreams)
# build a complete file list given all the things on the command line
for filename in opt.files:
filename = os.path.abspath(filename)
this_root = destination
if os.path.isdir(filename):
if filename[-1] != "/":
if os.path.basename(filename) != os.path.basename(destination):
this_root = os.path.join(destination,
os.path.basename(filename))
mkdirs(this_root)
node_dict[this_root] = client.get_node(this_root, limit=None)
try:
build_file_list(filename, destination_root=this_root,
recursive=opt.recursive, ignore=opt.exclude)
except Exception as e:
logging.error(str(e))
logging.error("ignoring error")
elif os.path.isfile(filename):
if dest_is_dir:
this_root = os.path.join(destination,
os.path.basename(filename))
copy(filename, this_root)
else:
logging.error("%s: No such file or directory." % filename)
logging.info(
("Waiting for transfers to complete "
r"******** CTRL-\ to interrupt ********"))
queue.join()
end_time = time.time()
bytes_sent = 0
files_sent = 0
bytes_skipped = 0
files_skipped = 0
files_erred = 0
for stream in streams:
bytes_sent += stream.bytesSent
bytes_skipped += stream.bytesSkipped
files_sent += stream.filesSent
files_skipped += stream.filesSkipped
files_erred += stream.filesErrored
logging.info("==== TRANSFER REPORT ====")
if bytes_sent > 0:
rate = bytes_sent / (end_time - start_time) / 1024.0
logging.info("Sent %d files (%8.1f kbytes @ %8.3f kBytes/s)" % (
files_sent, bytes_sent / 1024.0, rate))
speed_up = (bytes_skipped + bytes_sent) / bytes_sent
logging.info(
"Speedup: %f (skipped %d files)" % (speed_up, files_skipped))
if bytes_sent == 0:
logging.info("No files needed sending ")
if files_erred > 0:
logging.info(
"Error transferring %d files, please try again" % files_erred)
def vcp():
## handle interrupts nicely
signal.signal(signal.SIGINT, signal_handler)
parser = CommonParser(
"%prog filename vos:rootNode/destination",
description=("Copy a file or directory (always recursive) to a "
"VOSpace location. Try to be UNIX like. "))
parser.add_option("--exclude",
default=None,
help="exclude files that match pattern")
parser.add_option(
"--include",
default=None,
help="only include files that match pattern (overrides exclude)")
parser.add_option("-i",
"--interrogate",
action="store_true",
help="Ask before overwriting files")
parser.add_option(
"--overwrite",
action="store_true",
help=
"don't check destination MD5, just overwrite even if source matches destination"
)
parser.add_option(
"--quick",
action="store_true",
help=
"Use default CADC urls, for speed. Will fail if CADC changes data storage mechanism",
default=False)
parser.add_option(
"-L",
"--follow-links",
help=
"Should recursive copy follow symbolic links? Default is to not follow links.",
action="store_true",
default=False)
parser.add_option("--ignore",
action="store_true",
default=False,
help="ignore errors and continue with recursive copy")
(opt, args) = parser.parse_args()
parser.process_informational_options()
if len(args) < 2:
parser.error("Must give a source and a destination")
dest = args.pop()
this_destination = dest
if dest[0:4] != 'vos:':
dest = os.path.abspath(dest)
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token,
transfer_shortcut=opt.quick)
exit_code = 0
cutout_pattern = re.compile(
r'(.*?)(?P<cutout>(\[[\-\+]?[\d\*]+(:[\-\+]?[\d\*]+)?(,[\-\+]?[\d\*]+(:[\-\+]?[\d\*]+)?)?\])+)$'
)
ra_dec_cutout_pattern = re.compile("([^\(\)]*?)"
"(?P<cutout>\("
"(?P<ra>[\-\+]?\d*(\.\d*)?),"
"(?P<dec>[\-\+]?\d*(\.\d*)?),"
"(?P<rad>\d*(\.\d*)?)\))?")
# Warnings:
# vcp destination specified with a trailing '/' implies ContainerNode
#
# If destination has trailing '/' and exists but is a DataNode then
# error message is returned: "Invalid Argument (target node is not a DataNode)"
#
# vcp currently only works on the CADC VOSpace server.
# Version: %s """ % (version.version)
def size(filename):
if filename[0:4] == "vos:":
return client.size(filename)
return os.stat(filename).st_size
def create(filename):
if filename[0:4] == "vos:":
return client.create(vos.Node(client.fix_uri(filename)))
else:
open(filename, 'w').close()
pass
def get_node(filename, limit=None):
"""Get node, from cache if possible"""
return client.get_node(filename, limit=limit)
# here are a series of methods that choose between calling the system version or the vos version of various
# function, based on pattern matching.
# TODO: Put these function in a separate module.
def isdir(filename):
logging.debug("Doing an isdir on %s" % filename)
if filename[0:4] == "vos:":
return client.isdir(filename)
else:
return os.path.isdir(filename)
def islink(filename):
logging.debug("Doing an islink on %s" % filename)
if filename[0:4] == "vos:":
try:
return get_node(filename).islink()
except:
return False
else:
return os.path.islink(filename)
def access(filename, mode):
"""
Check if the file can be accessed.
@param filename: name of file or vospace node to check
@param mode: what access mode should be checked: see os.access
@return: True/False
"""
logging.debug("checking for access %s " % filename)
if filename[0:4] == "vos:":
try:
node = get_node(filename, limit=0)
return node is not None
except (exceptions.NotFoundException,
exceptions.ForbiddenException,
exceptions.UnauthorizedException) as ex:
return False
else:
return os.access(filename, mode)
def listdir(dirname):
"""Walk through the directory structure a al os.walk"""
logging.debug("getting a dirlist %s " % dirname)
if dirname[0:4] == "vos:":
return client.listdir(dirname, force=True)
else:
return os.listdir(dirname)
def mkdir(filename):
logging.debug("Making directory %s " % filename)
if filename[0:4] == 'vos:':
return client.mkdir(filename)
else:
return os.mkdir(filename)
def lglob(pathname):
"""
Call system glob if not vos path.
@param pathname: the pathname (aka pattern) to glob with.
@return: list of matched filenames.
"""
if pathname[0:4] == "vos:":
return client.glob(pathname)
else:
return glob.glob(pathname)
def get_md5(filename):
logging.debug("getting the MD5 for %s" % filename)
if filename[0:4] == 'vos:':
return get_node(filename).props.get('MD5', vos.ZERO_MD5)
else:
return md5_cache.MD5_Cache.computeMD5(filename)
def lglob(pathname):
if pathname[0:4] == "vos:":
return client.glob(pathname)
else:
return glob.glob(pathname)
def copy(source_name,
destination_name,
exclude=None,
include=None,
interrogate=False,
overwrite=False,
ignore=False):
"""
:param source_name:
:param destination_name:
:param exclude:
:param include:
:return: :raise e:
"""
global exit_code
## determine if this is a directory we are copying so need to be recursive
try:
if not opt.follow_links and islink(source_name):
#return link(source_name, destination_name)
logging.info(
"{}: Skipping (symbolic link)".format(source_name))
return
if isdir(source_name):
## make sure the destination exists...
if not isdir(destination_name):
mkdir(destination_name)
## for all files in the current source directory copy them to the destination directory
for filename in listdir(source_name):
logging.debug("%s -> %s" % (filename, source_name))
copy(os.path.join(source_name, filename),
os.path.join(destination_name, filename), exclude,
include, interrogate, overwrite, ignore)
else:
if interrogate:
if access(destination_name, os.F_OK):
sys.stderr.write(
"File %s exists. Overwrite? (y/n): " %
destination_name)
ans = sys.stdin.readline().strip()
if ans != 'y':
raise Exception("File exists")
if not overwrite and access(destination_name, os.F_OK):
### check if the MD5 of dest and source mathc, if they do then skip
if get_md5(destination_name) == get_md5(source_name):
logging.info("%s matches %s, skipping" %
(source_name, destination_name))
return
if not access(os.path.dirname(destination_name), os.F_OK):
raise OSError(
errno.EEXIST, "vcp: ContainerNode %s does not exist" %
os.path.dirname(destination_name))
if not isdir(os.path.dirname(destination_name)) and not islink(
os.path.dirname(destination_name)):
raise OSError(
errno.ENOTDIR,
"vcp: %s is not a ContainerNode or LinkNode" %
os.path.dirname(destination_name))
skip = False
if exclude is not None:
for thisIgnore in exclude.split(','):
if not destination_name.find(thisIgnore) < 0:
skip = True
continue
if include is not None:
skip = True
for thisIgnore in include.split(','):
if not destination_name.find(thisIgnore) < 0:
skip = False
continue
if not skip:
logging.info("%s -> %s " % (source_name, destination_name))
niters = 0
while not skip:
try:
logging.debug("Starting call to copy")
client.copy(source_name,
destination_name,
send_md5=True)
logging.debug("Call to copy returned")
break
except Exception as client_exception:
logging.debug("{}".format(client_exception))
if getattr(client_exception, 'errno', -1) == 104:
# 104 is connection reset by peer. Try again on this error
logging.warning(str(client_exception))
exit_code = getattr(client_exception, 'errno', -1)
elif getattr(client_exception, 'errno',
-1) == errno.EIO:
# retry on IO errors
logging.warning(
"{0}: Retrying".format(client_exception))
pass
elif ignore:
if niters > 100:
logging.error(
"%s (skipping after %d attempts)" %
(str(client_exception), niters))
skip = True
else:
logging.error("%s (retrying)" %
str(client_exception))
time.sleep(5)
niters += 1
else:
raise client_exception
except Exception as ex:
logging.debug("{}".format(ex))
raise
except OSError as os_exception:
logging.debug(str(os_exception))
if getattr(os_exception, 'errno', -1) == errno.EINVAL:
# not a valid uri, just skip those...
logging.warning("%s: Skipping" % str(os_exception))
else:
raise os_exception
# main loop
try:
for source_pattern in args:
# define this empty cutout string. Then we strip possible cutout strings off the end of the
# pattern before matching. This allows cutouts on the vos service.
# The shell does pattern matching for local files, so don't run glob on local files.
if source_pattern[0:4] != "vos:":
sources = [source_pattern]
else:
cutout_match = cutout_pattern.search(source_pattern)
cutout = None
if cutout_match is not None:
source_pattern = cutout_match.group(1)
cutout = cutout_match.group('cutout')
else:
ra_dec_match = ra_dec_cutout_pattern.search(source_pattern)
if ra_dec_match is not None:
cutout = ra_dec_match.group('cutout')
logging.debug("cutout: {}".format(cutout))
sources = lglob(source_pattern)
if cutout is not None:
# stick back on the cutout pattern if there was one.
sources = [s + cutout for s in sources]
for source in sources:
if source[0:4] != "vos:":
source = os.path.abspath(source)
# the source must exist, of course...
if not access(source, os.R_OK):
raise Exception("Can't access source: %s " % source)
if not opt.follow_links and islink(source):
logging.info("{}: Skipping (symbolic link)".format(source))
continue
# copying inside VOSpace not yet implemented
if source[0:4] == 'vos:' and dest[0:4] == 'vos:':
raise Exception(
"Can not (yet) copy from VOSpace to VOSpace.")
this_destination = dest
if isdir(source):
if not opt.follow_links and islink(source):
continue
logging.debug("%s is a directory or link to one" % source)
# To mimic unix fs behaviours if copying a directory and
# the destination directory exists then the actual
# destination in a recursive copy is the destination +
# source basename.
# This has an odd behaviour if more than one directory is given as a source and the copy is recursive.
if access(dest, os.F_OK):
if not isdir(dest):
raise Exception(
"Can't write a directory (%s) to a file (%s)" %
(source, dest))
# directory exists so we append the end of source to that (UNIX behaviour)
this_destination = os.path.normpath(
os.path.join(dest, os.path.basename(source)))
elif len(args) > 1:
raise Exception(
"vcp can not copy multiple things into a non-existent location (%s)"
% dest)
elif dest[-1] == '/' or isdir(dest):
# we're copying into a directory
this_destination = os.path.join(dest,
os.path.basename(source))
copy(source,
this_destination,
exclude=opt.exclude,
include=opt.include,
interrogate=opt.interrogate,
overwrite=opt.overwrite,
ignore=opt.ignore)
except KeyboardInterrupt as ke:
logging.info("Received keyboard interrupt. Execution aborted...\n")
exit_code = getattr(ke, 'errno', -1)
except ParseError as pe:
exit_code = errno.EREMOTE
logging.error("Failure at server while copying {0} -> {1}".format(
source, dest))
except Exception as e:
message = str(e)
if re.search('NodeLocked', str(e)) is not None:
logging.error(
"Use vlock to unlock the node before copying to %s." %
this_destination)
elif getattr(e, 'errno', -1) == errno.EREMOTE:
logging.error(
"Failure at remote server while copying {0} -> {1}".format(
source, dest))
else:
logging.debug("Exception throw: %s %s" % (type(e), str(e)))
logging.debug(traceback.format_exc())
logging.error(message)
exit_code = getattr(e, 'errno', -1)
sys.exit(exit_code)
