def vrmdir():
parser = CommonParser(description=DESCRIPTION)
parser.add_argument('nodes', help="Container nodes to delete from VOSpace",
nargs='+')
args = parser.parse_args()
set_logging_level_from_args(args)
try:
for container_node in args.nodes:
if not vos.is_remote_file(container_node):
raise ValueError(
"{} is not a valid VOSpace handle".format(container_node))
client = vos.Client(
vospace_certfile=args.certfile,
vospace_token=args.token)
if client.isdir(container_node):
logging.info("deleting {}".format(container_node))
client.delete(container_node)
else:
raise ValueError(
"{} is a not a container node".format(container_node))
except Exception as ex:
exit_on_exception(ex)
def __init__(self, this_queue):
super(ThreadCopy, self).__init__()
self.client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
self.queue = this_queue
self.filesSent = 0
self.filesSkipped = 0
self.bytesSent = 0
self.bytesSkipped = 0
self.filesErrored = 0
def getCutoutImage(ra, dec, tile, hw=1750):
'''Get a large cutout of CFIS tile. Input images are 10000x10000.
This function makes an (11 arcmin x 11 arcmin) of the CFIS tile.
It finds the (row,col) position of the target objID using ra,dec (degrees).
It then checks for boundary conditions on the cutout region and adapts.
The final cutout is 3501x3501 pixels, not always centered on the target.
`hw` is the half-width of the cutout size. Returns cutout filename.'''
warnings.filterwarnings('ignore')
client = vos.Client()
wcsName = 'WCS-{}'.format(tile)
cutoutName = 'Cutout-{}'.format(tile)
# get wcs file
while not os.access(wcsName, 0):
try:
client.copy(source='vos:cfis/tiles_DR2/{}[1:2,1:2]'.format(tile),
destination=wcsName)
except:
time.sleep(10)
# get wcs mapping
wcs = WCS(wcsName)
# determine column and row position in image
colc, rowc = wcs.all_world2pix(ra, dec, 1, ra_dec_order=True)
# convert to integers
colc, rowc = int(np.around(colc)), int(np.around(rowc))
# remove temporary file
if os.access(wcsName, 0): os.remove(wcsName)
# get boundaries
if colc - hw < 1:
colc_m = 1
colc_p = 2 * hw + 1
elif colc + hw > 10000:
colc_m = 10000 - (2 * hw)
colc_p = 10000
else:
colc_m = colc - hw
colc_p = colc + hw
if rowc - hw < 1:
rowc_m = 1
rowc_p = 2 * hw + 1
elif rowc + hw > 10000:
rowc_p = 10000
rowc_m = 10000 - (2 * hw)
else:
rowc_m = rowc - hw
rowc_p = rowc + hw
# get full file
while not os.access(cutoutName, 0):
try:
client.copy(source='vos:cfis/tiles_DR2/{}[{}:{},{}:{}]'.format(
tile, colc_m, colc_p, rowc_m, rowc_p),
destination=cutoutName)
except:
time.sleep(10)
return cutoutName
def _cat(vospace_uri, cert_filename=None):
"""Cat out the given uri."""
fh = None
try:
if vospace_uri[0:4] == "vos:":
fh = vos.Client(vospace_certfile=cert_filename).open(vospace_uri, view='data')
else:
fh = open(vospace_uri, 'r')
sys.stdout.write(fh.read())
finally:
if fh:
fh.close()
def vln():
usage = """
vln vos:VOSpaceSource vos:VOSpaceTarget
examples:
vln vos:vospace/junk.txt vos:vospace/linkToJunk.txt
vln vos:vospace/directory vos:vospace/linkToDirectory
vln http://external.data.source vos:vospace/linkToExternalDataSource
"""
parser = CommonParser(usage)
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt, args) = parser.parse_args()
parser.process_informational_options()
logger = logging.getLogger()
logger.setLevel(parser.log_level)
logger.addHandler(logging.StreamHandler())
if len(args) != 2:
parser.error("You must specify a source file and a target file")
sys.exit(-1)
if args[1][0:4] != "vos:":
parser.error("The target to source must be in vospace")
sys.exit(-1)
logger.debug("Connecting to vospace using certificate %s" % opt.certfile)
try:
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
except Exception as e:
logger.error("VOS connection failed: {0}".format(e))
sys.exit(-1)
try:
client.link(args[0], args[1])
except Exception as e:
logger.error("Failed to make link from %s to %s" % (args[0], args[1]))
logger.error(getattr(e, 'strerror', 'Unknown Error'))
sys.exit(-1)
def vrm():
usage="""
vrm vos:/root/node -- deletes a data node
Version: %s """ % (version.version)
parser=CommonParser(usage)
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt, args)=parser.parse_args()
parser.process_informational_options()
logger = logging.getLogger()
logger.setLevel(parser.log_level)
try:
client=vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token)
except Exception as e:
logger.error("Connection failed: %s" % (str(e)))
exit_code = getattr(e, 'errno', -1)
sys.exit(exit_code)
for arg in args:
if arg[0:4]!="vos:":
logger.error("%s is not a valid VOSpace handle" % (arg))
try:
if client.isfile(arg) or client.get_node(arg).islink():
logger.info("deleting %s" %(arg))
client.delete(arg)
elif client.isdir(arg):
logger.error("%s is a directory" % (arg))
elif client.access(arg):
logger.info("deleting link %s" %(arg))
client.delete(arg)
else:
logger.error("%s file not found" % (arg))
sys.exit(-1)
except Exception as e:
import re
if re.search('NodeLocked',str(e)) != None:
logger.error("Use vlock to unlock %s before deleting." % (arg))
exit_code = getattr(e, 'errno', -1)
logger.error("Failed trying to delete {0}: {1}".format( arg, str(e)))
sys.exit(exit_code)
def vmkdir():
usage="""
vmkdir vos:/root/node -- creates a new directory (ContainerNode) called node at vospace root
Version: %s """ % (version.version)
parser=CommonParser(usage)
parser.add_option("-p",action="store_true",help="Create intermediate directories as required.")
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt,args)=parser.parse_args()
parser.process_informational_options()
if len(args)>1:
parser.error("Only one directory can be built per call")
logging.info("Creating ContainerNode (directory) %s" % ( args[0]))
try:
client=vos.Client(vospace_certfile=opt.certfile,vospace_token=opt.token)
dirNames=[]
thisDir = args[0]
if opt.p:
while not client.access(thisDir):
dirNames.append(os.path.basename(thisDir))
thisDir = os.path.dirname(thisDir)
while len(dirNames) > 0:
thisDir = os.path.join(thisDir,dirNames.pop())
client.mkdir(thisDir)
else:
client.mkdir(thisDir)
except Exception as e:
logging.error(str(e))
sys.exit(getattr(e, 'errno', -1))
sys.exit(0)
def vrmdir():
usage="""
vrmdir vos:/root/node -- deletes a container node
Version: %s """ % (version.version)
parser = CommonParser(usage)
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt, args) = parser.parse_args()
parser.process_informational_options()
try:
client=vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
except Exception as e:
logging.error("Connection failed: %s" % (str(e)))
sys.exit(e.__getattribute__('errno', -1))
try:
for arg in args:
if arg[0:4]!="vos:":
logging.error("%s is not a valid VOSpace handle" % (arg))
sys.exit(-1)
if client.isdir(arg):
logging.info("deleting %s" %(arg))
client.delete(arg)
elif client.isfile(arg):
logging.error("%s is a file" % (arg))
sys.exit(-1)
else:
logging.error("%s file not found" % (arg))
sys.exit(-1)
except Exception as e:
import re
if re.search('NodeLocked', str(e)) != None:
logging.error("Use vlock to unlock %s before removing." %(arg))
logging.error("Connection failed: %s" % (str(e)))
sys.exit(-1)
def vlock():
signal.signal(signal.SIGINT, signal_handler)
parser = CommonParser()
parser.add_option("--lock", action="store_true", help="Lock the node")
parser.add_option("--unlock", action="store_true", help="unLock the node")
(opt, args) = parser.parse_args()
parser.process_informational_options()
logger = logging.getLogger()
logger.setLevel(parser.log_level)
logger.addHandler(logging.StreamHandler())
lock = None
if opt.lock:
lock = 'true'
elif opt.unlock:
lock = 'false'
exit_code = 0
try:
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
node = client.get_node(args[0])
if opt.lock or opt.unlock:
lock = not opt.unlock and opt.lock
node.is_locked = lock
client.update(node)
else:
print(node.is_locked)
if not node.is_locked:
exit_code = -1
except KeyboardInterrupt:
exit_code = -1
except Exception as e:
logger.error(str(e))
exit_code = -1
sys.exit(exit_code)
def vsync():
global_md5_cache = None
def signal_handler(h_stream, h_frame):
logging.debug("{} {}".format(h_stream, h_frame))
logging.critical("Interrupt\n")
sys.exit(-1)
# handle interrupts nicely
signal.signal(signal.SIGINT, signal_handler)
start_time = time.time()
parser = CommonParser(description=DESCRIPTION)
parser.add_option('files', nargs='+', help='Files to copy to VOSpace')
parser.add_option('destination', help='VOSpace location to sync files to')
parser.add_option('--ignore-checksum', action="store_true",
help='dont check MD5 sum, use size and time instead')
parser.add_option('--cache_nodes', action='store_true',
help='cache node MD5 sum in an sqllite db')
parser.add_option('--cache_filename',
help="Name of file to use for node cache",
default="{}/.config/vos/node_cache.db".format(HOME))
parser.add_option('--recursive', '-r', help="Do a recursive sync",
action="store_true")
parser.add_option('--nstreams', '-n', type=int,
help="Number of streams to run (MAX: 30)", default=5)
parser.add_option(
'--exclude',
help="ignore directories or files containing this pattern",
default=None)
parser.add_option('--include',
help="only include files matching this pattern",
default=None)
parser.add_option(
'--overwrite',
help=("overwrite copy on server regardless of modification/size/md5 "
"checks"),
action="store_true")
opt = parser.parse_args()
set_logging_level_from_args(opt)
if opt.nstreams > 30:
parser.error("Maximum of 30 streams exceeded")
if opt.cache_nodes:
global_md5_cache = md5_cache.MD5Cache(cache_db=opt.cache_filename)
destination = opt.destination
if not vos.is_remote_file(destination):
parser.error("Only allows sync FROM local copy TO VOSpace")
# Currently we don't create nodes in sync and we don't sync onto files
logging.info("Connecting to VOSpace")
client = vos.Client(
vospace_certfile=opt.certfile, vospace_token=opt.token)
logging.info("Confirming Destination is a directory")
dest_is_dir = client.isdir(destination)
queue = JoinableQueue(maxsize=10 * opt.nstreams)
good_dirs = []
node_dict = {}
def compute_md5(this_filename, block_size=None):
"""
Read through a file and compute that files MD5 checksum.
:param this_filename: name of the file on disk
:param block_size: number of bytes to read into memory,
defaults to 2**19 bytes
:return: md5 as a hexadecimal string
"""
block_size = block_size is None and 2 ** 19 or block_size
return md5_cache.MD5Cache.compute_md5(this_filename,
block_size=block_size)
def file_md5(this_filename):
import os
md5 = None
if global_md5_cache is not None:
md5 = global_md5_cache.get(this_filename)
if md5 is None or md5[2] < os.stat(this_filename).st_mtime:
md5 = compute_md5(this_filename)
if global_md5_cache is not None:
stat = os.stat(this_filename)
global_md5_cache.update(this_filename, md5, stat.st_size,
stat.st_mtime)
else:
md5 = md5[0]
return md5
class ThreadCopy(Process):
def __init__(self, this_queue):
super(ThreadCopy, self).__init__()
self.client = vos.Client(
vospace_certfile=opt.certfile,
vospace_token=opt.token)
self.queue = this_queue
self.filesSent = 0
self.filesSkipped = 0
self.bytesSent = 0
self.bytesSkipped = 0
self.filesErrored = 0
def run(self):
while True:
(current_source, current_destination) = self.queue.get()
requeue = (current_source, current_destination)
src_md5 = None
stat = os.stat(current_source)
if not opt.ignore_checksum and not opt.overwrite:
src_md5 = file_md5(current_source)
if not opt.overwrite:
# Check if the file is the same
try:
node_info = None
if opt.cache_nodes:
node_info = global_md5_cache.get(
current_destination)
if node_info is None:
logging.debug("Getting node info from VOSpace")
logging.debug(str(node_dict.keys()))
logging.debug(str(current_destination))
node = self.client.get_node(current_destination,
limit=None)
current_destination_md5 = node.props.get(
'MD5', 'd41d8cd98f00b204e9800998ecf8427e')
current_destination_length = node.attr['st_size']
current_destination_time = node.attr['st_ctime']
if opt.cache_nodes:
global_md5_cache.update(
current_destination,
current_destination_md5,
current_destination_length,
current_destination_time)
else:
current_destination_md5 = node_info[0]
current_destination_length = node_info[1]
current_destination_time = node_info[2]
logging.debug("Destination MD5: {}".format(
current_destination_md5))
if ((not opt.ignore_checksum and src_md5 ==
current_destination_md5) or
(opt.ignore_checksum and
current_destination_time >= stat.st_mtime and
current_destination_length == stat.st_size)):
logging.info("skipping: %s matches %s" % (
current_source, current_destination))
self.filesSkipped += 1
self.bytesSkipped += current_destination_length
self.queue.task_done()
continue
except (transfer_exceptions.AlreadyExistsException,
transfer_exceptions.NotFoundException):
pass
logging.info(
"%s -> %s" % (current_source, current_destination))
try:
self.client.copy(current_source, current_destination,
send_md5=True)
node = self.client.get_node(current_destination,
limit=None)
current_destination_md5 = node.props.get(
'MD5', 'd41d8cd98f00b204e9800998ecf8427e')
current_destination_length = node.attr['st_size']
current_destination_time = node.attr['st_ctime']
if opt.cache_nodes:
global_md5_cache.update(current_destination,
current_destination_md5,
current_destination_length,
current_destination_time)
self.filesSent += 1
self.bytesSent += stat.st_size
except (IOError, OSError) as exc:
logging.error(
"Error writing {} to server, skipping".format(
current_source))
logging.error(str(exc))
import re
if re.search('NodeLocked', str(exc)) is not None:
logging.error(
("Use vlock to unlock the node before syncing "
"to {}").format(current_destination))
try:
if exc.errno == 104:
self.queue.put(requeue)
except Exception as e2:
logging.error("Error during requeue")
logging.error(str(e2))
pass
self.filesErrored += 1
pass
self.queue.task_done()
def mkdirs(directory):
"""Recursively make all nodes in the path to directory.
:param directory: str, vospace location of ContainerNode (directory)
to make
:return:
"""
logging.debug("%s %s" % (directory, str(good_dirs)))
# if we've seen this before skip it.
if directory in good_dirs:
return
# try and make a new directory and return
# failure indicates we should see if subdirectories exist
try:
client.mkdir(directory)
logging.info("Made directory {}".format(directory))
good_dirs.append(directory)
return
except transfer_exceptions.AlreadyExistsException:
pass
# OK, must already have existed, add to list
good_dirs.append(directory)
return
def copy(current_source, current_destination):
"""
Copy current_source from local file system to current_destination.
:param current_source: name of local file
:param current_destination: name of localtion on VOSpace to copy file
to (includes filename part)
:return: None
"""
# strip down current_destination until we find a part that exists
# and then build up the path.
if os.path.islink(current_source):
logging.error("{} is a link, skipping".format(current_source))
return
if not os.access(current_source, os.R_OK):
logging.error(
"Failed to open file {}, skipping".format(current_source))
return
import re
if re.match(r'^[A-Za-z0-9._\-();:&*$@!+=/]*$', current_source) is None:
logging.error(
"filename %s contains illegal characters, skipping" %
current_source)
return
dirname = os.path.dirname(current_destination)
mkdirs(dirname)
if opt.include is not None and not re.search(opt.include,
current_source):
return
queue.put((current_source, current_destination), timeout=3600)
def start_streams(no_streams):
list_of_streams = []
for i in range(no_streams):
logging.info("Launching VOSpace connection stream %d" % i)
t = ThreadCopy(queue)
t.daemon = True
t.start()
list_of_streams.append(t)
return list_of_streams
def build_file_list(base_path, destination_root='', recursive=False,
ignore=None):
"""Build a list of files that should be copied into VOSpace"""
spinner = ['-', '\\', '|', '/', '-', '\\', '|', '/']
count = 0
for (root, dirs, filenames) in os.walk(base_path):
for this_dirname in dirs:
if not recursive:
continue
this_dirname = os.path.join(root, this_dirname)
skip = False
if ignore is not None:
for thisIgnore in ignore.split(','):
if not this_dirname.find(thisIgnore) < 0:
logging.info("excluding: %s " % this_dirname)
skip = True
continue
if skip:
continue
cprefix = os.path.commonprefix((base_path, this_dirname))
this_dirname = os.path.normpath(
destination_root + "/" + this_dirname[len(cprefix):])
mkdirs(this_dirname)
for thisfilename in filenames:
srcfilename = os.path.normpath(
os.path.join(root, thisfilename))
skip = False
if ignore is not None:
for thisIgnore in ignore.split(','):
if not srcfilename.find(thisIgnore) < 0:
logging.info("excluding: %s " % srcfilename)
skip = True
continue
if skip:
continue
cprefix = os.path.commonprefix((base_path, srcfilename))
destfilename = os.path.normpath(
destination_root + "/" + srcfilename[len(cprefix):])
this_dirname = os.path.dirname(destfilename)
mkdirs(this_dirname)
count += 1
if opt.verbose:
sys.stderr.write(
"Building list of files to transfer %s\r" % (
spinner[count % len(spinner)]))
copy(srcfilename, destfilename)
if not recursive:
return
return
streams = start_streams(opt.nstreams)
# build a complete file list given all the things on the command line
for filename in opt.files:
filename = os.path.abspath(filename)
this_root = destination
if os.path.isdir(filename):
if filename[-1] != "/":
if os.path.basename(filename) != os.path.basename(destination):
this_root = os.path.join(destination,
os.path.basename(filename))
mkdirs(this_root)
node_dict[this_root] = client.get_node(this_root, limit=None)
try:
build_file_list(filename, destination_root=this_root,
recursive=opt.recursive, ignore=opt.exclude)
except Exception as e:
logging.error(str(e))
logging.error("ignoring error")
elif os.path.isfile(filename):
if dest_is_dir:
this_root = os.path.join(destination,
os.path.basename(filename))
copy(filename, this_root)
else:
logging.error("%s: No such file or directory." % filename)
logging.info(
("Waiting for transfers to complete "
r"******** CTRL-\ to interrupt ********"))
queue.join()
end_time = time.time()
bytes_sent = 0
files_sent = 0
bytes_skipped = 0
files_skipped = 0
files_erred = 0
for stream in streams:
bytes_sent += stream.bytesSent
bytes_skipped += stream.bytesSkipped
files_sent += stream.filesSent
files_skipped += stream.filesSkipped
files_erred += stream.filesErrored
logging.info("==== TRANSFER REPORT ====")
if bytes_sent > 0:
rate = bytes_sent / (end_time - start_time) / 1024.0
logging.info("Sent %d files (%8.1f kbytes @ %8.3f kBytes/s)" % (
files_sent, bytes_sent / 1024.0, rate))
speed_up = (bytes_skipped + bytes_sent) / bytes_sent
logging.info(
"Speedup: %f (skipped %d files)" % (speed_up, files_skipped))
if bytes_sent == 0:
logging.info("No files needed sending ")
if files_erred > 0:
logging.info(
"Error transferring %d files, please try again" % files_erred)
def vls():
parser = CommonParser(usage, description=description, add_help_option=False)
parser.add_option("--help", action="store_true")
parser.add_option("-l", "--long", action="store_true", help="verbose listing sorted by name")
parser.add_option("-g", "--group", action="store_true", help="display group read/write information")
parser.add_option("-h", "--human", action="store_true", help="make sizes human readable", default=False)
parser.add_option("-S", "--Size", action="store_true", help="sort files by size", default=False)
parser.add_option("-r", "--reverse", action="store_true", help="reverse the sort order", default=False)
parser.add_option("-t", "--time", action="store_true", help="sort by time copied to VOSpace")
(opt, args) = parser.parse_args()
# We disabled -h/--help so vls can have a -h option.
# Here we re-enable it but only with --help
if opt.help:
parser.print_help()
sys.exit(0)
parser.process_informational_options()
if not len(args) > 0:
parser.error("missing VOSpace argument")
logger = logging.getLogger()
logger.setLevel(parser.log_level)
sortKey = (opt.time and "date") or (opt.Size and "size") or False
try:
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
except Exception as e:
logger.error("Connection failed: %s" % (str(e)))
sys.exit(-1)
columns = ['permissions']
if opt.long:
columns.extend(['creator'])
columns.extend(['readGroup', 'writeGroup', 'isLocked', 'size', 'date'])
def get_terminal_size():
"""Get the size of the terminal
@return: tuple(int, int) giving the row/column of the terminal screen.
"""
def ioctl_gwinsz(fd):
"""
@param fd: A file descriptor that points at the Screen Parameters
@return: the termios screeen structure unpacked into an array.
"""
try:
import fcntl
import termios
import struct
this_cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
except:
return None
return this_cr
cr = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2)
if not cr:
try:
td = os.open(os.ctermid(), os.O_RDONLY)
cr = ioctl_gwinsz(td)
td.close()
except:
pass
if not cr:
try:
cr = (os.environ['LINES'], os.environ['COLUMNS'])
except:
cr = (25, 80)
return int(cr[1]), int(cr[0])
def size_format(size):
"""Format a size value for listing"""
try:
size = float(size)
except Exception as ex:
logger.debug(str(ex))
size = 0.0
if opt.human:
size_unit = ['B', 'K', 'M', 'G', 'T']
try:
length = float(size)
scale = int(math.log(length) / math.log(1024))
length = "%.0f%s" % (length / (1024.0 ** scale), size_unit[scale])
except:
length = str(int(size))
else:
length = str(int(size))
return "%12s " % length
def date_format(epoch):
"""given a time object return a unix-ls like formatted string"""
time_tuple = time.localtime(epoch)
if time.localtime().tm_year != time_tuple.tm_year:
return time.strftime('%b %d %Y ', time_tuple)
return time.strftime('%b %d %H:%S ', time_tuple)
formats = {'permissions': lambda value: "{:<11}".format(value),
'creator': lambda value: " {:<20}".format(value),
'readGroup': lambda value: " {:<15}".format(value.replace(CADC_GMS_PREFIX, "")),
'writeGroup': lambda value: " {:<15}".format(value.replace(CADC_GMS_PREFIX, "")),
'isLocked': lambda value: " {:<8}".format("", "LOCKED")[value == "true"],
'size': size_format,
'date': date_format}
for node in args:
try:
if not node[0:4] == "vos:" :
raise OSError(errno.EBADF, "Invalid node name", node)
logger.debug("getting listing of: %s" % str(node))
infoList = client.get_info_list(node)
except SSLError as error:
logger.error(str(error))
sys.exit(errno.EPERM)
except Exception as e:
logger.error(str(e))
err_no = getattr(e, 'errno', None)
err_no = err_no is not None and err_no or errno.ENOMSG
sys.exit(err_no)
if sortKey:
try:
sorted_list = sorted(infoList, key=lambda name: name[1][sortKey], reverse=not opt.reverse)
except:
sorted_list = infoList
finally:
infoList = sorted_list
for item in infoList:
name_string = item[0]
if opt.long or opt.group:
for col in columns:
value = item[1].get(col, None)
value = value is not None and value or ""
if col in formats:
sys.stdout.write(formats[col](value))
if item[1]["permissions"][0] == 'l':
name_string = "%s -> %s" % (name_string, item[1]['target'])
sys.stdout.write("%s\n" % name_string)
def vtag():
usage = """
vtag [options] node [key[=value] [key[=value] ...]]
Version: %s """ % (version.version)
signal.signal(signal.SIGINT, signal_handler)
parser = CommonParser(usage)
parser.add_option('--remove',
action="store_true",
help='remove the listed property')
# Not yet supported
# parser.add_option("-R", "--recursive", action='store_const', const=True,
# help="Recursive set read/write properties")
(opt, args) = parser.parse_args()
parser.process_informational_options()
if len(args) == 0:
parser.error("no vospace supplied")
logger = logging.getLogger()
logger.setLevel(parser.log_level)
logger.addHandler(logging.StreamHandler())
## the node should be the first argument, the rest should contain the key/val pairs
node = args.pop(0)
props = []
if opt.remove:
## remove signified by blank value in key=value listing
for prop in args:
if '=' not in prop:
prop += "="
props.append(prop)
else:
props = args
try:
"""Lists/sets values of node properties based on context (length of props).
node: vos.vos.Node object
props: dictionary of properties to set. If dict is zero length, list them all.
"""
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
node = client.get_node(node)
if len(props) == 0:
# print all properties
for key in node.props:
print(key, node.props[key])
return pprint.pprint(node.props)
changed = False
for prop in props:
prop = prop.split('=')
if len(prop) == 1:
# get one property
logger.debug("just print this one out %s" % (prop[0]))
pprint.pprint(node.props.get(prop[0], None))
elif len(prop) == 2:
# update properties
key, value = prop
logger.debug("%s %s" % (len(key), len(value)))
if len(value) == 0:
value = None
logger.debug("%s: %s -> %s" %
(key, node.props.get(key, None), value))
if value != node.props.get(key, None):
node.props[key] = value
changed = True
else:
raise ValueError(
"Illigal keyword of value character ('=') used: %s" %
('='.join(prop)))
if changed:
client.add_props(node)
return 0
except KeyboardInterrupt:
logger.debug("Received keyboard interrupt. Execution aborted...\n")
pass
except Exception as e:
logger.error(str(e))
sys.exit(-1)
def vsync():
def signal_handler(signal, frame):
logger.critical("Interupt\n")
sys.exit(-1)
usage = """
vsync [options] files vos:Destination/
Version: %s """ % (version.version)
# handle interupts nicely
signal.signal(signal.SIGINT, signal_handler)
startTime = time.time()
parser = CommonParser(usage)
parser.add_option('--ignore-checksum',
action="store_true",
help='dont check MD5 sum, use size and time instead')
parser.add_option('--cache_nodes',
action='store_true',
help='cache node MD5 sum in an sqllite db')
parser.add_option('--recursive',
'-r',
help="Do a recursive sync",
action="store_true")
parser.add_option('--nstreams',
'-n',
type=int,
help="Number of streams to run (MAX: 30)",
default=1)
parser.add_option(
'--exclude',
help="ignore directories or files containing this pattern",
default=None)
parser.add_option('--include',
help="only include files matching this pattern",
default=None)
parser.add_option(
'--overwrite',
help=
"overwrite copy on server regardless of modification/size/md5 checks",
action="store_true")
parser.add_option(
'--load_test',
action="store_true",
help=
"Used to stress test the VOServer, also set --nstreams to a large value"
)
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt, args) = parser.parse_args()
parser.process_informational_options()
logger = logging.getLogger()
logger.setLevel(parser.log_level)
logger.addHandler(logging.StreamHandler())
if len(args) < 2:
parser.error(
"requires one or more source files and a single destination directory"
)
if opt.nstreams > 30 and not opt.load_test:
parser.error("Maximum of 30 streams exceeded")
if opt.cache_nodes:
from vos import md5_cache
md5Cache = md5_cache.MD5_Cache()
dest = args.pop()
if dest[0:4] != "vos:":
parser.error("Only allows sync FROM local copy TO VOSpace")
## Currently we don't create nodes in sync and we don't sync onto files
logger.info("Connecting to VOSpace")
client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token)
logger.info("Confirming Destination is a directory")
destIsDir = client.isdir(dest)
queue = JoinableQueue(maxsize=10 * opt.nstreams)
goodDirs = []
nodeDict = {}
def computeMD5(filename, block_size=None):
"""
Read through a file and compute that files MD5 checksum.
:param filename: name of the file on disk
:param block_size: number of bytes to read into memory, defaults to 2**19 bytes
:return: md5 as a hexadecimal string
"""
block_size = block_size is None and 2**19 or block_size
md5 = hashlib.md5()
r = open(filename, 'r')
while True:
buf = r.read(block_size)
if len(buf) == 0:
break
md5.update(buf)
r.close()
return md5.hexdigest()
def fileMD5(filename):
import os
md5 = None
if opt.cache_nodes:
md5 = md5Cache.get(filename)
if md5 is None or md5[2] < os.stat(filename).st_mtime:
md5 = computeMD5(filename)
if opt.cache_nodes:
stat = os.stat(filename)
md5Cache.update(filename, md5, stat.st_size, stat.st_mtime)
else:
md5 = md5[0]
return md5
class ThreadCopy(Process):
def __init__(self, queue, client):
super(ThreadCopy, self).__init__()
self.client = client
self.queue = queue
self.filesSent = 0
self.filesSkipped = 0
self.bytesSent = 0
self.bytesSkipped = 0
self.filesErrored = 0
def run(self):
while True:
(src, dest) = self.queue.get()
requeue = (src, dest)
srcMD5 = None
stat = os.stat(src)
if not opt.ignore_checksum and not opt.overwrite:
srcMD5 = fileMD5(src)
if not opt.overwrite:
# Check if the file is the same
try:
nodeInfo = None
if opt.cache_nodes:
nodeInfo = md5Cache.get(dest)
if nodeInfo is None:
logger.debug("Getting node info from VOSpace")
logger.debug(str(nodeDict.keys()))
logger.debug(str(dest))
node = self.client.get_node(dest, limit=None)
destMD5 = node.props.get(
'MD5', 'd41d8cd98f00b204e9800998ecf8427e')
destLength = node.attr['st_size']
destTime = node.attr['st_ctime']
if opt.cache_nodes:
md5Cache.update(dest, destMD5, destLength,
destTime)
else:
destMD5 = nodeInfo[0]
destLength = nodeInfo[1]
destTime = nodeInfo[2]
logger.debug("Dest MD5: %s " % (destMD5))
if (not opt.ignore_checksum and srcMD5 == destMD5) or (
opt.ignore_checksum
and destTime >= stat.st_mtime
and destLength == stat.st_size):
logger.info("skipping: %s matches %s" %
(src, dest))
self.filesSkipped += 1
self.bytesSkipped += destLength
self.queue.task_done()
continue
except (IOError, OSError) as node_error:
"""Ignore the erorr"""
logger.debug(str(node_error))
pass
logger.info("%s -> %s" % (src, dest))
try:
self.client.copy(src, dest, send_md5=True)
node = self.client.get_node(dest, limit=None)
destMD5 = node.props.get(
'MD5', 'd41d8cd98f00b204e9800998ecf8427e')
destLength = node.attr['st_size']
destTime = node.attr['st_ctime']
if opt.cache_nodes:
md5Cache.update(dest, destMD5, destLength, destTime)
self.filesSent += 1
self.bytesSent += stat.st_size
except (IOError, OSError) as e:
logger.error("Error writing %s to server, skipping" %
(src))
logger.error(str(e))
import re
if re.search('NodeLocked', str(e)) != None:
logger.error(
"Use vlock to unlock the node before syncing to %s."
% (dest))
try:
if e.errno == 104:
self.queue.put(requeue)
except Exception as e2:
logger.error("Error during requeue")
logger.error(str(e2))
pass
self.filesErrored += 1
pass
self.queue.task_done()
def mkdirs(dirs):
logger.debug("%s %s" % (dirs, str(goodDirs)))
## if we've seen this before skip it.
if dirs in goodDirs:
return
## try and make a new directory and return
## failure indicates we should see if subdirs exist
try:
client.mkdir(dirs)
logger.info("Made directory %s " % (dirs))
goodDirs.append(dirs)
return
except OSError as e:
exit_code = getattr(e, 'errno', -1)
if exit_code != errno.EEXIST:
raise e
## OK, must already have existed, add to list
goodDirs.append(dirs)
return
def copy(source, dest):
## strip down dest until we find a part that exists
## and then build up the path. Dest should include the filename
if os.path.islink(source):
logger.error("%s is a link, skipping" % (source))
return
if not os.access(source, os.R_OK):
logger.error("Failed to open file %s, skipping" % (source))
return
import re
if re.match('^[A-Za-z0-9\\._\\-\\(\\);:&\\*\\$@!+=\\/]*$',
source) is None:
logger.error("filename %s contains illegal characters, skipping" %
(source))
return
dirname = os.path.dirname(dest)
mkdirs(dirname)
if opt.include is not None and not re.search(opt.include, source):
return
queue.put((source, dest), timeout=3600)
def startStreams(nstreams, vospace_client):
streams = []
for i in range(nstreams):
logger.info("Launching vospace connection stream %d" % (i))
t = ThreadCopy(queue, client=vospace_client)
t.daemon = True
t.start()
streams.append(t)
return streams
def buildFileList(basePath, destRoot='', recursive=False, ignore=None):
"""Build a list of files that should be copied into VOSpace"""
import string
spinner = ['-', '\\', '|', '/', '-', '\\', '|', '/']
count = 0
import re
for (root, dirs, filenames) in os.walk(basePath):
for thisDirname in dirs:
if not recursive:
continue
thisDirname = os.path.join(root, thisDirname)
skip = False
if ignore is not None:
for thisIgnore in ignore.split(','):
if not thisDirname.find(thisIgnore) < 0:
logger.info("excluding: %s " % (thisDirname))
skip = True
continue
if skip:
continue
cprefix = os.path.commonprefix((basePath, thisDirname))
thisDirname = os.path.normpath(destRoot + "/" +
thisDirname[len(cprefix):])
mkdirs(thisDirname)
for thisfilename in filenames:
srcfilename = os.path.normpath(os.path.join(
root, thisfilename))
skip = False
if ignore is not None:
for thisIgnore in ignore.split(','):
if not srcfilename.find(thisIgnore) < 0:
logger.info("excluding: %s " % (srcfilename))
skip = True
continue
if skip:
continue
cprefix = os.path.commonprefix((basePath, srcfilename))
destfilename = os.path.normpath(destRoot + "/" +
srcfilename[len(cprefix):])
thisDirname = os.path.dirname(destfilename)
mkdirs(thisDirname)
count += 1
if opt.verbose:
sys.stderr.write(
"Building list of files to transfer %s\r" %
(spinner[count % len(spinner)]))
copy(srcfilename, destfilename)
if not recursive:
return
return
streams = startStreams(opt.nstreams, vospace_client=client)
### build a complete file list given all the things on the command line
for filename in args:
filename = os.path.abspath(filename)
thisRoot = dest
if os.path.isdir(filename):
if filename[-1] != "/":
if os.path.basename(filename) != os.path.basename(dest):
thisRoot = os.path.join(dest, os.path.basename(filename))
mkdirs(thisRoot)
nodeDict[thisRoot] = client.get_node(thisRoot, limit=None)
try:
buildFileList(filename,
destRoot=thisRoot,
recursive=opt.recursive,
ignore=opt.exclude)
except Exception as e:
logger.error(str(e))
logger.error("ignoring error")
elif os.path.isfile(filename):
if destIsDir:
thisRoot = os.path.join(dest, os.path.basename(filename))
copy(filename, thisRoot)
else:
logger.error("%s: No such file or directory." % (filename))
logger.info("\nWaiting for transfers to complete.\nCTRL-\ to interrupt\n")
queue.join()
endTime = time.time()
bytesSent = 0
filesSent = 0
bytesSkipped = 0
filesSkipped = 0
filesErrored = 0
for stream in streams:
bytesSent += stream.bytesSent
bytesSkipped += stream.bytesSkipped
filesSent += stream.filesSent
filesSkipped += stream.filesSkipped
filesErrored += stream.filesErrored
logger.info("\n\n==== TRANSFER REPORT ====\n\n")
if bytesSent > 0:
rate = bytesSent / (endTime - startTime) / 1024.0
logger.info("Sent %d files (%8.1f kbytes @ %8.3f kBytes/s)" %
(filesSent, bytesSent / 1024.0, rate))
speedUp = (bytesSkipped + bytesSent) / bytesSent
logger.info("Speedup: %f (skipped %d files)" %
(speedUp, filesSkipped))
if bytesSent == 0:
logger.info("No files needed sending ")
if filesErrored > 0:
logger.info("Error transferring %d files, please try again" %
(filesErrored))
try:
if len(sys.argv) not in [2, 3]:
raise Exception("Arguments are: [user id] [OPTIONAL: VOS token]")
print >> sys.stderr, sys.argv
session_uuid = sys.argv[1]
if (len(sys.argv) == 3) and (sys.argv[2] != ''):
# --- Authenticated session launcher here --------------------------
# Obtain CANFAR user name from the token and validate it by
# trying a 'vls'-like operation on the top node of the subtree
# of VOSpace for which the token is scoped
vospace_token = sys.argv[2]
username, scope = re.match('^userid=(.*?)&.*?scope=(.*?)&.*$',
vospace_token).groups()
client = vos.Client(vospace_token=vospace_token)
infoList = client.getInfoList(scope)
else:
# --- Anonymous session launcher here -----------------------------
username = session_uuid
vospace_token = ''
print >> sys.stderr, "Try to launch session for user '%s' token '%s'" % (
username, vospace_token)
container = get_or_make_container(
username, vospace_token,
username) # SJ added vospace_token and username
# SJ making sure authenticated user has directory on VOSpace:
if vospace_token is not '':
pass
def vchmod():
signal.signal(signal.SIGINT, signal_handler)
usage = """vchmod mode node [read/write group names in quotes - maximum of 4 each]
Accepted modes: (og|go|o|g)[+-=](rw|wr|r\w)
Sets read/write properties of a node.
Version: %s """ % version.version
parser = CommonParser(usage)
parser.add_option("-R", "--recursive", action='store_const', const=True,
help="Recursive set read/write properties")
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
(opt, args) = parser.parse_args()
parser.process_informational_options()
logger = logging.getLogger()
if len(args) < 2:
parser.error("Requires mode and node arguments")
cmdArgs = dict(zip(['mode', 'node'], args[0:2]))
groupNames = args[2:]
import re
mode = re.match(r"(?P<who>^og|^go|^o|^g)(?P<op>[\+\-=])(?P<what>rw$|wr$|r$|w$)", cmdArgs['mode'])
if not mode:
parser.print_help()
logger.error("\n\nAccepted modes: (og|go|o|g)[+-=](rw|wr|r\w)\n\n")
sys.exit(-1)
props = {}
if 'o' in mode.group('who'):
if not mode.group('what') == 'r': # read only
parser.print_help()
logger.error("\n\nPublic access is readonly, no public write available \n\n")
sys.exit(-1)
if mode.group('op') == '-':
props['ispublic'] = 'false'
else:
props['ispublic'] = 'true'
if 'g' in mode.group('who'):
if '-' == mode.group('op'):
if not len(groupNames) == 0:
parser.print_help()
logger.error("\n\nNames of groups not required with remove permission\n\n")
sys.exit(-1)
if 'r' in mode.group('what'):
props['readgroup'] = None
if "w" in mode.group('what'):
props['writegroup'] = None
else:
if not len(groupNames) == len(mode.group('what')):
parser.print_help()
logger.error("\n\n%s group names required for %s\n\n" %
(len(mode.group('what')), mode.group('what')))
sys.exit(-1)
if mode.group('what').find('r') > -1:
# remove duplicate whitespaces
rgroups = " ".join(groupNames[mode.group('what').find('r')].split())
props['readgroup'] = (vos.CADC_GMS_PREFIX +
rgroups.replace(" ", " " + vos.CADC_GMS_PREFIX))
if mode.group('what').find('w') > -1:
wgroups = " ".join(groupNames[mode.group('what').find('w')].split())
props['writegroup'] = (vos.CADC_GMS_PREFIX +
wgroups.replace(" ", " " + vos.CADC_GMS_PREFIX))
logger.debug("Properties: %s" % (str(props)))
logger.debug("Node: %s" % cmdArgs['node'])
returnCode = 0
try:
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token)
node = client.get_node(cmdArgs['node'])
if opt.recursive:
node.props.clear()
node.clear_properties()
# del node.node.findall(vos.Node.PROPERTIES)[0:]
if 'readgroup' in props:
node.chrgrp(props['readgroup'])
if 'writegroup' in props:
node.chwgrp(props['writegroup'])
if 'ispublic' in props:
node.set_public(props['ispublic'])
logger.debug("Node: {0}".format(node))
returnCode = client.update(node, opt.recursive)
except KeyboardInterrupt as ke:
logger.error("Received keyboard interrupt. Execution aborted...\n")
sys.exit(getattr(ke, 'errno', -1))
except Exception as e:
logger.error('Error {}: '.format(str(e)))
sys.exit(getattr(e, 'errno', -1))
sys.exit(returnCode)
def vcp():
## handle interrupts nicely
signal.signal(signal.SIGINT, signal_handler)
parser = CommonParser(
"%prog filename vos:rootNode/destination",
description=("Copy a file or directory (always recursive) to a "
"VOSpace location. Try to be UNIX like. "))
parser.add_option("--exclude",
default=None,
help="exclude files that match pattern")
parser.add_option(
"--include",
default=None,
help="only include files that match pattern (overrides exclude)")
parser.add_option("-i",
"--interrogate",
action="store_true",
help="Ask before overwriting files")
parser.add_option(
"--overwrite",
action="store_true",
help=
"don't check destination MD5, just overwrite even if source matches destination"
)
parser.add_option(
"--quick",
action="store_true",
help=
"Use default CADC urls, for speed. Will fail if CADC changes data storage mechanism",
default=False)
parser.add_option(
"-L",
"--follow-links",
help=
"Should recursive copy follow symbolic links? Default is to not follow links.",
action="store_true",
default=False)
parser.add_option("--ignore",
action="store_true",
default=False,
help="ignore errors and continue with recursive copy")
(opt, args) = parser.parse_args()
parser.process_informational_options()
if len(args) < 2:
parser.error("Must give a source and a destination")
dest = args.pop()
this_destination = dest
if dest[0:4] != 'vos:':
dest = os.path.abspath(dest)
client = vos.Client(vospace_certfile=opt.certfile,
vospace_token=opt.token,
transfer_shortcut=opt.quick)
exit_code = 0
cutout_pattern = re.compile(
r'(.*?)(?P<cutout>(\[[\-\+]?[\d\*]+(:[\-\+]?[\d\*]+)?(,[\-\+]?[\d\*]+(:[\-\+]?[\d\*]+)?)?\])+)$'
)
ra_dec_cutout_pattern = re.compile("([^\(\)]*?)"
"(?P<cutout>\("
"(?P<ra>[\-\+]?\d*(\.\d*)?),"
"(?P<dec>[\-\+]?\d*(\.\d*)?),"
"(?P<rad>\d*(\.\d*)?)\))?")
# Warnings:
# vcp destination specified with a trailing '/' implies ContainerNode
#
# If destination has trailing '/' and exists but is a DataNode then
# error message is returned: "Invalid Argument (target node is not a DataNode)"
#
# vcp currently only works on the CADC VOSpace server.
# Version: %s """ % (version.version)
def size(filename):
if filename[0:4] == "vos:":
return client.size(filename)
return os.stat(filename).st_size
def create(filename):
if filename[0:4] == "vos:":
return client.create(vos.Node(client.fix_uri(filename)))
else:
open(filename, 'w').close()
pass
def get_node(filename, limit=None):
"""Get node, from cache if possible"""
return client.get_node(filename, limit=limit)
# here are a series of methods that choose between calling the system version or the vos version of various
# function, based on pattern matching.
# TODO: Put these function in a separate module.
def isdir(filename):
logging.debug("Doing an isdir on %s" % filename)
if filename[0:4] == "vos:":
return client.isdir(filename)
else:
return os.path.isdir(filename)
def islink(filename):
logging.debug("Doing an islink on %s" % filename)
if filename[0:4] == "vos:":
try:
return get_node(filename).islink()
except:
return False
else:
return os.path.islink(filename)
def access(filename, mode):
"""
Check if the file can be accessed.
@param filename: name of file or vospace node to check
@param mode: what access mode should be checked: see os.access
@return: True/False
"""
logging.debug("checking for access %s " % filename)
if filename[0:4] == "vos:":
try:
node = get_node(filename, limit=0)
return node is not None
except (exceptions.NotFoundException,
exceptions.ForbiddenException,
exceptions.UnauthorizedException) as ex:
return False
else:
return os.access(filename, mode)
def listdir(dirname):
"""Walk through the directory structure a al os.walk"""
logging.debug("getting a dirlist %s " % dirname)
if dirname[0:4] == "vos:":
return client.listdir(dirname, force=True)
else:
return os.listdir(dirname)
def mkdir(filename):
logging.debug("Making directory %s " % filename)
if filename[0:4] == 'vos:':
return client.mkdir(filename)
else:
return os.mkdir(filename)
def lglob(pathname):
"""
Call system glob if not vos path.
@param pathname: the pathname (aka pattern) to glob with.
@return: list of matched filenames.
"""
if pathname[0:4] == "vos:":
return client.glob(pathname)
else:
return glob.glob(pathname)
def get_md5(filename):
logging.debug("getting the MD5 for %s" % filename)
if filename[0:4] == 'vos:':
return get_node(filename).props.get('MD5', vos.ZERO_MD5)
else:
return md5_cache.MD5_Cache.computeMD5(filename)
def lglob(pathname):
if pathname[0:4] == "vos:":
return client.glob(pathname)
else:
return glob.glob(pathname)
def copy(source_name,
destination_name,
exclude=None,
include=None,
interrogate=False,
overwrite=False,
ignore=False):
"""
:param source_name:
:param destination_name:
:param exclude:
:param include:
:return: :raise e:
"""
global exit_code
## determine if this is a directory we are copying so need to be recursive
try:
if not opt.follow_links and islink(source_name):
#return link(source_name, destination_name)
logging.info(
"{}: Skipping (symbolic link)".format(source_name))
return
if isdir(source_name):
## make sure the destination exists...
if not isdir(destination_name):
mkdir(destination_name)
## for all files in the current source directory copy them to the destination directory
for filename in listdir(source_name):
logging.debug("%s -> %s" % (filename, source_name))
copy(os.path.join(source_name, filename),
os.path.join(destination_name, filename), exclude,
include, interrogate, overwrite, ignore)
else:
if interrogate:
if access(destination_name, os.F_OK):
sys.stderr.write(
"File %s exists. Overwrite? (y/n): " %
destination_name)
ans = sys.stdin.readline().strip()
if ans != 'y':
raise Exception("File exists")
if not overwrite and access(destination_name, os.F_OK):
### check if the MD5 of dest and source mathc, if they do then skip
if get_md5(destination_name) == get_md5(source_name):
logging.info("%s matches %s, skipping" %
(source_name, destination_name))
return
if not access(os.path.dirname(destination_name), os.F_OK):
raise OSError(
errno.EEXIST, "vcp: ContainerNode %s does not exist" %
os.path.dirname(destination_name))
if not isdir(os.path.dirname(destination_name)) and not islink(
os.path.dirname(destination_name)):
raise OSError(
errno.ENOTDIR,
"vcp: %s is not a ContainerNode or LinkNode" %
os.path.dirname(destination_name))
skip = False
if exclude is not None:
for thisIgnore in exclude.split(','):
if not destination_name.find(thisIgnore) < 0:
skip = True
continue
if include is not None:
skip = True
for thisIgnore in include.split(','):
if not destination_name.find(thisIgnore) < 0:
skip = False
continue
if not skip:
logging.info("%s -> %s " % (source_name, destination_name))
niters = 0
while not skip:
try:
logging.debug("Starting call to copy")
client.copy(source_name,
destination_name,
send_md5=True)
logging.debug("Call to copy returned")
break
except Exception as client_exception:
logging.debug("{}".format(client_exception))
if getattr(client_exception, 'errno', -1) == 104:
# 104 is connection reset by peer. Try again on this error
logging.warning(str(client_exception))
exit_code = getattr(client_exception, 'errno', -1)
elif getattr(client_exception, 'errno',
-1) == errno.EIO:
# retry on IO errors
logging.warning(
"{0}: Retrying".format(client_exception))
pass
elif ignore:
if niters > 100:
logging.error(
"%s (skipping after %d attempts)" %
(str(client_exception), niters))
skip = True
else:
logging.error("%s (retrying)" %
str(client_exception))
time.sleep(5)
niters += 1
else:
raise client_exception
except Exception as ex:
logging.debug("{}".format(ex))
raise
except OSError as os_exception:
logging.debug(str(os_exception))
if getattr(os_exception, 'errno', -1) == errno.EINVAL:
# not a valid uri, just skip those...
logging.warning("%s: Skipping" % str(os_exception))
else:
raise os_exception
# main loop
try:
for source_pattern in args:
# define this empty cutout string. Then we strip possible cutout strings off the end of the
# pattern before matching. This allows cutouts on the vos service.
# The shell does pattern matching for local files, so don't run glob on local files.
if source_pattern[0:4] != "vos:":
sources = [source_pattern]
else:
cutout_match = cutout_pattern.search(source_pattern)
cutout = None
if cutout_match is not None:
source_pattern = cutout_match.group(1)
cutout = cutout_match.group('cutout')
else:
ra_dec_match = ra_dec_cutout_pattern.search(source_pattern)
if ra_dec_match is not None:
cutout = ra_dec_match.group('cutout')
logging.debug("cutout: {}".format(cutout))
sources = lglob(source_pattern)
if cutout is not None:
# stick back on the cutout pattern if there was one.
sources = [s + cutout for s in sources]
for source in sources:
if source[0:4] != "vos:":
source = os.path.abspath(source)
# the source must exist, of course...
if not access(source, os.R_OK):
raise Exception("Can't access source: %s " % source)
if not opt.follow_links and islink(source):
logging.info("{}: Skipping (symbolic link)".format(source))
continue
# copying inside VOSpace not yet implemented
if source[0:4] == 'vos:' and dest[0:4] == 'vos:':
raise Exception(
"Can not (yet) copy from VOSpace to VOSpace.")
this_destination = dest
if isdir(source):
if not opt.follow_links and islink(source):
continue
logging.debug("%s is a directory or link to one" % source)
# To mimic unix fs behaviours if copying a directory and
# the destination directory exists then the actual
# destination in a recursive copy is the destination +
# source basename.
# This has an odd behaviour if more than one directory is given as a source and the copy is recursive.
if access(dest, os.F_OK):
if not isdir(dest):
raise Exception(
"Can't write a directory (%s) to a file (%s)" %
(source, dest))
# directory exists so we append the end of source to that (UNIX behaviour)
this_destination = os.path.normpath(
os.path.join(dest, os.path.basename(source)))
elif len(args) > 1:
raise Exception(
"vcp can not copy multiple things into a non-existent location (%s)"
% dest)
elif dest[-1] == '/' or isdir(dest):
# we're copying into a directory
this_destination = os.path.join(dest,
os.path.basename(source))
copy(source,
this_destination,
exclude=opt.exclude,
include=opt.include,
interrogate=opt.interrogate,
overwrite=opt.overwrite,
ignore=opt.ignore)
except KeyboardInterrupt as ke:
logging.info("Received keyboard interrupt. Execution aborted...\n")
exit_code = getattr(ke, 'errno', -1)
except ParseError as pe:
exit_code = errno.EREMOTE
logging.error("Failure at server while copying {0} -> {1}".format(
source, dest))
except Exception as e:
message = str(e)
if re.search('NodeLocked', str(e)) is not None:
logging.error(
"Use vlock to unlock the node before copying to %s." %
this_destination)
elif getattr(e, 'errno', -1) == errno.EREMOTE:
logging.error(
"Failure at remote server while copying {0} -> {1}".format(
source, dest))
else:
logging.debug("Exception throw: %s %s" % (type(e), str(e)))
logging.debug(traceback.format_exc())
logging.error(message)
exit_code = getattr(e, 'errno', -1)
sys.exit(exit_code)