def vrmdir(): parser = CommonParser(description=DESCRIPTION) parser.add_argument('nodes', help="Container nodes to delete from VOSpace", nargs='+') args = parser.parse_args() set_logging_level_from_args(args) try: for container_node in args.nodes: if not vos.is_remote_file(container_node): raise ValueError( "{} is not a valid VOSpace handle".format(container_node)) client = vos.Client( vospace_certfile=args.certfile, vospace_token=args.token) if client.isdir(container_node): logging.info("deleting {}".format(container_node)) client.delete(container_node) else: raise ValueError( "{} is a not a container node".format(container_node)) except Exception as ex: exit_on_exception(ex)
def __init__(self, this_queue): super(ThreadCopy, self).__init__() self.client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token) self.queue = this_queue self.filesSent = 0 self.filesSkipped = 0 self.bytesSent = 0 self.bytesSkipped = 0 self.filesErrored = 0
def getCutoutImage(ra, dec, tile, hw=1750): '''Get a large cutout of CFIS tile. Input images are 10000x10000. This function makes an (11 arcmin x 11 arcmin) of the CFIS tile. It finds the (row,col) position of the target objID using ra,dec (degrees). It then checks for boundary conditions on the cutout region and adapts. The final cutout is 3501x3501 pixels, not always centered on the target. `hw` is the half-width of the cutout size. Returns cutout filename.''' warnings.filterwarnings('ignore') client = vos.Client() wcsName = 'WCS-{}'.format(tile) cutoutName = 'Cutout-{}'.format(tile) # get wcs file while not os.access(wcsName, 0): try: client.copy(source='vos:cfis/tiles_DR2/{}[1:2,1:2]'.format(tile), destination=wcsName) except: time.sleep(10) # get wcs mapping wcs = WCS(wcsName) # determine column and row position in image colc, rowc = wcs.all_world2pix(ra, dec, 1, ra_dec_order=True) # convert to integers colc, rowc = int(np.around(colc)), int(np.around(rowc)) # remove temporary file if os.access(wcsName, 0): os.remove(wcsName) # get boundaries if colc - hw < 1: colc_m = 1 colc_p = 2 * hw + 1 elif colc + hw > 10000: colc_m = 10000 - (2 * hw) colc_p = 10000 else: colc_m = colc - hw colc_p = colc + hw if rowc - hw < 1: rowc_m = 1 rowc_p = 2 * hw + 1 elif rowc + hw > 10000: rowc_p = 10000 rowc_m = 10000 - (2 * hw) else: rowc_m = rowc - hw rowc_p = rowc + hw # get full file while not os.access(cutoutName, 0): try: client.copy(source='vos:cfis/tiles_DR2/{}[{}:{},{}:{}]'.format( tile, colc_m, colc_p, rowc_m, rowc_p), destination=cutoutName) except: time.sleep(10) return cutoutName
def _cat(vospace_uri, cert_filename=None): """Cat out the given uri.""" fh = None try: if vospace_uri[0:4] == "vos:": fh = vos.Client(vospace_certfile=cert_filename).open(vospace_uri, view='data') else: fh = open(vospace_uri, 'r') sys.stdout.write(fh.read()) finally: if fh: fh.close()
def vln(): usage = """ vln vos:VOSpaceSource vos:VOSpaceTarget examples: vln vos:vospace/junk.txt vos:vospace/linkToJunk.txt vln vos:vospace/directory vos:vospace/linkToDirectory vln http://external.data.source vos:vospace/linkToExternalDataSource """ parser = CommonParser(usage) if len(sys.argv) == 1: parser.print_help() sys.exit() (opt, args) = parser.parse_args() parser.process_informational_options() logger = logging.getLogger() logger.setLevel(parser.log_level) logger.addHandler(logging.StreamHandler()) if len(args) != 2: parser.error("You must specify a source file and a target file") sys.exit(-1) if args[1][0:4] != "vos:": parser.error("The target to source must be in vospace") sys.exit(-1) logger.debug("Connecting to vospace using certificate %s" % opt.certfile) try: client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token) except Exception as e: logger.error("VOS connection failed: {0}".format(e)) sys.exit(-1) try: client.link(args[0], args[1]) except Exception as e: logger.error("Failed to make link from %s to %s" % (args[0], args[1])) logger.error(getattr(e, 'strerror', 'Unknown Error')) sys.exit(-1)
def vrm(): usage=""" vrm vos:/root/node -- deletes a data node Version: %s """ % (version.version) parser=CommonParser(usage) if len(sys.argv) == 1: parser.print_help() sys.exit() (opt, args)=parser.parse_args() parser.process_informational_options() logger = logging.getLogger() logger.setLevel(parser.log_level) try: client=vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token) except Exception as e: logger.error("Connection failed: %s" % (str(e))) exit_code = getattr(e, 'errno', -1) sys.exit(exit_code) for arg in args: if arg[0:4]!="vos:": logger.error("%s is not a valid VOSpace handle" % (arg)) try: if client.isfile(arg) or client.get_node(arg).islink(): logger.info("deleting %s" %(arg)) client.delete(arg) elif client.isdir(arg): logger.error("%s is a directory" % (arg)) elif client.access(arg): logger.info("deleting link %s" %(arg)) client.delete(arg) else: logger.error("%s file not found" % (arg)) sys.exit(-1) except Exception as e: import re if re.search('NodeLocked',str(e)) != None: logger.error("Use vlock to unlock %s before deleting." % (arg)) exit_code = getattr(e, 'errno', -1) logger.error("Failed trying to delete {0}: {1}".format( arg, str(e))) sys.exit(exit_code)
def vmkdir(): usage=""" vmkdir vos:/root/node -- creates a new directory (ContainerNode) called node at vospace root Version: %s """ % (version.version) parser=CommonParser(usage) parser.add_option("-p",action="store_true",help="Create intermediate directories as required.") if len(sys.argv) == 1: parser.print_help() sys.exit() (opt,args)=parser.parse_args() parser.process_informational_options() if len(args)>1: parser.error("Only one directory can be built per call") logging.info("Creating ContainerNode (directory) %s" % ( args[0])) try: client=vos.Client(vospace_certfile=opt.certfile,vospace_token=opt.token) dirNames=[] thisDir = args[0] if opt.p: while not client.access(thisDir): dirNames.append(os.path.basename(thisDir)) thisDir = os.path.dirname(thisDir) while len(dirNames) > 0: thisDir = os.path.join(thisDir,dirNames.pop()) client.mkdir(thisDir) else: client.mkdir(thisDir) except Exception as e: logging.error(str(e)) sys.exit(getattr(e, 'errno', -1)) sys.exit(0)
def vrmdir(): usage=""" vrmdir vos:/root/node -- deletes a container node Version: %s """ % (version.version) parser = CommonParser(usage) if len(sys.argv) == 1: parser.print_help() sys.exit() (opt, args) = parser.parse_args() parser.process_informational_options() try: client=vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token) except Exception as e: logging.error("Connection failed: %s" % (str(e))) sys.exit(e.__getattribute__('errno', -1)) try: for arg in args: if arg[0:4]!="vos:": logging.error("%s is not a valid VOSpace handle" % (arg)) sys.exit(-1) if client.isdir(arg): logging.info("deleting %s" %(arg)) client.delete(arg) elif client.isfile(arg): logging.error("%s is a file" % (arg)) sys.exit(-1) else: logging.error("%s file not found" % (arg)) sys.exit(-1) except Exception as e: import re if re.search('NodeLocked', str(e)) != None: logging.error("Use vlock to unlock %s before removing." %(arg)) logging.error("Connection failed: %s" % (str(e))) sys.exit(-1)
def vlock(): signal.signal(signal.SIGINT, signal_handler) parser = CommonParser() parser.add_option("--lock", action="store_true", help="Lock the node") parser.add_option("--unlock", action="store_true", help="unLock the node") (opt, args) = parser.parse_args() parser.process_informational_options() logger = logging.getLogger() logger.setLevel(parser.log_level) logger.addHandler(logging.StreamHandler()) lock = None if opt.lock: lock = 'true' elif opt.unlock: lock = 'false' exit_code = 0 try: client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token) node = client.get_node(args[0]) if opt.lock or opt.unlock: lock = not opt.unlock and opt.lock node.is_locked = lock client.update(node) else: print(node.is_locked) if not node.is_locked: exit_code = -1 except KeyboardInterrupt: exit_code = -1 except Exception as e: logger.error(str(e)) exit_code = -1 sys.exit(exit_code)
def vsync(): global_md5_cache = None def signal_handler(h_stream, h_frame): logging.debug("{} {}".format(h_stream, h_frame)) logging.critical("Interrupt\n") sys.exit(-1) # handle interrupts nicely signal.signal(signal.SIGINT, signal_handler) start_time = time.time() parser = CommonParser(description=DESCRIPTION) parser.add_option('files', nargs='+', help='Files to copy to VOSpace') parser.add_option('destination', help='VOSpace location to sync files to') parser.add_option('--ignore-checksum', action="store_true", help='dont check MD5 sum, use size and time instead') parser.add_option('--cache_nodes', action='store_true', help='cache node MD5 sum in an sqllite db') parser.add_option('--cache_filename', help="Name of file to use for node cache", default="{}/.config/vos/node_cache.db".format(HOME)) parser.add_option('--recursive', '-r', help="Do a recursive sync", action="store_true") parser.add_option('--nstreams', '-n', type=int, help="Number of streams to run (MAX: 30)", default=5) parser.add_option( '--exclude', help="ignore directories or files containing this pattern", default=None) parser.add_option('--include', help="only include files matching this pattern", default=None) parser.add_option( '--overwrite', help=("overwrite copy on server regardless of modification/size/md5 " "checks"), action="store_true") opt = parser.parse_args() set_logging_level_from_args(opt) if opt.nstreams > 30: parser.error("Maximum of 30 streams exceeded") if opt.cache_nodes: global_md5_cache = md5_cache.MD5Cache(cache_db=opt.cache_filename) destination = opt.destination if not vos.is_remote_file(destination): parser.error("Only allows sync FROM local copy TO VOSpace") # Currently we don't create nodes in sync and we don't sync onto files logging.info("Connecting to VOSpace") client = vos.Client( vospace_certfile=opt.certfile, vospace_token=opt.token) logging.info("Confirming Destination is a directory") dest_is_dir = client.isdir(destination) queue = JoinableQueue(maxsize=10 * opt.nstreams) good_dirs = [] node_dict = {} def compute_md5(this_filename, block_size=None): """ Read through a file and compute that files MD5 checksum. :param this_filename: name of the file on disk :param block_size: number of bytes to read into memory, defaults to 2**19 bytes :return: md5 as a hexadecimal string """ block_size = block_size is None and 2 ** 19 or block_size return md5_cache.MD5Cache.compute_md5(this_filename, block_size=block_size) def file_md5(this_filename): import os md5 = None if global_md5_cache is not None: md5 = global_md5_cache.get(this_filename) if md5 is None or md5[2] < os.stat(this_filename).st_mtime: md5 = compute_md5(this_filename) if global_md5_cache is not None: stat = os.stat(this_filename) global_md5_cache.update(this_filename, md5, stat.st_size, stat.st_mtime) else: md5 = md5[0] return md5 class ThreadCopy(Process): def __init__(self, this_queue): super(ThreadCopy, self).__init__() self.client = vos.Client( vospace_certfile=opt.certfile, vospace_token=opt.token) self.queue = this_queue self.filesSent = 0 self.filesSkipped = 0 self.bytesSent = 0 self.bytesSkipped = 0 self.filesErrored = 0 def run(self): while True: (current_source, current_destination) = self.queue.get() requeue = (current_source, current_destination) src_md5 = None stat = os.stat(current_source) if not opt.ignore_checksum and not opt.overwrite: src_md5 = file_md5(current_source) if not opt.overwrite: # Check if the file is the same try: node_info = None if opt.cache_nodes: node_info = global_md5_cache.get( current_destination) if node_info is None: logging.debug("Getting node info from VOSpace") logging.debug(str(node_dict.keys())) logging.debug(str(current_destination)) node = self.client.get_node(current_destination, limit=None) current_destination_md5 = node.props.get( 'MD5', 'd41d8cd98f00b204e9800998ecf8427e') current_destination_length = node.attr['st_size'] current_destination_time = node.attr['st_ctime'] if opt.cache_nodes: global_md5_cache.update( current_destination, current_destination_md5, current_destination_length, current_destination_time) else: current_destination_md5 = node_info[0] current_destination_length = node_info[1] current_destination_time = node_info[2] logging.debug("Destination MD5: {}".format( current_destination_md5)) if ((not opt.ignore_checksum and src_md5 == current_destination_md5) or (opt.ignore_checksum and current_destination_time >= stat.st_mtime and current_destination_length == stat.st_size)): logging.info("skipping: %s matches %s" % ( current_source, current_destination)) self.filesSkipped += 1 self.bytesSkipped += current_destination_length self.queue.task_done() continue except (transfer_exceptions.AlreadyExistsException, transfer_exceptions.NotFoundException): pass logging.info( "%s -> %s" % (current_source, current_destination)) try: self.client.copy(current_source, current_destination, send_md5=True) node = self.client.get_node(current_destination, limit=None) current_destination_md5 = node.props.get( 'MD5', 'd41d8cd98f00b204e9800998ecf8427e') current_destination_length = node.attr['st_size'] current_destination_time = node.attr['st_ctime'] if opt.cache_nodes: global_md5_cache.update(current_destination, current_destination_md5, current_destination_length, current_destination_time) self.filesSent += 1 self.bytesSent += stat.st_size except (IOError, OSError) as exc: logging.error( "Error writing {} to server, skipping".format( current_source)) logging.error(str(exc)) import re if re.search('NodeLocked', str(exc)) is not None: logging.error( ("Use vlock to unlock the node before syncing " "to {}").format(current_destination)) try: if exc.errno == 104: self.queue.put(requeue) except Exception as e2: logging.error("Error during requeue") logging.error(str(e2)) pass self.filesErrored += 1 pass self.queue.task_done() def mkdirs(directory): """Recursively make all nodes in the path to directory. :param directory: str, vospace location of ContainerNode (directory) to make :return: """ logging.debug("%s %s" % (directory, str(good_dirs))) # if we've seen this before skip it. if directory in good_dirs: return # try and make a new directory and return # failure indicates we should see if subdirectories exist try: client.mkdir(directory) logging.info("Made directory {}".format(directory)) good_dirs.append(directory) return except transfer_exceptions.AlreadyExistsException: pass # OK, must already have existed, add to list good_dirs.append(directory) return def copy(current_source, current_destination): """ Copy current_source from local file system to current_destination. :param current_source: name of local file :param current_destination: name of localtion on VOSpace to copy file to (includes filename part) :return: None """ # strip down current_destination until we find a part that exists # and then build up the path. if os.path.islink(current_source): logging.error("{} is a link, skipping".format(current_source)) return if not os.access(current_source, os.R_OK): logging.error( "Failed to open file {}, skipping".format(current_source)) return import re if re.match(r'^[A-Za-z0-9._\-();:&*$@!+=/]*$', current_source) is None: logging.error( "filename %s contains illegal characters, skipping" % current_source) return dirname = os.path.dirname(current_destination) mkdirs(dirname) if opt.include is not None and not re.search(opt.include, current_source): return queue.put((current_source, current_destination), timeout=3600) def start_streams(no_streams): list_of_streams = [] for i in range(no_streams): logging.info("Launching VOSpace connection stream %d" % i) t = ThreadCopy(queue) t.daemon = True t.start() list_of_streams.append(t) return list_of_streams def build_file_list(base_path, destination_root='', recursive=False, ignore=None): """Build a list of files that should be copied into VOSpace""" spinner = ['-', '\\', '|', '/', '-', '\\', '|', '/'] count = 0 for (root, dirs, filenames) in os.walk(base_path): for this_dirname in dirs: if not recursive: continue this_dirname = os.path.join(root, this_dirname) skip = False if ignore is not None: for thisIgnore in ignore.split(','): if not this_dirname.find(thisIgnore) < 0: logging.info("excluding: %s " % this_dirname) skip = True continue if skip: continue cprefix = os.path.commonprefix((base_path, this_dirname)) this_dirname = os.path.normpath( destination_root + "/" + this_dirname[len(cprefix):]) mkdirs(this_dirname) for thisfilename in filenames: srcfilename = os.path.normpath( os.path.join(root, thisfilename)) skip = False if ignore is not None: for thisIgnore in ignore.split(','): if not srcfilename.find(thisIgnore) < 0: logging.info("excluding: %s " % srcfilename) skip = True continue if skip: continue cprefix = os.path.commonprefix((base_path, srcfilename)) destfilename = os.path.normpath( destination_root + "/" + srcfilename[len(cprefix):]) this_dirname = os.path.dirname(destfilename) mkdirs(this_dirname) count += 1 if opt.verbose: sys.stderr.write( "Building list of files to transfer %s\r" % ( spinner[count % len(spinner)])) copy(srcfilename, destfilename) if not recursive: return return streams = start_streams(opt.nstreams) # build a complete file list given all the things on the command line for filename in opt.files: filename = os.path.abspath(filename) this_root = destination if os.path.isdir(filename): if filename[-1] != "/": if os.path.basename(filename) != os.path.basename(destination): this_root = os.path.join(destination, os.path.basename(filename)) mkdirs(this_root) node_dict[this_root] = client.get_node(this_root, limit=None) try: build_file_list(filename, destination_root=this_root, recursive=opt.recursive, ignore=opt.exclude) except Exception as e: logging.error(str(e)) logging.error("ignoring error") elif os.path.isfile(filename): if dest_is_dir: this_root = os.path.join(destination, os.path.basename(filename)) copy(filename, this_root) else: logging.error("%s: No such file or directory." % filename) logging.info( ("Waiting for transfers to complete " r"******** CTRL-\ to interrupt ********")) queue.join() end_time = time.time() bytes_sent = 0 files_sent = 0 bytes_skipped = 0 files_skipped = 0 files_erred = 0 for stream in streams: bytes_sent += stream.bytesSent bytes_skipped += stream.bytesSkipped files_sent += stream.filesSent files_skipped += stream.filesSkipped files_erred += stream.filesErrored logging.info("==== TRANSFER REPORT ====") if bytes_sent > 0: rate = bytes_sent / (end_time - start_time) / 1024.0 logging.info("Sent %d files (%8.1f kbytes @ %8.3f kBytes/s)" % ( files_sent, bytes_sent / 1024.0, rate)) speed_up = (bytes_skipped + bytes_sent) / bytes_sent logging.info( "Speedup: %f (skipped %d files)" % (speed_up, files_skipped)) if bytes_sent == 0: logging.info("No files needed sending ") if files_erred > 0: logging.info( "Error transferring %d files, please try again" % files_erred)
def vls(): parser = CommonParser(usage, description=description, add_help_option=False) parser.add_option("--help", action="store_true") parser.add_option("-l", "--long", action="store_true", help="verbose listing sorted by name") parser.add_option("-g", "--group", action="store_true", help="display group read/write information") parser.add_option("-h", "--human", action="store_true", help="make sizes human readable", default=False) parser.add_option("-S", "--Size", action="store_true", help="sort files by size", default=False) parser.add_option("-r", "--reverse", action="store_true", help="reverse the sort order", default=False) parser.add_option("-t", "--time", action="store_true", help="sort by time copied to VOSpace") (opt, args) = parser.parse_args() # We disabled -h/--help so vls can have a -h option. # Here we re-enable it but only with --help if opt.help: parser.print_help() sys.exit(0) parser.process_informational_options() if not len(args) > 0: parser.error("missing VOSpace argument") logger = logging.getLogger() logger.setLevel(parser.log_level) sortKey = (opt.time and "date") or (opt.Size and "size") or False try: client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token) except Exception as e: logger.error("Connection failed: %s" % (str(e))) sys.exit(-1) columns = ['permissions'] if opt.long: columns.extend(['creator']) columns.extend(['readGroup', 'writeGroup', 'isLocked', 'size', 'date']) def get_terminal_size(): """Get the size of the terminal @return: tuple(int, int) giving the row/column of the terminal screen. """ def ioctl_gwinsz(fd): """ @param fd: A file descriptor that points at the Screen Parameters @return: the termios screeen structure unpacked into an array. """ try: import fcntl import termios import struct this_cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) except: return None return this_cr cr = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2) if not cr: try: td = os.open(os.ctermid(), os.O_RDONLY) cr = ioctl_gwinsz(td) td.close() except: pass if not cr: try: cr = (os.environ['LINES'], os.environ['COLUMNS']) except: cr = (25, 80) return int(cr[1]), int(cr[0]) def size_format(size): """Format a size value for listing""" try: size = float(size) except Exception as ex: logger.debug(str(ex)) size = 0.0 if opt.human: size_unit = ['B', 'K', 'M', 'G', 'T'] try: length = float(size) scale = int(math.log(length) / math.log(1024)) length = "%.0f%s" % (length / (1024.0 ** scale), size_unit[scale]) except: length = str(int(size)) else: length = str(int(size)) return "%12s " % length def date_format(epoch): """given a time object return a unix-ls like formatted string""" time_tuple = time.localtime(epoch) if time.localtime().tm_year != time_tuple.tm_year: return time.strftime('%b %d %Y ', time_tuple) return time.strftime('%b %d %H:%S ', time_tuple) formats = {'permissions': lambda value: "{:<11}".format(value), 'creator': lambda value: " {:<20}".format(value), 'readGroup': lambda value: " {:<15}".format(value.replace(CADC_GMS_PREFIX, "")), 'writeGroup': lambda value: " {:<15}".format(value.replace(CADC_GMS_PREFIX, "")), 'isLocked': lambda value: " {:<8}".format("", "LOCKED")[value == "true"], 'size': size_format, 'date': date_format} for node in args: try: if not node[0:4] == "vos:" : raise OSError(errno.EBADF, "Invalid node name", node) logger.debug("getting listing of: %s" % str(node)) infoList = client.get_info_list(node) except SSLError as error: logger.error(str(error)) sys.exit(errno.EPERM) except Exception as e: logger.error(str(e)) err_no = getattr(e, 'errno', None) err_no = err_no is not None and err_no or errno.ENOMSG sys.exit(err_no) if sortKey: try: sorted_list = sorted(infoList, key=lambda name: name[1][sortKey], reverse=not opt.reverse) except: sorted_list = infoList finally: infoList = sorted_list for item in infoList: name_string = item[0] if opt.long or opt.group: for col in columns: value = item[1].get(col, None) value = value is not None and value or "" if col in formats: sys.stdout.write(formats[col](value)) if item[1]["permissions"][0] == 'l': name_string = "%s -> %s" % (name_string, item[1]['target']) sys.stdout.write("%s\n" % name_string)
def vtag(): usage = """ vtag [options] node [key[=value] [key[=value] ...]] Version: %s """ % (version.version) signal.signal(signal.SIGINT, signal_handler) parser = CommonParser(usage) parser.add_option('--remove', action="store_true", help='remove the listed property') # Not yet supported # parser.add_option("-R", "--recursive", action='store_const', const=True, # help="Recursive set read/write properties") (opt, args) = parser.parse_args() parser.process_informational_options() if len(args) == 0: parser.error("no vospace supplied") logger = logging.getLogger() logger.setLevel(parser.log_level) logger.addHandler(logging.StreamHandler()) ## the node should be the first argument, the rest should contain the key/val pairs node = args.pop(0) props = [] if opt.remove: ## remove signified by blank value in key=value listing for prop in args: if '=' not in prop: prop += "=" props.append(prop) else: props = args try: """Lists/sets values of node properties based on context (length of props). node: vos.vos.Node object props: dictionary of properties to set. If dict is zero length, list them all. """ client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token) node = client.get_node(node) if len(props) == 0: # print all properties for key in node.props: print(key, node.props[key]) return pprint.pprint(node.props) changed = False for prop in props: prop = prop.split('=') if len(prop) == 1: # get one property logger.debug("just print this one out %s" % (prop[0])) pprint.pprint(node.props.get(prop[0], None)) elif len(prop) == 2: # update properties key, value = prop logger.debug("%s %s" % (len(key), len(value))) if len(value) == 0: value = None logger.debug("%s: %s -> %s" % (key, node.props.get(key, None), value)) if value != node.props.get(key, None): node.props[key] = value changed = True else: raise ValueError( "Illigal keyword of value character ('=') used: %s" % ('='.join(prop))) if changed: client.add_props(node) return 0 except KeyboardInterrupt: logger.debug("Received keyboard interrupt. Execution aborted...\n") pass except Exception as e: logger.error(str(e)) sys.exit(-1)
def vsync(): def signal_handler(signal, frame): logger.critical("Interupt\n") sys.exit(-1) usage = """ vsync [options] files vos:Destination/ Version: %s """ % (version.version) # handle interupts nicely signal.signal(signal.SIGINT, signal_handler) startTime = time.time() parser = CommonParser(usage) parser.add_option('--ignore-checksum', action="store_true", help='dont check MD5 sum, use size and time instead') parser.add_option('--cache_nodes', action='store_true', help='cache node MD5 sum in an sqllite db') parser.add_option('--recursive', '-r', help="Do a recursive sync", action="store_true") parser.add_option('--nstreams', '-n', type=int, help="Number of streams to run (MAX: 30)", default=1) parser.add_option( '--exclude', help="ignore directories or files containing this pattern", default=None) parser.add_option('--include', help="only include files matching this pattern", default=None) parser.add_option( '--overwrite', help= "overwrite copy on server regardless of modification/size/md5 checks", action="store_true") parser.add_option( '--load_test', action="store_true", help= "Used to stress test the VOServer, also set --nstreams to a large value" ) if len(sys.argv) == 1: parser.print_help() sys.exit() (opt, args) = parser.parse_args() parser.process_informational_options() logger = logging.getLogger() logger.setLevel(parser.log_level) logger.addHandler(logging.StreamHandler()) if len(args) < 2: parser.error( "requires one or more source files and a single destination directory" ) if opt.nstreams > 30 and not opt.load_test: parser.error("Maximum of 30 streams exceeded") if opt.cache_nodes: from vos import md5_cache md5Cache = md5_cache.MD5_Cache() dest = args.pop() if dest[0:4] != "vos:": parser.error("Only allows sync FROM local copy TO VOSpace") ## Currently we don't create nodes in sync and we don't sync onto files logger.info("Connecting to VOSpace") client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token) logger.info("Confirming Destination is a directory") destIsDir = client.isdir(dest) queue = JoinableQueue(maxsize=10 * opt.nstreams) goodDirs = [] nodeDict = {} def computeMD5(filename, block_size=None): """ Read through a file and compute that files MD5 checksum. :param filename: name of the file on disk :param block_size: number of bytes to read into memory, defaults to 2**19 bytes :return: md5 as a hexadecimal string """ block_size = block_size is None and 2**19 or block_size md5 = hashlib.md5() r = open(filename, 'r') while True: buf = r.read(block_size) if len(buf) == 0: break md5.update(buf) r.close() return md5.hexdigest() def fileMD5(filename): import os md5 = None if opt.cache_nodes: md5 = md5Cache.get(filename) if md5 is None or md5[2] < os.stat(filename).st_mtime: md5 = computeMD5(filename) if opt.cache_nodes: stat = os.stat(filename) md5Cache.update(filename, md5, stat.st_size, stat.st_mtime) else: md5 = md5[0] return md5 class ThreadCopy(Process): def __init__(self, queue, client): super(ThreadCopy, self).__init__() self.client = client self.queue = queue self.filesSent = 0 self.filesSkipped = 0 self.bytesSent = 0 self.bytesSkipped = 0 self.filesErrored = 0 def run(self): while True: (src, dest) = self.queue.get() requeue = (src, dest) srcMD5 = None stat = os.stat(src) if not opt.ignore_checksum and not opt.overwrite: srcMD5 = fileMD5(src) if not opt.overwrite: # Check if the file is the same try: nodeInfo = None if opt.cache_nodes: nodeInfo = md5Cache.get(dest) if nodeInfo is None: logger.debug("Getting node info from VOSpace") logger.debug(str(nodeDict.keys())) logger.debug(str(dest)) node = self.client.get_node(dest, limit=None) destMD5 = node.props.get( 'MD5', 'd41d8cd98f00b204e9800998ecf8427e') destLength = node.attr['st_size'] destTime = node.attr['st_ctime'] if opt.cache_nodes: md5Cache.update(dest, destMD5, destLength, destTime) else: destMD5 = nodeInfo[0] destLength = nodeInfo[1] destTime = nodeInfo[2] logger.debug("Dest MD5: %s " % (destMD5)) if (not opt.ignore_checksum and srcMD5 == destMD5) or ( opt.ignore_checksum and destTime >= stat.st_mtime and destLength == stat.st_size): logger.info("skipping: %s matches %s" % (src, dest)) self.filesSkipped += 1 self.bytesSkipped += destLength self.queue.task_done() continue except (IOError, OSError) as node_error: """Ignore the erorr""" logger.debug(str(node_error)) pass logger.info("%s -> %s" % (src, dest)) try: self.client.copy(src, dest, send_md5=True) node = self.client.get_node(dest, limit=None) destMD5 = node.props.get( 'MD5', 'd41d8cd98f00b204e9800998ecf8427e') destLength = node.attr['st_size'] destTime = node.attr['st_ctime'] if opt.cache_nodes: md5Cache.update(dest, destMD5, destLength, destTime) self.filesSent += 1 self.bytesSent += stat.st_size except (IOError, OSError) as e: logger.error("Error writing %s to server, skipping" % (src)) logger.error(str(e)) import re if re.search('NodeLocked', str(e)) != None: logger.error( "Use vlock to unlock the node before syncing to %s." % (dest)) try: if e.errno == 104: self.queue.put(requeue) except Exception as e2: logger.error("Error during requeue") logger.error(str(e2)) pass self.filesErrored += 1 pass self.queue.task_done() def mkdirs(dirs): logger.debug("%s %s" % (dirs, str(goodDirs))) ## if we've seen this before skip it. if dirs in goodDirs: return ## try and make a new directory and return ## failure indicates we should see if subdirs exist try: client.mkdir(dirs) logger.info("Made directory %s " % (dirs)) goodDirs.append(dirs) return except OSError as e: exit_code = getattr(e, 'errno', -1) if exit_code != errno.EEXIST: raise e ## OK, must already have existed, add to list goodDirs.append(dirs) return def copy(source, dest): ## strip down dest until we find a part that exists ## and then build up the path. Dest should include the filename if os.path.islink(source): logger.error("%s is a link, skipping" % (source)) return if not os.access(source, os.R_OK): logger.error("Failed to open file %s, skipping" % (source)) return import re if re.match('^[A-Za-z0-9\\._\\-\\(\\);:&\\*\\$@!+=\\/]*$', source) is None: logger.error("filename %s contains illegal characters, skipping" % (source)) return dirname = os.path.dirname(dest) mkdirs(dirname) if opt.include is not None and not re.search(opt.include, source): return queue.put((source, dest), timeout=3600) def startStreams(nstreams, vospace_client): streams = [] for i in range(nstreams): logger.info("Launching vospace connection stream %d" % (i)) t = ThreadCopy(queue, client=vospace_client) t.daemon = True t.start() streams.append(t) return streams def buildFileList(basePath, destRoot='', recursive=False, ignore=None): """Build a list of files that should be copied into VOSpace""" import string spinner = ['-', '\\', '|', '/', '-', '\\', '|', '/'] count = 0 import re for (root, dirs, filenames) in os.walk(basePath): for thisDirname in dirs: if not recursive: continue thisDirname = os.path.join(root, thisDirname) skip = False if ignore is not None: for thisIgnore in ignore.split(','): if not thisDirname.find(thisIgnore) < 0: logger.info("excluding: %s " % (thisDirname)) skip = True continue if skip: continue cprefix = os.path.commonprefix((basePath, thisDirname)) thisDirname = os.path.normpath(destRoot + "/" + thisDirname[len(cprefix):]) mkdirs(thisDirname) for thisfilename in filenames: srcfilename = os.path.normpath(os.path.join( root, thisfilename)) skip = False if ignore is not None: for thisIgnore in ignore.split(','): if not srcfilename.find(thisIgnore) < 0: logger.info("excluding: %s " % (srcfilename)) skip = True continue if skip: continue cprefix = os.path.commonprefix((basePath, srcfilename)) destfilename = os.path.normpath(destRoot + "/" + srcfilename[len(cprefix):]) thisDirname = os.path.dirname(destfilename) mkdirs(thisDirname) count += 1 if opt.verbose: sys.stderr.write( "Building list of files to transfer %s\r" % (spinner[count % len(spinner)])) copy(srcfilename, destfilename) if not recursive: return return streams = startStreams(opt.nstreams, vospace_client=client) ### build a complete file list given all the things on the command line for filename in args: filename = os.path.abspath(filename) thisRoot = dest if os.path.isdir(filename): if filename[-1] != "/": if os.path.basename(filename) != os.path.basename(dest): thisRoot = os.path.join(dest, os.path.basename(filename)) mkdirs(thisRoot) nodeDict[thisRoot] = client.get_node(thisRoot, limit=None) try: buildFileList(filename, destRoot=thisRoot, recursive=opt.recursive, ignore=opt.exclude) except Exception as e: logger.error(str(e)) logger.error("ignoring error") elif os.path.isfile(filename): if destIsDir: thisRoot = os.path.join(dest, os.path.basename(filename)) copy(filename, thisRoot) else: logger.error("%s: No such file or directory." % (filename)) logger.info("\nWaiting for transfers to complete.\nCTRL-\ to interrupt\n") queue.join() endTime = time.time() bytesSent = 0 filesSent = 0 bytesSkipped = 0 filesSkipped = 0 filesErrored = 0 for stream in streams: bytesSent += stream.bytesSent bytesSkipped += stream.bytesSkipped filesSent += stream.filesSent filesSkipped += stream.filesSkipped filesErrored += stream.filesErrored logger.info("\n\n==== TRANSFER REPORT ====\n\n") if bytesSent > 0: rate = bytesSent / (endTime - startTime) / 1024.0 logger.info("Sent %d files (%8.1f kbytes @ %8.3f kBytes/s)" % (filesSent, bytesSent / 1024.0, rate)) speedUp = (bytesSkipped + bytesSent) / bytesSent logger.info("Speedup: %f (skipped %d files)" % (speedUp, filesSkipped)) if bytesSent == 0: logger.info("No files needed sending ") if filesErrored > 0: logger.info("Error transferring %d files, please try again" % (filesErrored))
try: if len(sys.argv) not in [2, 3]: raise Exception("Arguments are: [user id] [OPTIONAL: VOS token]") print >> sys.stderr, sys.argv session_uuid = sys.argv[1] if (len(sys.argv) == 3) and (sys.argv[2] != ''): # --- Authenticated session launcher here -------------------------- # Obtain CANFAR user name from the token and validate it by # trying a 'vls'-like operation on the top node of the subtree # of VOSpace for which the token is scoped vospace_token = sys.argv[2] username, scope = re.match('^userid=(.*?)&.*?scope=(.*?)&.*$', vospace_token).groups() client = vos.Client(vospace_token=vospace_token) infoList = client.getInfoList(scope) else: # --- Anonymous session launcher here ----------------------------- username = session_uuid vospace_token = '' print >> sys.stderr, "Try to launch session for user '%s' token '%s'" % ( username, vospace_token) container = get_or_make_container( username, vospace_token, username) # SJ added vospace_token and username # SJ making sure authenticated user has directory on VOSpace: if vospace_token is not '': pass
def vchmod(): signal.signal(signal.SIGINT, signal_handler) usage = """vchmod mode node [read/write group names in quotes - maximum of 4 each] Accepted modes: (og|go|o|g)[+-=](rw|wr|r\w) Sets read/write properties of a node. Version: %s """ % version.version parser = CommonParser(usage) parser.add_option("-R", "--recursive", action='store_const', const=True, help="Recursive set read/write properties") if len(sys.argv) == 1: parser.print_help() sys.exit() (opt, args) = parser.parse_args() parser.process_informational_options() logger = logging.getLogger() if len(args) < 2: parser.error("Requires mode and node arguments") cmdArgs = dict(zip(['mode', 'node'], args[0:2])) groupNames = args[2:] import re mode = re.match(r"(?P<who>^og|^go|^o|^g)(?P<op>[\+\-=])(?P<what>rw$|wr$|r$|w$)", cmdArgs['mode']) if not mode: parser.print_help() logger.error("\n\nAccepted modes: (og|go|o|g)[+-=](rw|wr|r\w)\n\n") sys.exit(-1) props = {} if 'o' in mode.group('who'): if not mode.group('what') == 'r': # read only parser.print_help() logger.error("\n\nPublic access is readonly, no public write available \n\n") sys.exit(-1) if mode.group('op') == '-': props['ispublic'] = 'false' else: props['ispublic'] = 'true' if 'g' in mode.group('who'): if '-' == mode.group('op'): if not len(groupNames) == 0: parser.print_help() logger.error("\n\nNames of groups not required with remove permission\n\n") sys.exit(-1) if 'r' in mode.group('what'): props['readgroup'] = None if "w" in mode.group('what'): props['writegroup'] = None else: if not len(groupNames) == len(mode.group('what')): parser.print_help() logger.error("\n\n%s group names required for %s\n\n" % (len(mode.group('what')), mode.group('what'))) sys.exit(-1) if mode.group('what').find('r') > -1: # remove duplicate whitespaces rgroups = " ".join(groupNames[mode.group('what').find('r')].split()) props['readgroup'] = (vos.CADC_GMS_PREFIX + rgroups.replace(" ", " " + vos.CADC_GMS_PREFIX)) if mode.group('what').find('w') > -1: wgroups = " ".join(groupNames[mode.group('what').find('w')].split()) props['writegroup'] = (vos.CADC_GMS_PREFIX + wgroups.replace(" ", " " + vos.CADC_GMS_PREFIX)) logger.debug("Properties: %s" % (str(props))) logger.debug("Node: %s" % cmdArgs['node']) returnCode = 0 try: client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token) node = client.get_node(cmdArgs['node']) if opt.recursive: node.props.clear() node.clear_properties() # del node.node.findall(vos.Node.PROPERTIES)[0:] if 'readgroup' in props: node.chrgrp(props['readgroup']) if 'writegroup' in props: node.chwgrp(props['writegroup']) if 'ispublic' in props: node.set_public(props['ispublic']) logger.debug("Node: {0}".format(node)) returnCode = client.update(node, opt.recursive) except KeyboardInterrupt as ke: logger.error("Received keyboard interrupt. Execution aborted...\n") sys.exit(getattr(ke, 'errno', -1)) except Exception as e: logger.error('Error {}: '.format(str(e))) sys.exit(getattr(e, 'errno', -1)) sys.exit(returnCode)
def vcp(): ## handle interrupts nicely signal.signal(signal.SIGINT, signal_handler) parser = CommonParser( "%prog filename vos:rootNode/destination", description=("Copy a file or directory (always recursive) to a " "VOSpace location. Try to be UNIX like. ")) parser.add_option("--exclude", default=None, help="exclude files that match pattern") parser.add_option( "--include", default=None, help="only include files that match pattern (overrides exclude)") parser.add_option("-i", "--interrogate", action="store_true", help="Ask before overwriting files") parser.add_option( "--overwrite", action="store_true", help= "don't check destination MD5, just overwrite even if source matches destination" ) parser.add_option( "--quick", action="store_true", help= "Use default CADC urls, for speed. Will fail if CADC changes data storage mechanism", default=False) parser.add_option( "-L", "--follow-links", help= "Should recursive copy follow symbolic links? Default is to not follow links.", action="store_true", default=False) parser.add_option("--ignore", action="store_true", default=False, help="ignore errors and continue with recursive copy") (opt, args) = parser.parse_args() parser.process_informational_options() if len(args) < 2: parser.error("Must give a source and a destination") dest = args.pop() this_destination = dest if dest[0:4] != 'vos:': dest = os.path.abspath(dest) client = vos.Client(vospace_certfile=opt.certfile, vospace_token=opt.token, transfer_shortcut=opt.quick) exit_code = 0 cutout_pattern = re.compile( r'(.*?)(?P<cutout>(\[[\-\+]?[\d\*]+(:[\-\+]?[\d\*]+)?(,[\-\+]?[\d\*]+(:[\-\+]?[\d\*]+)?)?\])+)$' ) ra_dec_cutout_pattern = re.compile("([^\(\)]*?)" "(?P<cutout>\(" "(?P<ra>[\-\+]?\d*(\.\d*)?)," "(?P<dec>[\-\+]?\d*(\.\d*)?)," "(?P<rad>\d*(\.\d*)?)\))?") # Warnings: # vcp destination specified with a trailing '/' implies ContainerNode # # If destination has trailing '/' and exists but is a DataNode then # error message is returned: "Invalid Argument (target node is not a DataNode)" # # vcp currently only works on the CADC VOSpace server. # Version: %s """ % (version.version) def size(filename): if filename[0:4] == "vos:": return client.size(filename) return os.stat(filename).st_size def create(filename): if filename[0:4] == "vos:": return client.create(vos.Node(client.fix_uri(filename))) else: open(filename, 'w').close() pass def get_node(filename, limit=None): """Get node, from cache if possible""" return client.get_node(filename, limit=limit) # here are a series of methods that choose between calling the system version or the vos version of various # function, based on pattern matching. # TODO: Put these function in a separate module. def isdir(filename): logging.debug("Doing an isdir on %s" % filename) if filename[0:4] == "vos:": return client.isdir(filename) else: return os.path.isdir(filename) def islink(filename): logging.debug("Doing an islink on %s" % filename) if filename[0:4] == "vos:": try: return get_node(filename).islink() except: return False else: return os.path.islink(filename) def access(filename, mode): """ Check if the file can be accessed. @param filename: name of file or vospace node to check @param mode: what access mode should be checked: see os.access @return: True/False """ logging.debug("checking for access %s " % filename) if filename[0:4] == "vos:": try: node = get_node(filename, limit=0) return node is not None except (exceptions.NotFoundException, exceptions.ForbiddenException, exceptions.UnauthorizedException) as ex: return False else: return os.access(filename, mode) def listdir(dirname): """Walk through the directory structure a al os.walk""" logging.debug("getting a dirlist %s " % dirname) if dirname[0:4] == "vos:": return client.listdir(dirname, force=True) else: return os.listdir(dirname) def mkdir(filename): logging.debug("Making directory %s " % filename) if filename[0:4] == 'vos:': return client.mkdir(filename) else: return os.mkdir(filename) def lglob(pathname): """ Call system glob if not vos path. @param pathname: the pathname (aka pattern) to glob with. @return: list of matched filenames. """ if pathname[0:4] == "vos:": return client.glob(pathname) else: return glob.glob(pathname) def get_md5(filename): logging.debug("getting the MD5 for %s" % filename) if filename[0:4] == 'vos:': return get_node(filename).props.get('MD5', vos.ZERO_MD5) else: return md5_cache.MD5_Cache.computeMD5(filename) def lglob(pathname): if pathname[0:4] == "vos:": return client.glob(pathname) else: return glob.glob(pathname) def copy(source_name, destination_name, exclude=None, include=None, interrogate=False, overwrite=False, ignore=False): """ :param source_name: :param destination_name: :param exclude: :param include: :return: :raise e: """ global exit_code ## determine if this is a directory we are copying so need to be recursive try: if not opt.follow_links and islink(source_name): #return link(source_name, destination_name) logging.info( "{}: Skipping (symbolic link)".format(source_name)) return if isdir(source_name): ## make sure the destination exists... if not isdir(destination_name): mkdir(destination_name) ## for all files in the current source directory copy them to the destination directory for filename in listdir(source_name): logging.debug("%s -> %s" % (filename, source_name)) copy(os.path.join(source_name, filename), os.path.join(destination_name, filename), exclude, include, interrogate, overwrite, ignore) else: if interrogate: if access(destination_name, os.F_OK): sys.stderr.write( "File %s exists. Overwrite? (y/n): " % destination_name) ans = sys.stdin.readline().strip() if ans != 'y': raise Exception("File exists") if not overwrite and access(destination_name, os.F_OK): ### check if the MD5 of dest and source mathc, if they do then skip if get_md5(destination_name) == get_md5(source_name): logging.info("%s matches %s, skipping" % (source_name, destination_name)) return if not access(os.path.dirname(destination_name), os.F_OK): raise OSError( errno.EEXIST, "vcp: ContainerNode %s does not exist" % os.path.dirname(destination_name)) if not isdir(os.path.dirname(destination_name)) and not islink( os.path.dirname(destination_name)): raise OSError( errno.ENOTDIR, "vcp: %s is not a ContainerNode or LinkNode" % os.path.dirname(destination_name)) skip = False if exclude is not None: for thisIgnore in exclude.split(','): if not destination_name.find(thisIgnore) < 0: skip = True continue if include is not None: skip = True for thisIgnore in include.split(','): if not destination_name.find(thisIgnore) < 0: skip = False continue if not skip: logging.info("%s -> %s " % (source_name, destination_name)) niters = 0 while not skip: try: logging.debug("Starting call to copy") client.copy(source_name, destination_name, send_md5=True) logging.debug("Call to copy returned") break except Exception as client_exception: logging.debug("{}".format(client_exception)) if getattr(client_exception, 'errno', -1) == 104: # 104 is connection reset by peer. Try again on this error logging.warning(str(client_exception)) exit_code = getattr(client_exception, 'errno', -1) elif getattr(client_exception, 'errno', -1) == errno.EIO: # retry on IO errors logging.warning( "{0}: Retrying".format(client_exception)) pass elif ignore: if niters > 100: logging.error( "%s (skipping after %d attempts)" % (str(client_exception), niters)) skip = True else: logging.error("%s (retrying)" % str(client_exception)) time.sleep(5) niters += 1 else: raise client_exception except Exception as ex: logging.debug("{}".format(ex)) raise except OSError as os_exception: logging.debug(str(os_exception)) if getattr(os_exception, 'errno', -1) == errno.EINVAL: # not a valid uri, just skip those... logging.warning("%s: Skipping" % str(os_exception)) else: raise os_exception # main loop try: for source_pattern in args: # define this empty cutout string. Then we strip possible cutout strings off the end of the # pattern before matching. This allows cutouts on the vos service. # The shell does pattern matching for local files, so don't run glob on local files. if source_pattern[0:4] != "vos:": sources = [source_pattern] else: cutout_match = cutout_pattern.search(source_pattern) cutout = None if cutout_match is not None: source_pattern = cutout_match.group(1) cutout = cutout_match.group('cutout') else: ra_dec_match = ra_dec_cutout_pattern.search(source_pattern) if ra_dec_match is not None: cutout = ra_dec_match.group('cutout') logging.debug("cutout: {}".format(cutout)) sources = lglob(source_pattern) if cutout is not None: # stick back on the cutout pattern if there was one. sources = [s + cutout for s in sources] for source in sources: if source[0:4] != "vos:": source = os.path.abspath(source) # the source must exist, of course... if not access(source, os.R_OK): raise Exception("Can't access source: %s " % source) if not opt.follow_links and islink(source): logging.info("{}: Skipping (symbolic link)".format(source)) continue # copying inside VOSpace not yet implemented if source[0:4] == 'vos:' and dest[0:4] == 'vos:': raise Exception( "Can not (yet) copy from VOSpace to VOSpace.") this_destination = dest if isdir(source): if not opt.follow_links and islink(source): continue logging.debug("%s is a directory or link to one" % source) # To mimic unix fs behaviours if copying a directory and # the destination directory exists then the actual # destination in a recursive copy is the destination + # source basename. # This has an odd behaviour if more than one directory is given as a source and the copy is recursive. if access(dest, os.F_OK): if not isdir(dest): raise Exception( "Can't write a directory (%s) to a file (%s)" % (source, dest)) # directory exists so we append the end of source to that (UNIX behaviour) this_destination = os.path.normpath( os.path.join(dest, os.path.basename(source))) elif len(args) > 1: raise Exception( "vcp can not copy multiple things into a non-existent location (%s)" % dest) elif dest[-1] == '/' or isdir(dest): # we're copying into a directory this_destination = os.path.join(dest, os.path.basename(source)) copy(source, this_destination, exclude=opt.exclude, include=opt.include, interrogate=opt.interrogate, overwrite=opt.overwrite, ignore=opt.ignore) except KeyboardInterrupt as ke: logging.info("Received keyboard interrupt. Execution aborted...\n") exit_code = getattr(ke, 'errno', -1) except ParseError as pe: exit_code = errno.EREMOTE logging.error("Failure at server while copying {0} -> {1}".format( source, dest)) except Exception as e: message = str(e) if re.search('NodeLocked', str(e)) is not None: logging.error( "Use vlock to unlock the node before copying to %s." % this_destination) elif getattr(e, 'errno', -1) == errno.EREMOTE: logging.error( "Failure at remote server while copying {0} -> {1}".format( source, dest)) else: logging.debug("Exception throw: %s %s" % (type(e), str(e))) logging.debug(traceback.format_exc()) logging.error(message) exit_code = getattr(e, 'errno', -1) sys.exit(exit_code)