def main(): COMPRESSION_FILTERS = ('blosclz', 'lz4', 'lz4hc', 'snappy', 'gzip', 'zstd') loglevel = logging.ERROR verbose = False compression = None compression_opts = None s3path = None dataload = "ingest" # or None, or "link" cfg["cmd"] = sys.argv[0].split('/')[-1] if cfg["cmd"].endswith(".py"): cfg["cmd"] = "python " + cfg["cmd"] cfg["logfname"] = None logfname = None s3 = None # s3fs instance mode = 'w' retries = 10 # number of retry attempts for HSDS requests src_files = [] argn = 1 while argn < len(sys.argv): arg = sys.argv[argn] val = None if arg[0] == '-' and len(src_files) > 0: # options must be placed before filenames sys.stderr.write("options must precede source files") usage() sys.exit(-1) if len(sys.argv) > argn + 1: val = sys.argv[argn + 1] if arg in ("-v", "--verbose"): verbose = True argn += 1 elif arg == "--link": if dataload != "ingest": sys.stderr.write("--nodata flag can't be used with link flag") sys.exit(1) dataload = "link" argn += 1 elif arg == "--nodata": if dataload != "ingest": sys.stderr.write("--nodata flag can't be used with link flag") sys.exit(1) dataload = None argn += 1 elif arg == "--loglevel": if val == "debug": loglevel = logging.DEBUG elif val == "info": loglevel = logging.INFO elif val == "warning": loglevel = logging.WARNING elif val == "error": loglevel = logging.ERROR else: sys.stderr.write("unknown loglevel") usage() sys.exit(-1) argn += 2 elif arg == '--logfile': logfname = val argn += 2 elif arg in ("-b", "--bucket"): cfg["hs_bucket"] = val argn += 2 elif arg in ("-h", "--help"): usage() sys.exit(0) elif arg in ("-e", "--endpoint"): cfg["hs_endpoint"] = val argn += 2 elif arg in ("-u", "--username"): cfg["hs_username"] = val argn += 2 elif arg in ("-p", "--password"): cfg["hs_password"] = val argn += 2 elif arg in ("-a", "--append"): mode = 'a' argn += 1 elif arg == '--cnf-eg': print_config_example() sys.exit(0) elif arg.startswith("-z"): compression_opts = 4 if len(arg) > 2: try: compression_opts = int(arg[2:]) except ValueError: sys.stderr.write( "Compression Level must be int between 0 and 9") sys.exit(-1) if not compression: compression = 'lz4' argn += 1 elif arg in ("-c", "--compression"): if val not in COMPRESSION_FILTERS: sys.stderr.write("unknown compression filter") usage() sys.exit(-1) compression = val argn += 2 elif arg == "--retries": retries = int(val) argn += 2 elif arg[0] == '-': usage() sys.exit(-1) else: src_files.append(arg) argn += 1 # setup logging logging.basicConfig( filename=logfname, format='%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s', level=loglevel) logging.debug("set log_level to {}".format(loglevel)) # end arg parsing logging.info("username: {}".format(cfg["hs_username"])) logging.info("endpoint: {}".format(cfg["hs_endpoint"])) logging.info("verbose: {}".format(verbose)) if len(src_files) < 2: # need at least a src and destination usage() sys.exit(-1) domain = src_files[-1] src_files = src_files[:-1] logging.info("source files: {}".format(src_files)) logging.info("target domain: {}".format(domain)) if len(src_files) > 1 and (domain[0] != '/' or domain[-1] != '/'): sys.stderr.write( "target must be a folder if multiple source files are provided") usage() sys.exit(-1) if cfg["hs_endpoint"] is None: sys.stderr.write('No endpoint given, try -h for help\n') sys.exit(1) logging.info("endpoint: {}".format(cfg["hs_endpoint"])) # check we have min HDF5 lib version for chunk query if dataload == "link": logging.info("checking libversion") if h5py.version.version_tuple.major == 2 and h5py.version.version_tuple.minor < 10: sys.stderr.write( "link option requires h5py version 2.10 or higher") sys.exit(1) if h5py.version.hdf5_version_tuple < (1, 10, 6): sys.stderr.write( "link option requires hdf5 lib version 1.10.6 or higher") sys.exit(1) try: for src_file in src_files: # check if this is a non local file, if it is remote (http, etc...) stage it first then insert it into hsds src_file_chk = urlparse(src_file) logging.debug(src_file_chk) tgt = domain if tgt[-1] == '/': # folder destination tgt = tgt + op.basename(src_file) # get a handle to input file if src_file.startswith("s3://"): s3path = src_file if not S3FS_IMPORT: sys.stderr.write("Install S3FS package to load s3 files") sys.exit(1) if not s3: s3 = s3fs.S3FileSystem(use_ssl=False) try: fin = h5py.File(s3.open(src_file, "rb"), moe="r") except IOError as ioe: logging.error("Error opening file {}: {}".format( src_file, ioe)) sys.exit(1) else: if dataload == "link": if op.isabs(src_file): sys.stderr.write( "source file must be s3path (for HSDS using S3 storage) or relative path from server root directory (for HSDS using posix storage)" ) sys.exit(1) s3path = src_file else: s3path = None try: fin = h5py.File(src_file, mode='r') except IOError as ioe: logging.error("Error opening file {}: {}".format( src_file, ioe)) sys.exit(1) # create the output domain try: username = cfg["hs_username"] password = cfg["hs_password"] endpoint = cfg["hs_endpoint"] bucket = cfg["hs_bucket"] fout = h5pyd.File(tgt, mode, endpoint=endpoint, username=username, password=password, bucket=bucket, retries=retries) except IOError as ioe: if ioe.errno == 404: logging.error("Domain: {} not found".format(tgt)) elif ioe.errno == 403: logging.error("No write access to domain: {}".format(tgt)) else: logging.error("Error creating file {}: {}".format( tgt, ioe)) sys.exit(1) # do the actual load load_file(fin, fout, verbose=verbose, dataload=dataload, s3path=s3path, compression=compression, compression_opts=compression_opts) msg = "File {} uploaded to domain: {}".format(src_file, tgt) logging.info(msg) if verbose: print(msg) except KeyboardInterrupt: logging.error('Aborted by user via keyboard interrupt.') sys.exit(1)
def main(): loglevel = logging.ERROR verbose = False nodata = False cfg["cmd"] = sys.argv[0].split('/')[-1] if cfg["cmd"].endswith(".py"): cfg["cmd"] = "python " + cfg["cmd"] cfg["verbose"] = False endpoint=cfg["hs_endpoint"] username=cfg["hs_username"] password=cfg["hs_password"] bucket = cfg["hs_bucket"] logfname=None ipvfam=None des_file = None src_domain = None argn = 1 while argn < len(sys.argv): arg = sys.argv[argn] val = None if arg[0] == '-' and src_domain is not None: # options must be placed before filenames print("options must precead source files") usage() sys.exit(-1) if len(sys.argv) > argn + 1: val = sys.argv[argn+1] if arg in ("-v", "--verbose"): verbose = True argn += 1 elif arg == "--nodata": nodata = True argn += 1 elif arg == "--loglevel": if val == "debug": loglevel = logging.DEBUG elif val == "info": loglevel = logging.INFO elif val == "warning": loglevel = logging.WARNING elif val == "error": loglevel = logging.ERROR else: print("unknown loglevel") usage() sys.exit(-1) argn += 2 elif arg == '--logfile': logfname = val argn += 2 elif arg in ("-b", "--bucket"): bucket = val argn += 2 elif arg == '-4': ipvfam = 4 elif arg == '-6': ipvfam = 6 elif arg in ("-h", "--help"): usage() sys.exit(0) elif arg in ("-e", "--endpoint"): endpoint = val argn += 2 elif arg in ("-u", "--username"): username = val argn += 2 elif arg in ("-p", "--password"): password = val argn += 2 elif arg == '--cnf-eg': print_config_example() sys.exit(0) elif arg[0] == '-': usage() sys.exit(-1) elif src_domain is None: src_domain = arg argn += 1 elif des_file is None: des_file = arg argn += 1 else: usage() sys.exit(-1) # setup logging logging.basicConfig(filename=logfname, format='%(asctime)s %(message)s', level=loglevel) logging.debug("set log_level to {}".format(loglevel)) # end arg parsing logging.info("username: {}".format(username)) logging.info("password: {}".format(password)) logging.info("endpoint: {}".format(endpoint)) logging.info("verbose: {}".format(verbose)) if src_domain is None or des_file is None: # need at least a src and destination usage() sys.exit(-1) logging.info("source domain: {}".format(src_domain)) logging.info("target file: {}".format(des_file)) if endpoint is None: logging.error('No endpoint given, try -h for help\n') sys.exit(1) logging.info("endpoint: {}".format(endpoint)) # get a handle to input domain try: fin = h5pyd.File(src_domain, mode='r', endpoint=endpoint, username=username, password=password, bucket=bucket, use_cache=True) except IOError as ioe: if ioe.errno == 403: logging.error("No read access to domain: {}".format(src_domain)) elif ioe.errno == 404: logging.error("Domain: {} not found".format(src_domain)) elif ioe.errno == 410: logging.error("Domain: {} has been recently deleted".format(src_domain)) else: logging.error("Error opening domain {}: {}".format(src_domain, ioe)) sys.exit(1) # create the output HDF5 file try: fout = h5py.File(des_file, 'w') except IOError as ioe: logging.error("Error creating file {}: {}".format(des_file, ioe)) sys.exit(1) try: load_file(fin, fout, verbose=verbose, nodata=nodata) msg = "Domain {} downloaded to file: {}".format(src_domain, des_file) logging.info(msg) if verbose: print(msg) except KeyboardInterrupt: logging.error('Aborted by user via keyboard interrupt.') sys.exit(1)
def main(): loglevel = logging.ERROR verbose = False nodata = False deflate = None cfg["logfname"] = None logfname = None ipvfam = None src_files = [] argn = 1 while argn < len(sys.argv): arg = sys.argv[argn] val = None if arg[0] == '-' and len(src_files) > 0: # options must be placed before filenames print("options must precead source files") usage() sys.exit(-1) if len(sys.argv) > argn + 1: val = sys.argv[argn + 1] if arg in ("-v", "--verbose"): verbose = True argn += 1 elif arg == "--nodata": nodata = True argn += 1 elif arg == "--loglevel": if val == "debug": loglevel = logging.DEBUG elif val == "info": loglevel = logging.INFO elif val == "warning": loglevel = logging.WARNING elif val == "error": loglevel = logging.ERROR else: print("unknown loglevel") usage() sys.exit(-1) argn += 2 elif arg == '--logfile': logfname = val argn += 2 elif arg == '-4': ipvfam = 4 elif arg == '-6': ipvfam = 6 elif arg in ("-h", "--help"): usage() sys.exit(0) elif arg in ("-e", "--endpoint"): cfg["hs_endpoint"] = val argn += 2 elif arg in ("-u", "--username"): cfg["hs_username"] = val argn += 2 elif arg in ("-p", "--password"): cfg["hs_password"] = val argn += 2 elif arg == '--cnf-eg': print_config_example() sys.exit(0) elif arg.startswith("-z"): compressLevel = 4 if len(arg) > 2: try: compressLevel = int(arg[2:]) except ValueError: print("Compression Level must be int between 0 and 9") sys.exit(-1) deflate = compressLevel argn += 1 elif arg[0] == '-': usage() sys.exit(-1) else: src_files.append(arg) argn += 1 # setup logging logging.basicConfig( filename=logfname, format='%(asctime)s %(filename)s:%(lineno)d %(message)s', level=loglevel) logging.debug("set log_level to {}".format(loglevel)) # end arg parsing logging.info("username: {}".format(cfg["hs_username"])) logging.info("password: {}".format(cfg["hs_password"])) logging.info("endpoint: {}".format(cfg["hs_password"])) logging.info("verbose: {}".format(verbose)) if len(src_files) < 2: # need at least a src and destination usage() sys.exit(-1) domain = src_files[-1] src_files = src_files[:-1] logging.info("source files: {}".format(src_files)) logging.info("target domain: {}".format(domain)) if len(src_files) > 1 and (domain[0] != '/' or domain[-1] != '/'): print("target must be a folder if multiple source files are provided") usage() sys.exit(-1) if cfg["hs_endpoint"] is None: logging.error('No endpoint given, try -h for help\n') sys.exit(1) logging.info("endpoint: {}".format(cfg["hs_endpoint"])) try: for src_file in src_files: # check if this is a non local file, if it is remote (http, etc...) stage it first then insert it into hsds src_file_chk = urlparse(src_file) logging.debug(src_file_chk) if src_file_chk.scheme == 'http' or src_file_chk.scheme == 'https' or src_file_chk.scheme == 'ftp': src_file = stage_file(src_file, netfam=ipvfam) if src_file == None: continue istmp = True logging.info('temp source data: ' + str(src_file)) else: istmp = False tgt = domain if tgt[-1] == '/': # folder destination tgt = tgt + op.basename(src_file) # get a handle to input file try: fin = h5py.File(src_file, mode='r') except IOError as ioe: logging.error("Error opening file {}: {}".format( src_file, ioe)) sys.exit(1) # create the output domain try: username = cfg["hs_username"] password = cfg["hs_password"] endpoint = cfg["hs_endpoint"] fout = h5pyd.File(tgt, 'w', endpoint=endpoint, username=username, password=password) except IOError as ioe: if ioe.errno == 404: logging.error("Domain: {} not found".format(tgt)) elif ioe.errno == 403: logging.error("No write access to domain: {}".format(tgt)) else: logging.error("Error creating file {}: {}".format( tgt, ioe)) sys.exit(1) # do the actual load load_file(fin, fout, verbose=verbose, nodata=nodata, deflate=deflate) # cleanup if needed if istmp: try: os.unlink(src_file) except IOError as e: logging.warn("failed to delete %s : %s" % (src_file, str(e))) msg = "File {} uploaded to domain: {}".format(src_file, tgt) logging.info(msg) if verbose: print(msg) except KeyboardInterrupt: logging.error('Aborted by user via keyboard interrupt.') sys.exit(1)
def load(filename): verbose = True print(f"load_file: {filename}") print(f"got filename: {filename} from {inventory_domain}") # got filename: data/hdf5test/snp500.h5 from /home/john/bucketloader/inventory.h5 s3path = f"s3://{src_bucket}/{filename}" print(f"using s3path: {s3path}") tgt_path = tgt_folder + filename print(f"tgt_path: {tgt_path}") ensure_folder(os.path.dirname(tgt_path)) # tbd ignore 409 errors? # make sure the local hsds is up (if being used) if hsds_local: endpoint = hsds_local else: endpoint = hsds_global print(f"running loading {s3path} to {tgt_path}") s3 = s3fs.S3FileSystem(use_ssl=False) try: fin = h5py.File(s3.open(s3path, "rb"), "r") except IOError as ioe: logging.error("Error opening s3path {}: {}".format(s3path, ioe)) raise # create output domain try: fout = h5pyd.File(tgt_path, 'w', endpoint=endpoint, username=username, password=password, bucket=tgt_bucket) except IOError as ioe: if ioe.errno == 404: logging.error("Domain: {} not found".format(tgt_path)) elif ioe.errno == 403: logging.error("No write access to domain: {}".format(tgt_path)) else: logging.error("Error creating file {}: {}".format(tgt_path, ioe)) raise # do the actual load try: if link_files: dataload = "link" else: dataload = "ingest" compression = None # TBD compression_opts = None # TBD load_file(fin, fout, verbose=verbose, dataload=dataload, s3path=s3path, compression=compression, compression_opts=compression_opts) except IOError as ioe: logging.error("load_file error: {}".format(ioe)) raise if config.get("public_read"): # make public read, and get acl print("adding public read ACL") acl = {"userName": "******"} acl["create"] = False acl["read"] = True acl["update"] = False acl["delete"] = False acl["readACL"] = True acl["updateACL"] = False fout.putACL(acl) fout.close()
def main(): loglevel = logging.ERROR verbose = False nodata = False deflate = None cfg["cmd"] = sys.argv[0].split('/')[-1] if cfg["cmd"].endswith(".py"): cfg["cmd"] = "python " + cfg["cmd"] cfg["logfname"] = None logfname = None src_files = [] argn = 1 while argn < len(sys.argv): arg = sys.argv[argn] val = None if arg[0] == '-' and len(src_files) > 0: # options must be placed before filenames print("options must precead source files") usage() sys.exit(-1) if len(sys.argv) > argn + 1: val = sys.argv[argn + 1] if arg in ("-v", "--verbose"): verbose = True argn += 1 elif arg == "--nodata": nodata = True argn += 1 elif arg == "--loglevel": if val == "debug": loglevel = logging.DEBUG elif val == "info": loglevel = logging.INFO elif val == "warning": loglevel = logging.WARNING elif val == "error": loglevel = logging.ERROR else: print("unknown loglevel") usage() sys.exit(-1) argn += 2 elif arg == '--logfile': logfname = val argn += 2 elif arg in ("-h", "--help"): usage() sys.exit(0) elif arg in ("-e", "--endpoint"): cfg["hs_endpoint"] = val argn += 2 elif arg in ("-u", "--username"): cfg["hs_username"] = val argn += 2 elif arg in ("-p", "--password"): cfg["hs_password"] = val argn += 2 elif arg in ("-b", "--bucket"): cfg["hs_bucket"] = val argn += 2 elif arg == '--cnf-eg': print_config_example() sys.exit(0) elif arg.startswith("-z"): compressLevel = 4 if len(arg) > 2: try: compressLevel = int(arg[2:]) except ValueError: print("Compression Level must be int between 0 and 9") sys.exit(-1) deflate = compressLevel argn += 1 elif arg[0] == '-': usage() sys.exit(-1) else: src_files.append(arg) argn += 1 # setup logging logging.basicConfig( filename=logfname, format='%(asctime)s %(filename)s:%(lineno)d %(message)s', level=loglevel) logging.debug("set log_level to {}".format(loglevel)) # end arg parsing logging.info("username: {}".format(cfg["hs_username"])) logging.info("endpoint: {}".format(cfg["hs_endpoint"])) logging.info("verbose: {}".format(verbose)) if len(src_files) < 2: # need at least a src and destination usage() sys.exit(-1) src_domain = src_files[0] des_domain = src_files[1] logging.info("source domain: {}".format(src_domain)) logging.info("target domain: {}".format(des_domain)) if src_domain[0] != '/' or src_domain[-1] == '/': print("source domain must be an absolute path, non-folder domain") usage() sys.exit(-1) if des_domain[0] != '/' or des_domain[-1] == '/': print("source domain must be an absolute path, non-folder domain") usage() sys.exit(-1) if cfg["hs_endpoint"] is None: logging.error('No endpoint given, try -h for help\n') sys.exit(1) logging.info("endpoint: {}".format(cfg["hs_endpoint"])) try: username = cfg["hs_username"] password = cfg["hs_password"] endpoint = cfg["hs_endpoint"] bucket = cfg["hs_bucket"] # get a handle to input file try: fin = h5pyd.File(src_domain, mode='r', endpoint=endpoint, username=username, password=password, bucket=bucket) except IOError as ioe: logging.error("Error opening file {}: {}".format(src_domain, ioe)) sys.exit(1) # create the output domain try: fout = h5pyd.File(des_domain, 'x', endpoint=endpoint, username=username, password=password, bucket=bucket) except IOError as ioe: if ioe.errno == 403: logging.error( "No write access to domain: {}".format(des_domain)) else: logging.error("Error creating file {}: {}".format( des_domain, ioe)) sys.exit(1) # do the actual load load_file(fin, fout, verbose=verbose, nodata=nodata, deflate=deflate) msg = "File {} uploaded to domain: {}".format(src_domain, des_domain) logging.info(msg) if verbose: print(msg) except KeyboardInterrupt: logging.error('Aborted by user via keyboard interrupt.') sys.exit(1)
def main(): loglevel = logging.ERROR verbose = False deflate = None s3path = None dataload = "ingest" # or None, or "s3link" cfg["cmd"] = sys.argv[0].split('/')[-1] if cfg["cmd"].endswith(".py"): cfg["cmd"] = "python " + cfg["cmd"] cfg["logfname"] = None logfname=None ipvfam=None s3 = None # s3fs instance src_files = [] argn = 1 while argn < len(sys.argv): arg = sys.argv[argn] val = None if arg[0] == '-' and len(src_files) > 0: # options must be placed before filenames print("options must precead source files") usage() sys.exit(-1) if len(sys.argv) > argn + 1: val = sys.argv[argn+1] if arg in ("-v", "--verbose"): verbose = True argn += 1 elif arg == "--s3link": if dataload != "ingest": sys.stderr("--nodata flag can't be used with s3link flag") sys.exit(1) dataload = "s3link" argn += 1 elif arg == "--nodata": if dataload != "ingest": sys.stderr("--nodata flag can't be used with s3link flag") sys.exit(1) dataload = None argn += 1 elif arg == "--loglevel": if val == "debug": loglevel = logging.DEBUG elif val == "info": loglevel = logging.INFO elif val == "warning": loglevel = logging.WARNING elif val == "error": loglevel = logging.ERROR else: print("unknown loglevel") usage() sys.exit(-1) argn += 2 elif arg == '--logfile': logfname = val argn += 2 elif arg in ("-b", "--bucket"): cfg["hs_bucket"] = val argn += 2 elif arg == '-4': ipvfam = 4 elif arg == '-6': ipvfam = 6 elif arg in ("-h", "--help"): usage() sys.exit(0) elif arg in ("-e", "--endpoint"): cfg["hs_endpoint"] = val argn += 2 elif arg in ("-u", "--username"): cfg["hs_username"] = val argn += 2 elif arg in ("-p", "--password"): cfg["hs_password"] = val argn += 2 elif arg == '--cnf-eg': print_config_example() sys.exit(0) elif arg.startswith("-z"): compressLevel = 4 if len(arg) > 2: try: compressLevel = int(arg[2:]) except ValueError: print("Compression Level must be int between 0 and 9") sys.exit(-1) deflate = compressLevel argn += 1 elif arg[0] == '-': usage() sys.exit(-1) else: src_files.append(arg) argn += 1 # setup logging logging.basicConfig(filename=logfname, format='%(asctime)s %(filename)s:%(lineno)d %(message)s', level=loglevel) logging.debug("set log_level to {}".format(loglevel)) # end arg parsing logging.info("username: {}".format(cfg["hs_username"])) logging.info("endpoint: {}".format(cfg["hs_endpoint"])) logging.info("verbose: {}".format(verbose)) if len(src_files) < 2: # need at least a src and destination usage() sys.exit(-1) domain = src_files[-1] src_files = src_files[:-1] logging.info("source files: {}".format(src_files)) logging.info("target domain: {}".format(domain)) if len(src_files) > 1 and (domain[0] != '/' or domain[-1] != '/'): print("target must be a folder if multiple source files are provided") usage() sys.exit(-1) if cfg["hs_endpoint"] is None: logging.error('No endpoint given, try -h for help\n') sys.exit(1) logging.info("endpoint: {}".format(cfg["hs_endpoint"])) try: for src_file in src_files: # check if this is a non local file, if it is remote (http, etc...) stage it first then insert it into hsds src_file_chk = urlparse(src_file) logging.debug(src_file_chk) if src_file_chk.scheme == 'http' or src_file_chk.scheme == 'https' or src_file_chk.scheme == 'ftp': src_file = stage_file(src_file, netfam=ipvfam) if src_file == None: continue istmp = True logging.info('temp source data: '+str(src_file)) else: istmp = False tgt = domain if tgt[-1] == '/': # folder destination tgt = tgt + op.basename(src_file) # get a handle to input file if src_file.startswith("s3://"): s3path = src_file if not S3FS_IMPORT: sys.stderr.write("Install S3FS package to load s3 files") sys.exit(1) if h5py.version.version_tuple.major != 2 or h5py.version.version_tuple.minor < 10: print("s3path source requires h5py version 2.10 or higher") sys.exit(1) if h5py.version.hdf5_version_tuple[0] != 1 or h5py.version.hdf5_version_tuple[1] != 10 or h5py.version.hdf5_version_tuple[2] < 6: print("s3path source requires hdf5 lib version 1.10.6 or higher") sys.exit(1) if not s3: s3 = s3fs.S3FileSystem() try: fin = h5py.File(s3.open(src_file, "rb"), "r") except IOError as ioe: logging.error("Error opening file {}: {}".format(src_file, ioe)) sys.exit(1) else: s3path = None try: fin = h5py.File(src_file, mode='r') except IOError as ioe: logging.error("Error opening file {}: {}".format(src_file, ioe)) sys.exit(1) # create the output domain try: username = cfg["hs_username"] password = cfg["hs_password"] endpoint = cfg["hs_endpoint"] bucket = cfg["hs_bucket"] fout = h5pyd.File(tgt, 'x', endpoint=endpoint, username=username, password=password, bucket=bucket) except IOError as ioe: if ioe.errno == 404: logging.error("Domain: {} not found".format(tgt)) elif ioe.errno == 403: logging.error("No write access to domain: {}".format(tgt)) else: logging.error("Error creating file {}: {}".format(tgt, ioe)) sys.exit(1) # do the actual load load_file(fin, fout, verbose=verbose, dataload=dataload, s3path=s3path, deflate=deflate,) # cleanup if needed if istmp: try: os.unlink(src_file) except IOError as e: logging.warn("failed to delete %s : %s" % (src_file, str(e))) msg = "File {} uploaded to domain: {}".format(src_file, tgt) logging.info(msg) if verbose: print(msg) except KeyboardInterrupt: logging.error('Aborted by user via keyboard interrupt.') sys.exit(1)