def main():
COMPRESSION_FILTERS = ('blosclz', 'lz4', 'lz4hc', 'snappy', 'gzip', 'zstd')
loglevel = logging.ERROR
verbose = False
compression = None
compression_opts = None
s3path = None
dataload = "ingest" # or None, or "link"
cfg["cmd"] = sys.argv[0].split('/')[-1]
if cfg["cmd"].endswith(".py"):
cfg["cmd"] = "python " + cfg["cmd"]
cfg["logfname"] = None
logfname = None
s3 = None # s3fs instance
mode = 'w'
retries = 10 # number of retry attempts for HSDS requests
src_files = []
argn = 1
while argn < len(sys.argv):
arg = sys.argv[argn]
val = None
if arg[0] == '-' and len(src_files) > 0:
# options must be placed before filenames
sys.stderr.write("options must precede source files")
usage()
sys.exit(-1)
if len(sys.argv) > argn + 1:
val = sys.argv[argn + 1]
if arg in ("-v", "--verbose"):
verbose = True
argn += 1
elif arg == "--link":
if dataload != "ingest":
sys.stderr.write("--nodata flag can't be used with link flag")
sys.exit(1)
dataload = "link"
argn += 1
elif arg == "--nodata":
if dataload != "ingest":
sys.stderr.write("--nodata flag can't be used with link flag")
sys.exit(1)
dataload = None
argn += 1
elif arg == "--loglevel":
if val == "debug":
loglevel = logging.DEBUG
elif val == "info":
loglevel = logging.INFO
elif val == "warning":
loglevel = logging.WARNING
elif val == "error":
loglevel = logging.ERROR
else:
sys.stderr.write("unknown loglevel")
usage()
sys.exit(-1)
argn += 2
elif arg == '--logfile':
logfname = val
argn += 2
elif arg in ("-b", "--bucket"):
cfg["hs_bucket"] = val
argn += 2
elif arg in ("-h", "--help"):
usage()
sys.exit(0)
elif arg in ("-e", "--endpoint"):
cfg["hs_endpoint"] = val
argn += 2
elif arg in ("-u", "--username"):
cfg["hs_username"] = val
argn += 2
elif arg in ("-p", "--password"):
cfg["hs_password"] = val
argn += 2
elif arg in ("-a", "--append"):
mode = 'a'
argn += 1
elif arg == '--cnf-eg':
print_config_example()
sys.exit(0)
elif arg.startswith("-z"):
compression_opts = 4
if len(arg) > 2:
try:
compression_opts = int(arg[2:])
except ValueError:
sys.stderr.write(
"Compression Level must be int between 0 and 9")
sys.exit(-1)
if not compression:
compression = 'lz4'
argn += 1
elif arg in ("-c", "--compression"):
if val not in COMPRESSION_FILTERS:
sys.stderr.write("unknown compression filter")
usage()
sys.exit(-1)
compression = val
argn += 2
elif arg == "--retries":
retries = int(val)
argn += 2
elif arg[0] == '-':
usage()
sys.exit(-1)
else:
src_files.append(arg)
argn += 1
# setup logging
logging.basicConfig(
filename=logfname,
format='%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s',
level=loglevel)
logging.debug("set log_level to {}".format(loglevel))
# end arg parsing
logging.info("username: {}".format(cfg["hs_username"]))
logging.info("endpoint: {}".format(cfg["hs_endpoint"]))
logging.info("verbose: {}".format(verbose))
if len(src_files) < 2:
# need at least a src and destination
usage()
sys.exit(-1)
domain = src_files[-1]
src_files = src_files[:-1]
logging.info("source files: {}".format(src_files))
logging.info("target domain: {}".format(domain))
if len(src_files) > 1 and (domain[0] != '/' or domain[-1] != '/'):
sys.stderr.write(
"target must be a folder if multiple source files are provided")
usage()
sys.exit(-1)
if cfg["hs_endpoint"] is None:
sys.stderr.write('No endpoint given, try -h for help\n')
sys.exit(1)
logging.info("endpoint: {}".format(cfg["hs_endpoint"]))
# check we have min HDF5 lib version for chunk query
if dataload == "link":
logging.info("checking libversion")
if h5py.version.version_tuple.major == 2 and h5py.version.version_tuple.minor < 10:
sys.stderr.write(
"link option requires h5py version 2.10 or higher")
sys.exit(1)
if h5py.version.hdf5_version_tuple < (1, 10, 6):
sys.stderr.write(
"link option requires hdf5 lib version 1.10.6 or higher")
sys.exit(1)
try:
for src_file in src_files:
# check if this is a non local file, if it is remote (http, etc...) stage it first then insert it into hsds
src_file_chk = urlparse(src_file)
logging.debug(src_file_chk)
tgt = domain
if tgt[-1] == '/':
# folder destination
tgt = tgt + op.basename(src_file)
# get a handle to input file
if src_file.startswith("s3://"):
s3path = src_file
if not S3FS_IMPORT:
sys.stderr.write("Install S3FS package to load s3 files")
sys.exit(1)
if not s3:
s3 = s3fs.S3FileSystem(use_ssl=False)
try:
fin = h5py.File(s3.open(src_file, "rb"), moe="r")
except IOError as ioe:
logging.error("Error opening file {}: {}".format(
src_file, ioe))
sys.exit(1)
else:
if dataload == "link":
if op.isabs(src_file):
sys.stderr.write(
"source file must be s3path (for HSDS using S3 storage) or relative path from server root directory (for HSDS using posix storage)"
)
sys.exit(1)
s3path = src_file
else:
s3path = None
try:
fin = h5py.File(src_file, mode='r')
except IOError as ioe:
logging.error("Error opening file {}: {}".format(
src_file, ioe))
sys.exit(1)
# create the output domain
try:
username = cfg["hs_username"]
password = cfg["hs_password"]
endpoint = cfg["hs_endpoint"]
bucket = cfg["hs_bucket"]
fout = h5pyd.File(tgt,
mode,
endpoint=endpoint,
username=username,
password=password,
bucket=bucket,
retries=retries)
except IOError as ioe:
if ioe.errno == 404:
logging.error("Domain: {} not found".format(tgt))
elif ioe.errno == 403:
logging.error("No write access to domain: {}".format(tgt))
else:
logging.error("Error creating file {}: {}".format(
tgt, ioe))
sys.exit(1)
# do the actual load
load_file(fin,
fout,
verbose=verbose,
dataload=dataload,
s3path=s3path,
compression=compression,
compression_opts=compression_opts)
msg = "File {} uploaded to domain: {}".format(src_file, tgt)
logging.info(msg)
if verbose:
print(msg)
except KeyboardInterrupt:
logging.error('Aborted by user via keyboard interrupt.')
sys.exit(1)
def main():
loglevel = logging.ERROR
verbose = False
nodata = False
cfg["cmd"] = sys.argv[0].split('/')[-1]
if cfg["cmd"].endswith(".py"):
cfg["cmd"] = "python " + cfg["cmd"]
cfg["verbose"] = False
endpoint=cfg["hs_endpoint"]
username=cfg["hs_username"]
password=cfg["hs_password"]
bucket = cfg["hs_bucket"]
logfname=None
ipvfam=None
des_file = None
src_domain = None
argn = 1
while argn < len(sys.argv):
arg = sys.argv[argn]
val = None
if arg[0] == '-' and src_domain is not None:
# options must be placed before filenames
print("options must precead source files")
usage()
sys.exit(-1)
if len(sys.argv) > argn + 1:
val = sys.argv[argn+1]
if arg in ("-v", "--verbose"):
verbose = True
argn += 1
elif arg == "--nodata":
nodata = True
argn += 1
elif arg == "--loglevel":
if val == "debug":
loglevel = logging.DEBUG
elif val == "info":
loglevel = logging.INFO
elif val == "warning":
loglevel = logging.WARNING
elif val == "error":
loglevel = logging.ERROR
else:
print("unknown loglevel")
usage()
sys.exit(-1)
argn += 2
elif arg == '--logfile':
logfname = val
argn += 2
elif arg in ("-b", "--bucket"):
bucket = val
argn += 2
elif arg == '-4':
ipvfam = 4
elif arg == '-6':
ipvfam = 6
elif arg in ("-h", "--help"):
usage()
sys.exit(0)
elif arg in ("-e", "--endpoint"):
endpoint = val
argn += 2
elif arg in ("-u", "--username"):
username = val
argn += 2
elif arg in ("-p", "--password"):
password = val
argn += 2
elif arg == '--cnf-eg':
print_config_example()
sys.exit(0)
elif arg[0] == '-':
usage()
sys.exit(-1)
elif src_domain is None:
src_domain = arg
argn += 1
elif des_file is None:
des_file = arg
argn += 1
else:
usage()
sys.exit(-1)
# setup logging
logging.basicConfig(filename=logfname, format='%(asctime)s %(message)s', level=loglevel)
logging.debug("set log_level to {}".format(loglevel))
# end arg parsing
logging.info("username: {}".format(username))
logging.info("password: {}".format(password))
logging.info("endpoint: {}".format(endpoint))
logging.info("verbose: {}".format(verbose))
if src_domain is None or des_file is None:
# need at least a src and destination
usage()
sys.exit(-1)
logging.info("source domain: {}".format(src_domain))
logging.info("target file: {}".format(des_file))
if endpoint is None:
logging.error('No endpoint given, try -h for help\n')
sys.exit(1)
logging.info("endpoint: {}".format(endpoint))
# get a handle to input domain
try:
fin = h5pyd.File(src_domain, mode='r', endpoint=endpoint, username=username, password=password, bucket=bucket, use_cache=True)
except IOError as ioe:
if ioe.errno == 403:
logging.error("No read access to domain: {}".format(src_domain))
elif ioe.errno == 404:
logging.error("Domain: {} not found".format(src_domain))
elif ioe.errno == 410:
logging.error("Domain: {} has been recently deleted".format(src_domain))
else:
logging.error("Error opening domain {}: {}".format(src_domain, ioe))
sys.exit(1)
# create the output HDF5 file
try:
fout = h5py.File(des_file, 'w')
except IOError as ioe:
logging.error("Error creating file {}: {}".format(des_file, ioe))
sys.exit(1)
try:
load_file(fin, fout, verbose=verbose, nodata=nodata)
msg = "Domain {} downloaded to file: {}".format(src_domain, des_file)
logging.info(msg)
if verbose:
print(msg)
except KeyboardInterrupt:
logging.error('Aborted by user via keyboard interrupt.')
sys.exit(1)
def main():
loglevel = logging.ERROR
verbose = False
nodata = False
deflate = None
cfg["logfname"] = None
logfname = None
ipvfam = None
src_files = []
argn = 1
while argn < len(sys.argv):
arg = sys.argv[argn]
val = None
if arg[0] == '-' and len(src_files) > 0:
# options must be placed before filenames
print("options must precead source files")
usage()
sys.exit(-1)
if len(sys.argv) > argn + 1:
val = sys.argv[argn + 1]
if arg in ("-v", "--verbose"):
verbose = True
argn += 1
elif arg == "--nodata":
nodata = True
argn += 1
elif arg == "--loglevel":
if val == "debug":
loglevel = logging.DEBUG
elif val == "info":
loglevel = logging.INFO
elif val == "warning":
loglevel = logging.WARNING
elif val == "error":
loglevel = logging.ERROR
else:
print("unknown loglevel")
usage()
sys.exit(-1)
argn += 2
elif arg == '--logfile':
logfname = val
argn += 2
elif arg == '-4':
ipvfam = 4
elif arg == '-6':
ipvfam = 6
elif arg in ("-h", "--help"):
usage()
sys.exit(0)
elif arg in ("-e", "--endpoint"):
cfg["hs_endpoint"] = val
argn += 2
elif arg in ("-u", "--username"):
cfg["hs_username"] = val
argn += 2
elif arg in ("-p", "--password"):
cfg["hs_password"] = val
argn += 2
elif arg == '--cnf-eg':
print_config_example()
sys.exit(0)
elif arg.startswith("-z"):
compressLevel = 4
if len(arg) > 2:
try:
compressLevel = int(arg[2:])
except ValueError:
print("Compression Level must be int between 0 and 9")
sys.exit(-1)
deflate = compressLevel
argn += 1
elif arg[0] == '-':
usage()
sys.exit(-1)
else:
src_files.append(arg)
argn += 1
# setup logging
logging.basicConfig(
filename=logfname,
format='%(asctime)s %(filename)s:%(lineno)d %(message)s',
level=loglevel)
logging.debug("set log_level to {}".format(loglevel))
# end arg parsing
logging.info("username: {}".format(cfg["hs_username"]))
logging.info("password: {}".format(cfg["hs_password"]))
logging.info("endpoint: {}".format(cfg["hs_password"]))
logging.info("verbose: {}".format(verbose))
if len(src_files) < 2:
# need at least a src and destination
usage()
sys.exit(-1)
domain = src_files[-1]
src_files = src_files[:-1]
logging.info("source files: {}".format(src_files))
logging.info("target domain: {}".format(domain))
if len(src_files) > 1 and (domain[0] != '/' or domain[-1] != '/'):
print("target must be a folder if multiple source files are provided")
usage()
sys.exit(-1)
if cfg["hs_endpoint"] is None:
logging.error('No endpoint given, try -h for help\n')
sys.exit(1)
logging.info("endpoint: {}".format(cfg["hs_endpoint"]))
try:
for src_file in src_files:
# check if this is a non local file, if it is remote (http, etc...) stage it first then insert it into hsds
src_file_chk = urlparse(src_file)
logging.debug(src_file_chk)
if src_file_chk.scheme == 'http' or src_file_chk.scheme == 'https' or src_file_chk.scheme == 'ftp':
src_file = stage_file(src_file, netfam=ipvfam)
if src_file == None:
continue
istmp = True
logging.info('temp source data: ' + str(src_file))
else:
istmp = False
tgt = domain
if tgt[-1] == '/':
# folder destination
tgt = tgt + op.basename(src_file)
# get a handle to input file
try:
fin = h5py.File(src_file, mode='r')
except IOError as ioe:
logging.error("Error opening file {}: {}".format(
src_file, ioe))
sys.exit(1)
# create the output domain
try:
username = cfg["hs_username"]
password = cfg["hs_password"]
endpoint = cfg["hs_endpoint"]
fout = h5pyd.File(tgt,
'w',
endpoint=endpoint,
username=username,
password=password)
except IOError as ioe:
if ioe.errno == 404:
logging.error("Domain: {} not found".format(tgt))
elif ioe.errno == 403:
logging.error("No write access to domain: {}".format(tgt))
else:
logging.error("Error creating file {}: {}".format(
tgt, ioe))
sys.exit(1)
# do the actual load
load_file(fin,
fout,
verbose=verbose,
nodata=nodata,
deflate=deflate)
# cleanup if needed
if istmp:
try:
os.unlink(src_file)
except IOError as e:
logging.warn("failed to delete %s : %s" %
(src_file, str(e)))
msg = "File {} uploaded to domain: {}".format(src_file, tgt)
logging.info(msg)
if verbose:
print(msg)
except KeyboardInterrupt:
logging.error('Aborted by user via keyboard interrupt.')
sys.exit(1)
def load(filename):
verbose = True
print(f"load_file: {filename}")
print(f"got filename: {filename} from {inventory_domain}")
# got filename: data/hdf5test/snp500.h5 from /home/john/bucketloader/inventory.h5
s3path = f"s3://{src_bucket}/{filename}"
print(f"using s3path: {s3path}")
tgt_path = tgt_folder + filename
print(f"tgt_path: {tgt_path}")
ensure_folder(os.path.dirname(tgt_path)) # tbd ignore 409 errors?
# make sure the local hsds is up (if being used)
if hsds_local:
endpoint = hsds_local
else:
endpoint = hsds_global
print(f"running loading {s3path} to {tgt_path}")
s3 = s3fs.S3FileSystem(use_ssl=False)
try:
fin = h5py.File(s3.open(s3path, "rb"), "r")
except IOError as ioe:
logging.error("Error opening s3path {}: {}".format(s3path, ioe))
raise
# create output domain
try:
fout = h5pyd.File(tgt_path,
'w',
endpoint=endpoint,
username=username,
password=password,
bucket=tgt_bucket)
except IOError as ioe:
if ioe.errno == 404:
logging.error("Domain: {} not found".format(tgt_path))
elif ioe.errno == 403:
logging.error("No write access to domain: {}".format(tgt_path))
else:
logging.error("Error creating file {}: {}".format(tgt_path, ioe))
raise
# do the actual load
try:
if link_files:
dataload = "link"
else:
dataload = "ingest"
compression = None # TBD
compression_opts = None # TBD
load_file(fin,
fout,
verbose=verbose,
dataload=dataload,
s3path=s3path,
compression=compression,
compression_opts=compression_opts)
except IOError as ioe:
logging.error("load_file error: {}".format(ioe))
raise
if config.get("public_read"):
# make public read, and get acl
print("adding public read ACL")
acl = {"userName": "******"}
acl["create"] = False
acl["read"] = True
acl["update"] = False
acl["delete"] = False
acl["readACL"] = True
acl["updateACL"] = False
fout.putACL(acl)
fout.close()
def main():
loglevel = logging.ERROR
verbose = False
nodata = False
deflate = None
cfg["cmd"] = sys.argv[0].split('/')[-1]
if cfg["cmd"].endswith(".py"):
cfg["cmd"] = "python " + cfg["cmd"]
cfg["logfname"] = None
logfname = None
src_files = []
argn = 1
while argn < len(sys.argv):
arg = sys.argv[argn]
val = None
if arg[0] == '-' and len(src_files) > 0:
# options must be placed before filenames
print("options must precead source files")
usage()
sys.exit(-1)
if len(sys.argv) > argn + 1:
val = sys.argv[argn + 1]
if arg in ("-v", "--verbose"):
verbose = True
argn += 1
elif arg == "--nodata":
nodata = True
argn += 1
elif arg == "--loglevel":
if val == "debug":
loglevel = logging.DEBUG
elif val == "info":
loglevel = logging.INFO
elif val == "warning":
loglevel = logging.WARNING
elif val == "error":
loglevel = logging.ERROR
else:
print("unknown loglevel")
usage()
sys.exit(-1)
argn += 2
elif arg == '--logfile':
logfname = val
argn += 2
elif arg in ("-h", "--help"):
usage()
sys.exit(0)
elif arg in ("-e", "--endpoint"):
cfg["hs_endpoint"] = val
argn += 2
elif arg in ("-u", "--username"):
cfg["hs_username"] = val
argn += 2
elif arg in ("-p", "--password"):
cfg["hs_password"] = val
argn += 2
elif arg in ("-b", "--bucket"):
cfg["hs_bucket"] = val
argn += 2
elif arg == '--cnf-eg':
print_config_example()
sys.exit(0)
elif arg.startswith("-z"):
compressLevel = 4
if len(arg) > 2:
try:
compressLevel = int(arg[2:])
except ValueError:
print("Compression Level must be int between 0 and 9")
sys.exit(-1)
deflate = compressLevel
argn += 1
elif arg[0] == '-':
usage()
sys.exit(-1)
else:
src_files.append(arg)
argn += 1
# setup logging
logging.basicConfig(
filename=logfname,
format='%(asctime)s %(filename)s:%(lineno)d %(message)s',
level=loglevel)
logging.debug("set log_level to {}".format(loglevel))
# end arg parsing
logging.info("username: {}".format(cfg["hs_username"]))
logging.info("endpoint: {}".format(cfg["hs_endpoint"]))
logging.info("verbose: {}".format(verbose))
if len(src_files) < 2:
# need at least a src and destination
usage()
sys.exit(-1)
src_domain = src_files[0]
des_domain = src_files[1]
logging.info("source domain: {}".format(src_domain))
logging.info("target domain: {}".format(des_domain))
if src_domain[0] != '/' or src_domain[-1] == '/':
print("source domain must be an absolute path, non-folder domain")
usage()
sys.exit(-1)
if des_domain[0] != '/' or des_domain[-1] == '/':
print("source domain must be an absolute path, non-folder domain")
usage()
sys.exit(-1)
if cfg["hs_endpoint"] is None:
logging.error('No endpoint given, try -h for help\n')
sys.exit(1)
logging.info("endpoint: {}".format(cfg["hs_endpoint"]))
try:
username = cfg["hs_username"]
password = cfg["hs_password"]
endpoint = cfg["hs_endpoint"]
bucket = cfg["hs_bucket"]
# get a handle to input file
try:
fin = h5pyd.File(src_domain,
mode='r',
endpoint=endpoint,
username=username,
password=password,
bucket=bucket)
except IOError as ioe:
logging.error("Error opening file {}: {}".format(src_domain, ioe))
sys.exit(1)
# create the output domain
try:
fout = h5pyd.File(des_domain,
'x',
endpoint=endpoint,
username=username,
password=password,
bucket=bucket)
except IOError as ioe:
if ioe.errno == 403:
logging.error(
"No write access to domain: {}".format(des_domain))
else:
logging.error("Error creating file {}: {}".format(
des_domain, ioe))
sys.exit(1)
# do the actual load
load_file(fin, fout, verbose=verbose, nodata=nodata, deflate=deflate)
msg = "File {} uploaded to domain: {}".format(src_domain, des_domain)
logging.info(msg)
if verbose:
print(msg)
except KeyboardInterrupt:
logging.error('Aborted by user via keyboard interrupt.')
sys.exit(1)
def main():
loglevel = logging.ERROR
verbose = False
deflate = None
s3path = None
dataload = "ingest" # or None, or "s3link"
cfg["cmd"] = sys.argv[0].split('/')[-1]
if cfg["cmd"].endswith(".py"):
cfg["cmd"] = "python " + cfg["cmd"]
cfg["logfname"] = None
logfname=None
ipvfam=None
s3 = None # s3fs instance
src_files = []
argn = 1
while argn < len(sys.argv):
arg = sys.argv[argn]
val = None
if arg[0] == '-' and len(src_files) > 0:
# options must be placed before filenames
print("options must precead source files")
usage()
sys.exit(-1)
if len(sys.argv) > argn + 1:
val = sys.argv[argn+1]
if arg in ("-v", "--verbose"):
verbose = True
argn += 1
elif arg == "--s3link":
if dataload != "ingest":
sys.stderr("--nodata flag can't be used with s3link flag")
sys.exit(1)
dataload = "s3link"
argn += 1
elif arg == "--nodata":
if dataload != "ingest":
sys.stderr("--nodata flag can't be used with s3link flag")
sys.exit(1)
dataload = None
argn += 1
elif arg == "--loglevel":
if val == "debug":
loglevel = logging.DEBUG
elif val == "info":
loglevel = logging.INFO
elif val == "warning":
loglevel = logging.WARNING
elif val == "error":
loglevel = logging.ERROR
else:
print("unknown loglevel")
usage()
sys.exit(-1)
argn += 2
elif arg == '--logfile':
logfname = val
argn += 2
elif arg in ("-b", "--bucket"):
cfg["hs_bucket"] = val
argn += 2
elif arg == '-4':
ipvfam = 4
elif arg == '-6':
ipvfam = 6
elif arg in ("-h", "--help"):
usage()
sys.exit(0)
elif arg in ("-e", "--endpoint"):
cfg["hs_endpoint"] = val
argn += 2
elif arg in ("-u", "--username"):
cfg["hs_username"] = val
argn += 2
elif arg in ("-p", "--password"):
cfg["hs_password"] = val
argn += 2
elif arg == '--cnf-eg':
print_config_example()
sys.exit(0)
elif arg.startswith("-z"):
compressLevel = 4
if len(arg) > 2:
try:
compressLevel = int(arg[2:])
except ValueError:
print("Compression Level must be int between 0 and 9")
sys.exit(-1)
deflate = compressLevel
argn += 1
elif arg[0] == '-':
usage()
sys.exit(-1)
else:
src_files.append(arg)
argn += 1
# setup logging
logging.basicConfig(filename=logfname, format='%(asctime)s %(filename)s:%(lineno)d %(message)s', level=loglevel)
logging.debug("set log_level to {}".format(loglevel))
# end arg parsing
logging.info("username: {}".format(cfg["hs_username"]))
logging.info("endpoint: {}".format(cfg["hs_endpoint"]))
logging.info("verbose: {}".format(verbose))
if len(src_files) < 2:
# need at least a src and destination
usage()
sys.exit(-1)
domain = src_files[-1]
src_files = src_files[:-1]
logging.info("source files: {}".format(src_files))
logging.info("target domain: {}".format(domain))
if len(src_files) > 1 and (domain[0] != '/' or domain[-1] != '/'):
print("target must be a folder if multiple source files are provided")
usage()
sys.exit(-1)
if cfg["hs_endpoint"] is None:
logging.error('No endpoint given, try -h for help\n')
sys.exit(1)
logging.info("endpoint: {}".format(cfg["hs_endpoint"]))
try:
for src_file in src_files:
# check if this is a non local file, if it is remote (http, etc...) stage it first then insert it into hsds
src_file_chk = urlparse(src_file)
logging.debug(src_file_chk)
if src_file_chk.scheme == 'http' or src_file_chk.scheme == 'https' or src_file_chk.scheme == 'ftp':
src_file = stage_file(src_file, netfam=ipvfam)
if src_file == None:
continue
istmp = True
logging.info('temp source data: '+str(src_file))
else:
istmp = False
tgt = domain
if tgt[-1] == '/':
# folder destination
tgt = tgt + op.basename(src_file)
# get a handle to input file
if src_file.startswith("s3://"):
s3path = src_file
if not S3FS_IMPORT:
sys.stderr.write("Install S3FS package to load s3 files")
sys.exit(1)
if h5py.version.version_tuple.major != 2 or h5py.version.version_tuple.minor < 10:
print("s3path source requires h5py version 2.10 or higher")
sys.exit(1)
if h5py.version.hdf5_version_tuple[0] != 1 or h5py.version.hdf5_version_tuple[1] != 10 or h5py.version.hdf5_version_tuple[2] < 6:
print("s3path source requires hdf5 lib version 1.10.6 or higher")
sys.exit(1)
if not s3:
s3 = s3fs.S3FileSystem()
try:
fin = h5py.File(s3.open(src_file, "rb"), "r")
except IOError as ioe:
logging.error("Error opening file {}: {}".format(src_file, ioe))
sys.exit(1)
else:
s3path = None
try:
fin = h5py.File(src_file, mode='r')
except IOError as ioe:
logging.error("Error opening file {}: {}".format(src_file, ioe))
sys.exit(1)
# create the output domain
try:
username = cfg["hs_username"]
password = cfg["hs_password"]
endpoint = cfg["hs_endpoint"]
bucket = cfg["hs_bucket"]
fout = h5pyd.File(tgt, 'x', endpoint=endpoint, username=username, password=password, bucket=bucket)
except IOError as ioe:
if ioe.errno == 404:
logging.error("Domain: {} not found".format(tgt))
elif ioe.errno == 403:
logging.error("No write access to domain: {}".format(tgt))
else:
logging.error("Error creating file {}: {}".format(tgt, ioe))
sys.exit(1)
# do the actual load
load_file(fin, fout, verbose=verbose, dataload=dataload, s3path=s3path, deflate=deflate,)
# cleanup if needed
if istmp:
try:
os.unlink(src_file)
except IOError as e:
logging.warn("failed to delete %s : %s" % (src_file, str(e)))
msg = "File {} uploaded to domain: {}".format(src_file, tgt)
logging.info(msg)
if verbose:
print(msg)
except KeyboardInterrupt:
logging.error('Aborted by user via keyboard interrupt.')
sys.exit(1)
