def storageBackend(script):
if len(script.args) <= 1:
logger.error(
"Storage backend must be specified, either 'swift', 's3', 'noop', or 'disk'"
)
sys.exit(1)
if script.args[1].lower() == "disk":
import disk
return disk.Disk(script.options.bucket, script.options.noop)
if script.args[1].lower() == "noop":
import noop
return noop.NoOP(script.options.bucket, script.options.noop)
if script.args[1].lower() == "s3":
import s3
if len(script.args) > 2:
region = script.args[2]
else:
region = "us-east-1"
return s3.S3(script.options.bucket, script.options.sse_c_key, region,
script.options.noop)
if script.args[1].lower() == "swift":
import swift
return swift.Swift(script.options.bucket, script.options.noop)
logger.error(
"Invalid storage backend, must be 'swift', 's3', 'noop', or 'disk'")
sys.exit(1)
def storageBackend(script):
if len(script.args) <= 1:
logger.error(
"Storage backend must be specified, either: disk, gcs, noop, s3, or swift"
)
sys.exit(1)
if script.args[1].lower() == "disk":
import disk
return disk.Disk(script.options.bucket, script.options.noop), None
if script.args[1].lower() == "hdfs":
import hdfs
import disk
hdfsargs = {"endpoint": "localhost:8087"}
for i in script.args[2:]:
fields = i.split("=")
if len(fields) > 1:
hdfsargs[fields[0]] = fields[1]
else:
hdfsargs["endpoint"] = fields[0]
return disk.Disk(script.options.bucket,
script.options.noop), hdfs.hdfs(
script.options.bucket, hdfsargs["endpoint"],
script.options.noop)
if script.args[1].lower() == "noop":
import noop
return noop.NoOP(script.options.bucket, script.options.noop), None
logger.error(
"Invalid storage backend, must be: hdfs, disk, gcs, noop, s3, or swift"
)
sys.exit(1)
def storageBackend(script):
if len(script.args) <= 1:
logger.error("Storage backend must be specified, either 'swift' or 's3'")
sys.exit(1)
if script.args[1].lower() == "noop":
import noop
return noop.NoOP(script.options.bucket, script.options.noop)
if script.args[1].lower() == "s3":
import s3
return s3.S3(script.options.bucket,
script.options.keyprefix,
script.options.noop)
if script.args[1].lower() == "swift":
import swift
return swift.Swift(script.options.bucket, script.options.noop)
logger.error("Invalid storage backend, must be 'swift', 's3', or 'noop'")
sys.exit(1)
def storageBackend(script):
if len(script.args) <= 1:
logger.error(
"Storage backend must be specified, either: disk, gcs, noop, s3, or swift"
)
sys.exit(1)
if script.args[1].lower() == "disk":
import disk
return disk.Disk(script.options.bucket, script.options.noop)
if script.args[1].lower() == "noop":
import noop
return noop.NoOP(script.options.bucket, script.options.noop)
if script.args[1].lower() == "s3":
import s3
s3args = {"region": "us-east-1"}
for i in script.args[2:]:
fields = i.split("=")
if len(fields) > 1:
gcsargs[fields[0]] = fields[1]
else:
gcsargs["region"] = fields[0]
return s3.S3(script.options.bucket, s3args["region"],
script.options.noop)
if script.args[1].lower() == "swift":
import swift
return swift.Swift(script.options.bucket, script.options.noop)
if script.args[1].lower() == "gcs":
import gcs
gcsargs = {"project": "", "region": "us"}
for i in script.args[2:]:
fields = i.split("=")
if len(fields) > 1:
gcsargs[fields[0]] = fields[1]
else:
gcsargs["region"] = fields[0]
return gcs.GCS(script.options.bucket, gcsargs["project"],
gcsargs["region"], script.options.noop)
logger.error(
"Invalid storage backend, must be: disk, gcs, noop, s3, or swift")
sys.exit(1)