def upload(self, filepath, shard_size=0, start_pos=0, read_size=1024, callback=None):
""" Uploads a shard via POST to the specified node
to the web-core API. See API docs:
https://github.com/Storj/web-core#api-documentation
:param filepath: Path to file as a string
:return: upstream.shard.Shard
:raise LookupError: IF
"""
# Open the file and upload it via POST
url = self.server + "/api/upload" # web-core API
r = self._upload_form_encoded(
url, filepath, shard_size=shard_size, start_pos=start_pos, read_size=read_size, callback=callback
)
# Make sure that the API call is actually there
if r.status_code == 404:
raise ResponseError("API call not found.")
elif r.status_code == 402:
raise ResponseError("Payment required.")
elif r.status_code == 500:
raise ResponseError("Server error.")
elif r.status_code == 201:
shard = Shard()
shard.from_json(r.text)
return shard
else:
err = ResponseError("Received status code %s %s" % (r.status_code, r.reason))
err.response = r
raise err
def setUp(self):
self.cryptkey = ("2b77e64156f9f7eb16d74b98f70417e4"
"d665d977d0ef00e793d41767acf13e8c")
self.filehash = ("5547a152337de9ff6a97f6f099bb024e"
"08af419cee613b18da76a33e581d49ac")
self.raw_uri = (
"5547a152337de9ff6a97f6f099bb024e08af419cee613b18da76a33e581d49ac"
"?key=2b77e64156f9f7eb16d74b98f70417e4d665d977d0ef00e793d41767acf"
"13e8c")
# Create empty object
self.empty_shard = Shard()
# Create half object
self.half_shard = Shard(self.filehash)
# Create full object
self.full_shard = Shard(self.filehash, self.cryptkey)
# Create new objects
self.shard1 = Shard()
self.shard2 = Shard()
# Load content
self.json_dict = {
"key": "2b77e64156f9f7eb16d74b98f70417e4"
"d665d977d0ef00e793d41767acf13e8c",
"filehash": "5547a152337de9ff6a97f6f099bb024e"
"08af419cee613b18da76a33e581d49ac"
}
self.shard1.from_json(json.dumps(self.json_dict))
self.shard2.from_uri(self.raw_uri)
def setUp(self):
self.cryptkey = ("2b77e64156f9f7eb16d74b98f70417e4"
"d665d977d0ef00e793d41767acf13e8c")
self.filehash = ("5547a152337de9ff6a97f6f099bb024e"
"08af419cee613b18da76a33e581d49ac")
self.raw_uri = (
"5547a152337de9ff6a97f6f099bb024e08af419cee613b18da76a33e581d49ac"
"?key=2b77e64156f9f7eb16d74b98f70417e4d665d977d0ef00e793d41767acf"
"13e8c")
# Create empty object
self.empty_shard = Shard()
# Create half object
self.half_shard = Shard(self.filehash)
# Create full object
self.full_shard = Shard(self.filehash, self.cryptkey)
# Create new objects
self.shard1 = Shard()
self.shard2 = Shard()
# Load content
self.json_dict = {
"key":
"2b77e64156f9f7eb16d74b98f70417e4"
"d665d977d0ef00e793d41767acf13e8c",
"filehash":
"5547a152337de9ff6a97f6f099bb024e"
"08af419cee613b18da76a33e581d49ac"
}
self.shard1.from_json(json.dumps(self.json_dict))
self.shard2.from_uri(self.raw_uri)
def setUp(self):
self.stream = Streamer("http://node1.metadisk.org")
self.orig_hash = None
self.uploadfile = "tests/1k.testfile"
self.downloadfile = "download.testfile"
self.shard = Shard(
"2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
"1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14")
def download(args):
""" Controls actions for downloading
:param args: Argparse namespace
"""
shards = []
for uri in args.uri:
if args.verbose:
print("Creating shard.")
shard = Shard()
shard.from_uri(uri)
shards.append(shard)
if args.verbose:
print("There are %d shards to download." % len(shards))
streamer = Streamer(args.server)
if args.verbose:
print("Connecting to %s..." % streamer.server)
path, fname = check_and_get_dest(args.dest)
savepath = os.path.join(path, fname)
for i, shard in enumerate(shards):
if args.verbose:
print("Downloading file %s..." % shard.uri)
else:
print("Downloading file %d..." % (i + 1))
sys.stdout.flush()
try:
r = streamer.download(shard, slicesize=8096)
except ResponseError as e:
sys.stderr.write("\nError!\n")
sys.stderr.write("%s %s\n" % (e.response.status_code,
e.response.reason))
sys.stderr.write("%s\n" % e.response.text)
raise
with open(savepath, 'ab') as f:
if args.verbose:
print("Writing shard.")
for _bytes in r.iter_content():
f.write(_bytes)
print("\nDownloaded to %s." % savepath)
return fname
def download(args):
""" Controls actions for downloading
:param args: Argparse namespace
"""
shards = []
for uri in args.uri:
if args.verbose:
print("Creating shard.")
shard = Shard()
shard.from_uri(uri)
shards.append(shard)
if args.verbose:
print("There are %d shards to download." % len(shards))
streamer = Streamer(args.server)
if args.verbose:
print("Connecting to %s..." % streamer.server)
path, fname = check_and_get_dest(args.dest)
savepath = os.path.join(path, fname)
for i, shard in enumerate(shards):
if args.verbose:
print("Downloading file %s..." % shard.uri)
else:
print("Downloading file %d..." % (i + 1))
sys.stdout.flush()
r = streamer.download(shard, slicesize=8096)
with open(savepath, 'ab') as f:
if args.verbose:
print("Writing shard.")
for _bytes in r.iter_content():
f.write(_bytes)
print("\nDownloaded to %s." % savepath)
return fname
def upload(self,
filepath,
shard_size=0,
start_pos=0,
read_size=1024,
callback=None):
""" Uploads a shard via POST to the specified node
to the web-core API. See API docs:
https://github.com/Storj/web-core#api-documentation
:param filepath: Path to file as a string
:return: upstream.shard.Shard
:raise LookupError: IF
"""
# Open the file and upload it via POST
url = self.server + "/api/upload" # web-core API
r = self._upload_form_encoded(url,
filepath,
shard_size=shard_size,
start_pos=start_pos,
read_size=read_size,
callback=callback)
# Make sure that the API call is actually there
if r.status_code == 404:
raise ResponseError("API call not found.")
elif r.status_code == 402:
raise ResponseError("Payment required.")
elif r.status_code == 500:
raise ResponseError("Server error.")
elif r.status_code == 201:
shard = Shard()
shard.from_json(r.text)
return shard
else:
err = ResponseError("Received status code %s %s" %
(r.status_code, r.reason))
err.response = r
raise err
def setUp(self):
self.stream = Streamer("http://node1.metadisk.org")
self.orig_hash = None
self.uploadfile = "tests/1k.testfile"
self.downloadfile = "download.testfile"
self.shard = Shard(
"2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
"1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
)
self.args = mock.MagicMock()
self.args.verbose = False
self.args.server = 'http://node1.metadisk.org'
self.args.uri = [self.shard.uri]
self.args.file = self.uploadfile
self.args.dest = self.downloadfile
self.args.shard_size = SizeHelpers.mib_to_bytes(250)
class TestShard(unittest.TestCase):
def setUp(self):
self.cryptkey = ("2b77e64156f9f7eb16d74b98f70417e4"
"d665d977d0ef00e793d41767acf13e8c")
self.filehash = ("5547a152337de9ff6a97f6f099bb024e"
"08af419cee613b18da76a33e581d49ac")
self.raw_uri = (
"5547a152337de9ff6a97f6f099bb024e08af419cee613b18da76a33e581d49ac"
"?key=2b77e64156f9f7eb16d74b98f70417e4d665d977d0ef00e793d41767acf"
"13e8c")
# Create empty object
self.empty_shard = Shard()
# Create half object
self.half_shard = Shard(self.filehash)
# Create full object
self.full_shard = Shard(self.filehash, self.cryptkey)
# Create new objects
self.shard1 = Shard()
self.shard2 = Shard()
# Load content
self.json_dict = {
"key": "2b77e64156f9f7eb16d74b98f70417e4"
"d665d977d0ef00e793d41767acf13e8c",
"filehash": "5547a152337de9ff6a97f6f099bb024e"
"08af419cee613b18da76a33e581d49ac"
}
self.shard1.from_json(json.dumps(self.json_dict))
self.shard2.from_uri(self.raw_uri)
def tearDown(self):
del self.empty_shard
del self.half_shard
del self.full_shard
del self.shard1
del self.shard2
del self.json_dict
def test_getters_empty_shard(self):
def _callable(meth):
meth = getattr(self.empty_shard, meth)
meth()
for method in ['uri', 'get_hashes', 'get_json']:
self.assertRaises(ShardError, _callable, method)
def test_getters_half_shard(self):
def _callable(meth):
meth = getattr(self.half_shard, meth)
meth()
for method in ['uri', 'get_hashes', 'get_json']:
self.assertRaises(ShardError, _callable, method)
def test_getters_full_shard(self):
uri = self.full_shard.uri
self.assertEqual(uri, self.raw_uri)
hash, key = self.full_shard.get_hashes()
self.assertEqual(hash, self.filehash)
self.assertEqual(key, self.cryptkey)
json_ = self.full_shard.get_json()
self.assertEqual(json_, json.dumps(self.json_dict))
def test_shards(self):
# Fit everything into one test case
self.assertEqual(self.shard1.get_hashes(), self.shard2.get_hashes())
def test_download_empty_shard(self):
shard = Shard()
with self.assertRaises(ShardError) as e:
self.stream.download(shard)
self.assertEqual(str(e.exception), "Shard missing filehash.")
def test_shard_with_malformed_uri(self):
shard = Shard()
self.assertRaises(ShardError, shard.from_uri, 'funkymonkey')
class TestShard(unittest.TestCase):
def setUp(self):
self.cryptkey = ("2b77e64156f9f7eb16d74b98f70417e4"
"d665d977d0ef00e793d41767acf13e8c")
self.filehash = ("5547a152337de9ff6a97f6f099bb024e"
"08af419cee613b18da76a33e581d49ac")
self.raw_uri = (
"5547a152337de9ff6a97f6f099bb024e08af419cee613b18da76a33e581d49ac"
"?key=2b77e64156f9f7eb16d74b98f70417e4d665d977d0ef00e793d41767acf"
"13e8c")
# Create empty object
self.empty_shard = Shard()
# Create half object
self.half_shard = Shard(self.filehash)
# Create full object
self.full_shard = Shard(self.filehash, self.cryptkey)
# Create new objects
self.shard1 = Shard()
self.shard2 = Shard()
# Load content
self.json_dict = {
"key":
"2b77e64156f9f7eb16d74b98f70417e4"
"d665d977d0ef00e793d41767acf13e8c",
"filehash":
"5547a152337de9ff6a97f6f099bb024e"
"08af419cee613b18da76a33e581d49ac"
}
self.shard1.from_json(json.dumps(self.json_dict))
self.shard2.from_uri(self.raw_uri)
def tearDown(self):
del self.empty_shard
del self.half_shard
del self.full_shard
del self.shard1
del self.shard2
del self.json_dict
def test_shard_with_malformed_uri(self):
shard = Shard()
self.assertRaises(ShardError, shard.from_uri, 'funkymonkey')
def test_getters_empty_shard(self):
def _callable(meth):
meth = getattr(self.empty_shard, meth)
meth()
for method in ['uri', 'get_hashes', 'get_json']:
self.assertRaises(ShardError, _callable, method)
def test_getters_half_shard(self):
def _callable(meth):
meth = getattr(self.half_shard, meth)
meth()
for method in ['uri', 'get_hashes', 'get_json']:
self.assertRaises(ShardError, _callable, method)
def test_getters_full_shard(self):
uri = self.full_shard.uri
self.assertEqual(uri, self.raw_uri)
hash, key = self.full_shard.get_hashes()
self.assertEqual(hash, self.filehash)
self.assertEqual(key, self.cryptkey)
json_ = self.full_shard.get_json()
self.assertEqual(json_, json.dumps(self.json_dict))
def test_shards(self):
# Fit everything into one test case
self.assertEqual(self.shard1.get_hashes(), self.shard2.get_hashes())
