def upload(args):
""" Controls actions for uploading
:param args: Parsed args namespace
"""
shard_size = parse_shard_size(args.shard_size)
try:
filepath = Streamer.check_path(args.file)
except FileError as e:
sys.stderr.write('%s\n' % str(e))
sys.exit(1)
streamer = Streamer(args.server)
shards = calculate_shards(args, shard_size, filepath)
shard_info = []
for idx, shard in enumerate(shards):
i = idx + 1
start = shard[0]
callback = ProgressCallback()
shard = streamer.upload(args.file,
start_pos=start,
shard_size=shard_size,
callback=callback.callback)
callback.bar.finish()
sys.stdout.flush()
if args.verbose:
print("\nShard %d - URI: %s\n" % (i, shard.uri))
shard_info.append(shard.uri)
print()
print("Download this file by using the following command: ")
print("upstream download --uri", " ".join(shard_info), "--dest <filename>")
def upload(args):
""" Controls actions for uploading
:param args: Parsed args namespace
"""
shard_size = parse_shard_size(args.shard_size)
try:
filepath = Streamer.check_path(args.file)
except FileError as e:
sys.stderr.write('%s\n' % str(e))
sys.exit(1)
streamer = Streamer(args.server)
shards = calculate_shards(args, shard_size, filepath)
shard_info = []
for idx, shard in enumerate(shards):
i = idx + 1
start = shard[0]
callback = ProgressCallback()
shard = streamer.upload(
args.file,
start_pos=start,
shard_size=shard_size,
callback=callback.callback
)
callback.bar.finish()
sys.stdout.flush()
if args.verbose:
print("\nShard %d - URI: %s\n" % (i, shard.uri))
shard_info.append(shard.uri)
print()
print("Download this file by using the following command: ")
print("upstream download --uri", " ".join(shard_info), "--dest <filename>")
def setUp(self):
self.stream = Streamer("http://node1.metadisk.org")
self.orig_hash = None
self.uploadfile = "tests/1k.testfile"
self.downloadfile = "download.testfile"
self.shard = Shard(
"2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
"1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14")
def setUp(self):
self.stream = Streamer("http://node1.metadisk.org")
self.orig_hash = None
self.uploadfile = "tests/1k.testfile"
self.downloadfile = "download.testfile"
self.shard = Shard(
"2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
"1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
)
def download(args):
""" Controls actions for downloading
:param args: Argparse namespace
"""
shards = []
for uri in args.uri:
if args.verbose:
print("Creating shard.")
shard = Shard()
shard.from_uri(uri)
shards.append(shard)
if args.verbose:
print("There are %d shards to download." % len(shards))
streamer = Streamer(args.server)
if args.verbose:
print("Connecting to %s..." % streamer.server)
path, fname = check_and_get_dest(args.dest)
savepath = os.path.join(path, fname)
for i, shard in enumerate(shards):
if args.verbose:
print("Downloading file %s..." % shard.uri)
else:
print("Downloading file %d..." % (i + 1))
sys.stdout.flush()
try:
r = streamer.download(shard, slicesize=8096)
except ResponseError as e:
sys.stderr.write("\nError!\n")
sys.stderr.write("%s %s\n" % (e.response.status_code,
e.response.reason))
sys.stderr.write("%s\n" % e.response.text)
raise
with open(savepath, 'ab') as f:
if args.verbose:
print("Writing shard.")
for _bytes in r.iter_content():
f.write(_bytes)
print("\nDownloaded to %s." % savepath)
return fname
def download(args):
""" Controls actions for downloading
:param args: Argparse namespace
"""
shards = []
for uri in args.uri:
if args.verbose:
print("Creating shard.")
shard = Shard()
shard.from_uri(uri)
shards.append(shard)
if args.verbose:
print("There are %d shards to download." % len(shards))
streamer = Streamer(args.server)
if args.verbose:
print("Connecting to %s..." % streamer.server)
path, fname = check_and_get_dest(args.dest)
savepath = os.path.join(path, fname)
for i, shard in enumerate(shards):
if args.verbose:
print("Downloading file %s..." % shard.uri)
else:
print("Downloading file %d..." % (i + 1))
sys.stdout.flush()
r = streamer.download(shard, slicesize=8096)
with open(savepath, 'ab') as f:
if args.verbose:
print("Writing shard.")
for _bytes in r.iter_content():
f.write(_bytes)
print("\nDownloaded to %s." % savepath)
return fname
def setUp(self):
self.stream = Streamer("http://node1.metadisk.org")
self.orig_hash = None
self.uploadfile = "tests/1k.testfile"
self.downloadfile = "download.testfile"
self.shard = Shard(
"2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
"1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
)
self.args = mock.MagicMock()
self.args.verbose = False
self.args.server = 'http://node1.metadisk.org'
self.args.uri = [self.shard.uri]
self.args.file = self.uploadfile
self.args.dest = self.downloadfile
self.args.shard_size = SizeHelpers.mib_to_bytes(250)
class TestStreamer(unittest.TestCase):
def setUp(self):
self.stream = Streamer("http://node1.metadisk.org")
self.orig_hash = None
self.uploadfile = "tests/1k.testfile"
self.downloadfile = "download.testfile"
self.shard = Shard(
"2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
"1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
)
def tearDown(self):
del self.stream
del self.orig_hash
del self.uploadfile
try:
os.remove(self.downloadfile)
except:
pass
try:
os.remove(self.shard.filehash)
except:
pass
del self.downloadfile
del self.shard
def test_initialization(self):
self.assertEqual(self.stream.server, "http://node1.metadisk.org")
def test_check_connectivity(self):
def _failing_connection():
Streamer("http://does.not.exist")
self.assertRaises(ConnectError, _failing_connection)
@mock.patch('requests.post')
def test_upload_form_encoded(self, post):
pass
@mock.patch('requests.post')
def test_upload_sharded_encoded(self, post):
pass
def test_upload(self):
# Upload file and check file
self.shard = self.stream.upload(self.uploadfile)
self.assertEqual(
self.shard.filehash,
"2032e4fd19d4ab49a74ead0984a5f672"
"c26e60da6e992eaf51f05dc874e94bd7")
self.assertEqual(
self.shard.decryptkey,
"1b1f463cef1807a127af668f3a4fdcc7"
"977c647bf2f357d9fa125f13548b1d14")
# Try to upload wrong file
def _failing_upload():
self.stream.upload("not-a-real-file")
self.assertRaises(FileError, _failing_upload)
def test_upload_patched_404(self):
self.stream._upload_form_encoded = mock.MagicMock()
self.stream._upload_form_encoded.return_value()
self.stream._upload_form_encoded.return_value.status_code = 404
def _fourohfour():
self.stream.upload(self.uploadfile)
with self.assertRaises(ResponseError) as ex:
_fourohfour()
self.assertEqual(ex.message, "API call not found.")
def test_upload_patched_402(self):
self.stream._upload_form_encoded = mock.MagicMock()
self.stream._upload_form_encoded.return_value()
self.stream._upload_form_encoded.return_value.status_code = 402
def _fourohtwo():
self.stream.upload(self.uploadfile)
with self.assertRaises(ResponseError) as ex:
_fourohtwo()
def test_upload_patched_500(self):
self.stream._upload_form_encoded = mock.MagicMock()
self.stream._upload_form_encoded.return_value()
self.stream._upload_form_encoded.return_value.status_code = 500
def _fivehundred():
self.stream.upload(self.uploadfile)
with self.assertRaises(ResponseError) as ex:
_fivehundred()
self.assertEqual(ex.message, "Server error.")
def test_upload_patched_501(self):
self.stream._upload_form_encoded = mock.MagicMock()
self.stream._upload_form_encoded.return_value()
self.stream._upload_form_encoded.return_value.status_code = 501
self.stream._upload_form_encoded.return_value.reason = "Not Implemented"
def _fiveohone():
self.stream.upload(self.uploadfile)
with self.assertRaises(ResponseError) as ex:
_fiveohone()
self.assertEqual(ex.message,
"Received status code 501 Not Implemented")
def test_upload_check_path(self):
homedir = os.path.expanduser(self.uploadfile)
result = self.stream.check_path(self.uploadfile)
self.assertEqual(homedir, result)
with self.assertRaises(FileError) as ex:
self.stream.check_path('~/does-not-exist')
self.assertEqual(
ex.message, '~/does-not-exist not a file or not found')
def test_download(self):
r = self.stream.download(self.shard)
self.assertTrue(r)
self.assertEquals(r.status_code, 200)
self.assertEqual(len(r.content), 1024)
def test_download_exception(self):
self.shard.filehash = self.shard.filehash[:-5]
with self.assertRaises(ResponseError) as ex:
r = self.stream.download(self.shard)
self.assertEqual(ex.exception.response.status_code, 404)
def test_download_empty_shard(self):
shard = Shard()
with self.assertRaises(ShardError) as e:
self.stream.download(shard)
self.assertEqual(str(e.exception), "Shard missing filehash.")
def _failing_connection():
Streamer("http://does.not.exist")
class TestStreamer(unittest.TestCase):
def setUp(self):
self.stream = Streamer("http://node1.metadisk.org")
self.orig_hash = None
self.uploadfile = "tests/1k.testfile"
self.downloadfile = "download.testfile"
self.shard = Shard(
"2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
"1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14")
def tearDown(self):
del self.stream
del self.orig_hash
del self.uploadfile
try:
os.remove(self.downloadfile)
except:
pass
try:
os.remove(self.shard.filehash)
except:
pass
del self.downloadfile
del self.shard
def test_initialization(self):
self.assertEqual(self.stream.server, "http://node1.metadisk.org")
def test_check_connectivity(self):
def _failing_connection():
Streamer("http://does.not.exist")
self.assertRaises(ConnectError, _failing_connection)
@mock.patch('requests.post')
def test_upload_form_encoded(self, post):
pass
@mock.patch('requests.post')
def test_upload_sharded_encoded(self, post):
with self.assertRaises(NotImplementedError):
self.stream._upload_sharded_encoded('http://fake.url', 'fake.path')
@mock.patch('requests.post')
def test_filestream(self, post):
with self.assertRaises(NotImplementedError):
self.stream._filestream('fake.path')
def test_upload(self):
# Upload file and check file
self.shard = self.stream.upload(self.uploadfile)
self.assertEqual(
self.shard.filehash, "2032e4fd19d4ab49a74ead0984a5f672"
"c26e60da6e992eaf51f05dc874e94bd7")
self.assertEqual(
self.shard.decryptkey, "1b1f463cef1807a127af668f3a4fdcc7"
"977c647bf2f357d9fa125f13548b1d14")
# Try to upload wrong file
def _failing_upload():
self.stream.upload("not-a-real-file")
self.assertRaises(FileError, _failing_upload)
def test_upload_patched_404(self):
self.stream._upload_form_encoded = mock.MagicMock()
self.stream._upload_form_encoded.return_value()
self.stream._upload_form_encoded.return_value.status_code = 404
def _fourohfour():
self.stream.upload(self.uploadfile)
with self.assertRaises(ResponseError) as ex:
_fourohfour()
self.assertEqual(ex.message, "API call not found.")
def test_upload_patched_402(self):
self.stream._upload_form_encoded = mock.MagicMock()
self.stream._upload_form_encoded.return_value()
self.stream._upload_form_encoded.return_value.status_code = 402
def _fourohtwo():
self.stream.upload(self.uploadfile)
with self.assertRaises(ResponseError):
_fourohtwo()
def test_upload_patched_500(self):
self.stream._upload_form_encoded = mock.MagicMock()
self.stream._upload_form_encoded.return_value()
self.stream._upload_form_encoded.return_value.status_code = 500
def _fivehundred():
self.stream.upload(self.uploadfile)
with self.assertRaises(ResponseError) as ex:
_fivehundred()
self.assertEqual(ex.message, "Server error.")
def test_upload_patched_501(self):
self.stream._upload_form_encoded = mock.MagicMock()
self.stream._upload_form_encoded.return_value()
self.stream._upload_form_encoded.return_value.status_code = 501
self.stream._upload_form_encoded.return_value.reason =\
"Not Implemented"
def _fiveohone():
self.stream.upload(self.uploadfile)
with self.assertRaises(ResponseError) as ex:
_fiveohone()
self.assertEqual(ex.message,
"Received status code 501 Not Implemented")
def test_upload_check_path(self):
homedir = os.path.expanduser(self.uploadfile)
result = self.stream.check_path(self.uploadfile)
self.assertEqual(homedir, result)
with self.assertRaises(FileError) as ex:
self.stream.check_path('~/does-not-exist')
self.assertEqual(ex.message,
'~/does-not-exist not a file or not found')
def test_download(self):
r = self.stream.download(self.shard)
self.assertTrue(r)
self.assertEquals(r.status_code, 200)
self.assertEqual(len(r.content), 1024)
def test_download_exception(self):
self.shard.filehash = self.shard.filehash[:-5]
with self.assertRaises(ResponseError) as ex:
self.stream.download(self.shard)
self.assertEqual(ex.exception.response.status_code, 404)
def test_download_empty_shard(self):
shard = Shard()
with self.assertRaises(ShardError) as e:
self.stream.download(shard)
self.assertEqual(str(e.exception), "Shard missing filehash.")