def test_instantiation(self):
self.assertTrue(self.shard._f_obj)
self.assertTrue(SizeHelpers.mib_to_bytes(25), self.shard.shard_size)
self.assertEqual(self.shard._f_obj.tell(), 0)
self.assertEqual(self.shard.filesize, SizeHelpers.mib_to_bytes(1))
self.assertEqual(self.shard.max_seek, SizeHelpers.mib_to_bytes(1))
self.assertFalse(hasattr(self.shard, 'callback'))
def test_instantiation(self):
self.assertTrue(self.shard._f_obj)
self.assertTrue(SizeHelpers.mib_to_bytes(25), self.shard.shard_size)
self.assertEqual(self.shard._f_obj.tell(), 0)
self.assertEqual(self.shard.filesize, SizeHelpers.mib_to_bytes(1))
self.assertEqual(self.shard.max_seek, SizeHelpers.mib_to_bytes(1))
self.assertFalse(hasattr(self.shard, 'callback'))
def _upload_form_encoded(self, url, filepath, shard_size, start_pos,
read_size=1024, callback=None):
""" Streams file from disk and uploads it.
:param url: API endpoint as URL to upload to
:param filepath: Path to file as string
:return: requests.Response
"""
validpath = self.check_path(filepath)
if shard_size == 0:
shard_size = SizeHelpers.mib_to_bytes(250)
shard = ShardFile(
validpath, 'rb',
shard_size=shard_size,
start_pos=start_pos,
read_size=read_size,
callback=callback
)
m = MultipartEncoder({
'file': ('file', shard)
})
headers = {
'Content-Type': m.content_type
}
return requests.post(url, data=m, headers=headers)
def _upload_form_encoded(self,
url,
filepath,
shard_size,
start_pos,
read_size=1024,
callback=None):
""" Streams file from disk and uploads it.
:param url: API endpoint as URL to upload to
:param filepath: Path to file as string
:return: requests.Response
"""
validpath = self.check_path(filepath)
if shard_size == 0:
shard_size = SizeHelpers.mib_to_bytes(250)
shard = ShardFile(validpath,
'rb',
shard_size=shard_size,
start_pos=start_pos,
read_size=read_size,
callback=callback)
m = MultipartEncoder({'file': ('file', shard)})
headers = {'Content-Type': m.content_type}
return requests.post(url, data=m, headers=headers)
def test_parse_shard_size(self):
result = clitool.parse_shard_size('1024')
self.assertEqual(result, 1024)
result = clitool.parse_shard_size('1024b')
self.assertEqual(result, 1024)
result = clitool.parse_shard_size('1k')
self.assertEqual(result, 1024)
result = clitool.parse_shard_size('1m')
self.assertEqual(result, SizeHelpers.mib_to_bytes(1))
result = clitool.parse_shard_size('1g')
self.assertIs(result, None)
def test_parse_shard_size(self):
result = clitool.parse_shard_size('1024')
self.assertEqual(result, 1024)
result = clitool.parse_shard_size('1024b')
self.assertEqual(result, 1024)
result = clitool.parse_shard_size('1k')
self.assertEqual(result, 1024)
result = clitool.parse_shard_size('1m')
self.assertEqual(result, SizeHelpers.mib_to_bytes(1))
result = clitool.parse_shard_size('1g')
self.assertIs(result, None)
def parse_shard_size(size):
""" Parse a string, expected to be in the format of, for example:
250m, 512k, 1024b. A string without a trailing letter is assumed to be
bytes. Accepts the following: m as mebibytes, k as kibibytes, b as bytes,
:param size:
:return:
"""
size = str(size)
if size.isdigit():
return int(size)
last = size[-1].lower()
number = size[:-1]
choices = ['b', 'k', 'm']
if last not in choices:
return
if last == 'b':
return int(number)
elif last == 'k':
return SizeHelpers.kib_to_bytes(int(number))
elif last == 'm':
return SizeHelpers.mib_to_bytes(int(number))
def parse_shard_size(size):
""" Parse a string, expected to be in the format of, for example:
250m, 512k, 1024b. A string without a trailing letter is assumed to be
bytes. Accepts the following: m as mebibytes, k as kibibytes, b as bytes,
:param size:
:return:
"""
size = str(size)
if size.isdigit():
return int(size)
last = size[-1].lower()
number = size[:-1]
choices = ['b', 'k', 'm']
if last not in choices:
return
if last == 'b':
return int(number)
elif last == 'k':
return SizeHelpers.kib_to_bytes(int(number))
elif last == 'm':
return SizeHelpers.mib_to_bytes(int(number))
def setUp(self):
self.stream = Streamer("http://node1.metadisk.org")
self.orig_hash = None
self.uploadfile = "tests/1k.testfile"
self.downloadfile = "download.testfile"
self.shard = Shard(
"2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
"1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
)
self.args = mock.MagicMock()
self.args.verbose = False
self.args.server = 'http://node1.metadisk.org'
self.args.uri = [self.shard.uri]
self.args.file = self.uploadfile
self.args.dest = self.downloadfile
self.args.shard_size = SizeHelpers.mib_to_bytes(250)
def setUp(self):
self.stream = Streamer("http://node1.metadisk.org")
self.orig_hash = None
self.uploadfile = "tests/1k.testfile"
self.downloadfile = "download.testfile"
self.shard = Shard(
"2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
"1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
)
self.args = mock.MagicMock()
self.args.verbose = False
self.args.server = 'http://node1.metadisk.org'
self.args.uri = [self.shard.uri]
self.args.file = self.uploadfile
self.args.dest = self.downloadfile
self.args.shard_size = SizeHelpers.mib_to_bytes(250)
def parse_args():
""" Parses args
:return: argparse namespace
"""
parser = argparse.ArgumentParser("Upstream",
description="Command line client for "
"the Storj web-core API")
parser.add_argument('--server',
default='http://node1.metadisk.org',
help='Metadisk node to connect to')
parser.add_argument('-v',
dest='verbose',
action='store_true',
help='Verbose output')
parser.add_argument('--version',
action='version',
version="%(prog)s " + upstream.__version__,
help='Display version.')
subparser = parser.add_subparsers(dest='action')
upload_parser = subparser.add_parser('upload',
help="Upload a file from API")
upload_parser.add_argument('--shard-size',
default=SizeHelpers.mib_to_bytes(250),
help='Size of shards to break file into and '
'to upload, max: 250m, default: 250m. '
'Ex. 25m - file will be broken into 25 MB '
'shards and uploaded shard by shard')
upload_parser.add_argument('file', help="Path to file to upload")
download_parser = subparser.add_parser('download',
help="Download a file from API")
download_parser.add_argument(
'--uri',
required=True,
nargs='+',
help='URI, or URIs, of file to download. Accepts multiple values, '
'space separated. If multiple URIs are specified, the URIs are '
'joined to create a single file')
download_parser.add_argument('--dest',
help="Folder or file to download file")
download_parser.add_argument('--shard-size', type=int, default=1024)
return parser.parse_args()
def parse_args():
""" Parses args
:return: argparse namespace
"""
parser = argparse.ArgumentParser("Upstream",
description="Command line client for "
"the Storj web-core API")
parser.add_argument('--server', default='http://node1.metadisk.org',
help='Metadisk node to connect to')
parser.add_argument('-v', dest='verbose',
action='store_true', help='Verbose output')
parser.add_argument('--version', action='version',
version="%(prog)s " + upstream.__version__,
help='Display version.')
subparser = parser.add_subparsers(dest='action')
upload_parser = subparser.add_parser('upload',
help="Upload a file from API")
upload_parser.add_argument('--shard-size',
default=SizeHelpers.mib_to_bytes(250),
help='Size of shards to break file into and '
'to upload, max: 250m, default: 250m. '
'Ex. 25m - file will be broken into 25 MB '
'shards and uploaded shard by shard')
upload_parser.add_argument('file', help="Path to file to upload")
download_parser = subparser.add_parser('download',
help="Download a file from API")
download_parser.add_argument(
'--uri',
required=True,
nargs='+',
help='URI, or URIs, of file to download. Accepts multiple values, '
'space separated. If multiple URIs are specified, the URIs are '
'joined to create a single file'
)
download_parser.add_argument('--dest',
help="Folder or file to download file")
download_parser.add_argument('--shard-size', type=int, default=1024)
return parser.parse_args()
def test_size_helpers_bytes_to_mib(self):
self.assertEqual(SizeHelpers.bytes_to_mib(1048576), 1)
def test_size_helpers_bytes_to_kib(self):
self.assertEqual(SizeHelpers.bytes_to_kib(1024), 1)
def test_size_helpers_bytes_to_kib(self):
self.assertEqual(SizeHelpers.bytes_to_kib(1024), 1)
def test_tell(self):
number = random.randint(1, SizeHelpers.kib_to_bytes(768))
self.shard.seek(number)
self.assertEqual(self.shard.tell(), number)
def test_read_with_size(self):
fivetwelvekay = SizeHelpers.kib_to_bytes(512)
read = self.shard.read(fivetwelvekay)
self.assertEqual(fivetwelvekay, len(read))
def test_seek(self):
number = random.randint(1, SizeHelpers.kib_to_bytes(768))
self.shard.seek(number)
self.assertEqual(self.shard._f_obj.tell(), number)
def test_read_no_size(self):
read = self.shard.read()
self.assertEqual(len(read), SizeHelpers.mib_to_bytes(1))
def test_read_with_size(self):
fivetwelvekay = SizeHelpers.kib_to_bytes(512)
read = self.shard.read(fivetwelvekay)
self.assertEqual(fivetwelvekay, len(read))
def test__len__(self):
fivetwelvekay = SizeHelpers.kib_to_bytes(512)
self.shard.read(fivetwelvekay)
self.assertTrue(self.shard.__len__() < self.shard.max_seek)
def test_calc_max_seek_beginning_small_shard(self):
self.shard.shard_size = SizeHelpers.kib_to_bytes(512)
self.shard._calc_max_seek()
self.assertEqual(self.shard.max_seek, self.shard.shard_size)
def test_size_helpers_bytes_to_mib(self):
self.assertEqual(SizeHelpers.bytes_to_mib(1048576), 1)
def test_calc_max_seek_near_end(self):
self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
self.shard._calc_max_seek()
self.assertEqual(self.shard.max_seek, self.shard.filesize)
def test__len__(self):
fivetwelvekay = SizeHelpers.kib_to_bytes(512)
self.shard.read(fivetwelvekay)
self.assertTrue(self.shard.__len__() < self.shard.max_seek)
def test_generate_slices_near_end(self):
gen = self.shard._generate_slices()
self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
for slice in gen:
self.assertEqual(len(slice), 10)
def test_read_no_size(self):
read = self.shard.read()
self.assertEqual(len(read), SizeHelpers.mib_to_bytes(1))
def test_calc_max_seek_near_end(self):
self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
self.shard._calc_max_seek()
self.assertEqual(self.shard.max_seek, self.shard.filesize)
def test_generate_slices_near_end(self):
gen = self.shard._generate_slices()
self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
for slice in gen:
self.assertEqual(len(slice), 10)
def test_calc_max_seek_beginning_small_shard(self):
self.shard.shard_size = SizeHelpers.kib_to_bytes(512)
self.shard._calc_max_seek()
self.assertEqual(self.shard.max_seek, self.shard.shard_size)
def test_generate_slices_at_max_seek(self):
gen = self.shard._generate_slices()
self.shard.seek(SizeHelpers.mib_to_bytes(1))
for slice in gen:
self.assertIs(slice, None)
def test_calc_total_read(self):
number = random.randint(1, SizeHelpers.mib_to_bytes(1))
self.shard.seek(number)
self.shard._calc_total_read()
self.assertEqual(self.shard.total_read_bytes,
self.shard.max_seek - number)
def test_generate_slices_at_max_seek(self):
gen = self.shard._generate_slices()
self.shard.seek(SizeHelpers.mib_to_bytes(1))
for slice in gen:
self.assertIs(slice, None)
def test_calc_total_read(self):
number = random.randint(1, SizeHelpers.mib_to_bytes(1))
self.shard.seek(number)
self.shard._calc_total_read()
self.assertEqual(
self.shard.total_read_bytes, self.shard.max_seek - number)
