Esempio n. 1
0
 def test_instantiation(self):
     self.assertTrue(self.shard._f_obj)
     self.assertTrue(SizeHelpers.mib_to_bytes(25), self.shard.shard_size)
     self.assertEqual(self.shard._f_obj.tell(), 0)
     self.assertEqual(self.shard.filesize, SizeHelpers.mib_to_bytes(1))
     self.assertEqual(self.shard.max_seek, SizeHelpers.mib_to_bytes(1))
     self.assertFalse(hasattr(self.shard, 'callback'))
Esempio n. 2
0
 def test_instantiation(self):
     self.assertTrue(self.shard._f_obj)
     self.assertTrue(SizeHelpers.mib_to_bytes(25), self.shard.shard_size)
     self.assertEqual(self.shard._f_obj.tell(), 0)
     self.assertEqual(self.shard.filesize, SizeHelpers.mib_to_bytes(1))
     self.assertEqual(self.shard.max_seek, SizeHelpers.mib_to_bytes(1))
     self.assertFalse(hasattr(self.shard, 'callback'))
Esempio n. 3
0
    def _upload_form_encoded(self, url, filepath, shard_size, start_pos,
                             read_size=1024, callback=None):
        """ Streams file from disk and uploads it.

        :param url: API endpoint as URL to upload to
        :param filepath: Path to file as string
        :return: requests.Response
        """
        validpath = self.check_path(filepath)
        if shard_size == 0:
            shard_size = SizeHelpers.mib_to_bytes(250)
        shard = ShardFile(
            validpath, 'rb',
            shard_size=shard_size,
            start_pos=start_pos,
            read_size=read_size,
            callback=callback
        )
        m = MultipartEncoder({
            'file': ('file', shard)
        })
        headers = {
            'Content-Type': m.content_type
        }
        return requests.post(url, data=m, headers=headers)
Esempio n. 4
0
    def _upload_form_encoded(self,
                             url,
                             filepath,
                             shard_size,
                             start_pos,
                             read_size=1024,
                             callback=None):
        """ Streams file from disk and uploads it.

        :param url: API endpoint as URL to upload to
        :param filepath: Path to file as string
        :return: requests.Response
        """
        validpath = self.check_path(filepath)
        if shard_size == 0:
            shard_size = SizeHelpers.mib_to_bytes(250)
        shard = ShardFile(validpath,
                          'rb',
                          shard_size=shard_size,
                          start_pos=start_pos,
                          read_size=read_size,
                          callback=callback)
        m = MultipartEncoder({'file': ('file', shard)})
        headers = {'Content-Type': m.content_type}
        return requests.post(url, data=m, headers=headers)
Esempio n. 5
0
 def test_parse_shard_size(self):
     result = clitool.parse_shard_size('1024')
     self.assertEqual(result, 1024)
     result = clitool.parse_shard_size('1024b')
     self.assertEqual(result, 1024)
     result = clitool.parse_shard_size('1k')
     self.assertEqual(result, 1024)
     result = clitool.parse_shard_size('1m')
     self.assertEqual(result, SizeHelpers.mib_to_bytes(1))
     result = clitool.parse_shard_size('1g')
     self.assertIs(result, None)
Esempio n. 6
0
 def test_parse_shard_size(self):
     result = clitool.parse_shard_size('1024')
     self.assertEqual(result, 1024)
     result = clitool.parse_shard_size('1024b')
     self.assertEqual(result, 1024)
     result = clitool.parse_shard_size('1k')
     self.assertEqual(result, 1024)
     result = clitool.parse_shard_size('1m')
     self.assertEqual(result, SizeHelpers.mib_to_bytes(1))
     result = clitool.parse_shard_size('1g')
     self.assertIs(result, None)
Esempio n. 7
0
def parse_shard_size(size):
    """ Parse a string, expected to be in the format of, for example:
    250m, 512k, 1024b.  A string without a trailing letter is assumed to be
    bytes. Accepts the following: m as mebibytes, k as kibibytes, b as bytes,


    :param size:
    :return:
    """
    size = str(size)
    if size.isdigit():
        return int(size)
    last = size[-1].lower()
    number = size[:-1]
    choices = ['b', 'k', 'm']
    if last not in choices:
        return
    if last == 'b':
        return int(number)
    elif last == 'k':
        return SizeHelpers.kib_to_bytes(int(number))
    elif last == 'm':
        return SizeHelpers.mib_to_bytes(int(number))
Esempio n. 8
0
def parse_shard_size(size):
    """ Parse a string, expected to be in the format of, for example:
    250m, 512k, 1024b.  A string without a trailing letter is assumed to be
    bytes. Accepts the following: m as mebibytes, k as kibibytes, b as bytes,


    :param size:
    :return:
    """
    size = str(size)
    if size.isdigit():
        return int(size)
    last = size[-1].lower()
    number = size[:-1]
    choices = ['b', 'k', 'm']
    if last not in choices:
        return
    if last == 'b':
        return int(number)
    elif last == 'k':
        return SizeHelpers.kib_to_bytes(int(number))
    elif last == 'm':
        return SizeHelpers.mib_to_bytes(int(number))
Esempio n. 9
0
 def setUp(self):
     self.stream = Streamer("http://node1.metadisk.org")
     self.orig_hash = None
     self.uploadfile = "tests/1k.testfile"
     self.downloadfile = "download.testfile"
     self.shard = Shard(
         "2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
         "1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
     )
     self.args = mock.MagicMock()
     self.args.verbose = False
     self.args.server = 'http://node1.metadisk.org'
     self.args.uri = [self.shard.uri]
     self.args.file = self.uploadfile
     self.args.dest = self.downloadfile
     self.args.shard_size = SizeHelpers.mib_to_bytes(250)
Esempio n. 10
0
 def setUp(self):
     self.stream = Streamer("http://node1.metadisk.org")
     self.orig_hash = None
     self.uploadfile = "tests/1k.testfile"
     self.downloadfile = "download.testfile"
     self.shard = Shard(
         "2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
         "1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
     )
     self.args = mock.MagicMock()
     self.args.verbose = False
     self.args.server = 'http://node1.metadisk.org'
     self.args.uri = [self.shard.uri]
     self.args.file = self.uploadfile
     self.args.dest = self.downloadfile
     self.args.shard_size = SizeHelpers.mib_to_bytes(250)
Esempio n. 11
0
def parse_args():
    """ Parses args

    :return: argparse namespace
    """
    parser = argparse.ArgumentParser("Upstream",
                                     description="Command line client for "
                                     "the Storj web-core API")
    parser.add_argument('--server',
                        default='http://node1.metadisk.org',
                        help='Metadisk node to connect to')
    parser.add_argument('-v',
                        dest='verbose',
                        action='store_true',
                        help='Verbose output')
    parser.add_argument('--version',
                        action='version',
                        version="%(prog)s " + upstream.__version__,
                        help='Display version.')
    subparser = parser.add_subparsers(dest='action')

    upload_parser = subparser.add_parser('upload',
                                         help="Upload a file from API")
    upload_parser.add_argument('--shard-size',
                               default=SizeHelpers.mib_to_bytes(250),
                               help='Size of shards to break file into and '
                               'to upload, max: 250m, default: 250m. '
                               'Ex. 25m - file will be broken into 25 MB '
                               'shards and uploaded shard by shard')
    upload_parser.add_argument('file', help="Path to file to upload")

    download_parser = subparser.add_parser('download',
                                           help="Download a file from API")
    download_parser.add_argument(
        '--uri',
        required=True,
        nargs='+',
        help='URI, or URIs, of file to download. Accepts multiple values, '
        'space separated. If multiple URIs are specified, the URIs are '
        'joined to create a single file')
    download_parser.add_argument('--dest',
                                 help="Folder or file to download file")
    download_parser.add_argument('--shard-size', type=int, default=1024)

    return parser.parse_args()
Esempio n. 12
0
def parse_args():
    """ Parses args

    :return: argparse namespace
    """
    parser = argparse.ArgumentParser("Upstream",
                                     description="Command line client for "
                                                 "the Storj web-core API")
    parser.add_argument('--server', default='http://node1.metadisk.org',
                        help='Metadisk node to connect to')
    parser.add_argument('-v', dest='verbose',
                        action='store_true', help='Verbose output')
    parser.add_argument('--version', action='version',
                        version="%(prog)s " + upstream.__version__,
                        help='Display version.')
    subparser = parser.add_subparsers(dest='action')

    upload_parser = subparser.add_parser('upload',
                                         help="Upload a file from API")
    upload_parser.add_argument('--shard-size',
                               default=SizeHelpers.mib_to_bytes(250),
                               help='Size of shards to break file into and '
                                    'to upload, max: 250m, default: 250m. '
                                    'Ex. 25m - file will be broken into 25 MB '
                                    'shards and uploaded shard by shard')
    upload_parser.add_argument('file', help="Path to file to upload")

    download_parser = subparser.add_parser('download',
                                           help="Download a file from API")
    download_parser.add_argument(
        '--uri',
        required=True,
        nargs='+',
        help='URI, or URIs, of file to download. Accepts multiple values, '
             'space separated. If multiple URIs are specified, the URIs are '
             'joined to create a single file'
    )
    download_parser.add_argument('--dest',
                                 help="Folder or file to download file")
    download_parser.add_argument('--shard-size', type=int, default=1024)

    return parser.parse_args()
Esempio n. 13
0
 def test_size_helpers_bytes_to_mib(self):
     self.assertEqual(SizeHelpers.bytes_to_mib(1048576), 1)
Esempio n. 14
0
 def test_size_helpers_bytes_to_kib(self):
     self.assertEqual(SizeHelpers.bytes_to_kib(1024), 1)
Esempio n. 15
0
 def test_size_helpers_bytes_to_kib(self):
     self.assertEqual(SizeHelpers.bytes_to_kib(1024), 1)
Esempio n. 16
0
 def test_tell(self):
     number = random.randint(1, SizeHelpers.kib_to_bytes(768))
     self.shard.seek(number)
     self.assertEqual(self.shard.tell(), number)
Esempio n. 17
0
 def test_read_with_size(self):
     fivetwelvekay = SizeHelpers.kib_to_bytes(512)
     read = self.shard.read(fivetwelvekay)
     self.assertEqual(fivetwelvekay, len(read))
Esempio n. 18
0
 def test_seek(self):
     number = random.randint(1, SizeHelpers.kib_to_bytes(768))
     self.shard.seek(number)
     self.assertEqual(self.shard._f_obj.tell(), number)
Esempio n. 19
0
 def test_read_no_size(self):
     read = self.shard.read()
     self.assertEqual(len(read), SizeHelpers.mib_to_bytes(1))
Esempio n. 20
0
 def test_read_with_size(self):
     fivetwelvekay = SizeHelpers.kib_to_bytes(512)
     read = self.shard.read(fivetwelvekay)
     self.assertEqual(fivetwelvekay, len(read))
Esempio n. 21
0
 def test__len__(self):
     fivetwelvekay = SizeHelpers.kib_to_bytes(512)
     self.shard.read(fivetwelvekay)
     self.assertTrue(self.shard.__len__() < self.shard.max_seek)
Esempio n. 22
0
 def test_calc_max_seek_beginning_small_shard(self):
     self.shard.shard_size = SizeHelpers.kib_to_bytes(512)
     self.shard._calc_max_seek()
     self.assertEqual(self.shard.max_seek, self.shard.shard_size)
Esempio n. 23
0
 def test_size_helpers_bytes_to_mib(self):
     self.assertEqual(SizeHelpers.bytes_to_mib(1048576), 1)
Esempio n. 24
0
 def test_calc_max_seek_near_end(self):
     self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
     self.shard._calc_max_seek()
     self.assertEqual(self.shard.max_seek, self.shard.filesize)
Esempio n. 25
0
 def test__len__(self):
     fivetwelvekay = SizeHelpers.kib_to_bytes(512)
     self.shard.read(fivetwelvekay)
     self.assertTrue(self.shard.__len__() < self.shard.max_seek)
Esempio n. 26
0
 def test_generate_slices_near_end(self):
     gen = self.shard._generate_slices()
     self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
     for slice in gen:
         self.assertEqual(len(slice), 10)
Esempio n. 27
0
 def test_read_no_size(self):
     read = self.shard.read()
     self.assertEqual(len(read), SizeHelpers.mib_to_bytes(1))
Esempio n. 28
0
 def test_calc_max_seek_near_end(self):
     self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
     self.shard._calc_max_seek()
     self.assertEqual(self.shard.max_seek, self.shard.filesize)
Esempio n. 29
0
 def test_generate_slices_near_end(self):
     gen = self.shard._generate_slices()
     self.shard.seek(SizeHelpers.mib_to_bytes(1) - 10)
     for slice in gen:
         self.assertEqual(len(slice), 10)
Esempio n. 30
0
 def test_calc_max_seek_beginning_small_shard(self):
     self.shard.shard_size = SizeHelpers.kib_to_bytes(512)
     self.shard._calc_max_seek()
     self.assertEqual(self.shard.max_seek, self.shard.shard_size)
Esempio n. 31
0
 def test_generate_slices_at_max_seek(self):
     gen = self.shard._generate_slices()
     self.shard.seek(SizeHelpers.mib_to_bytes(1))
     for slice in gen:
         self.assertIs(slice, None)
Esempio n. 32
0
 def test_calc_total_read(self):
     number = random.randint(1, SizeHelpers.mib_to_bytes(1))
     self.shard.seek(number)
     self.shard._calc_total_read()
     self.assertEqual(self.shard.total_read_bytes,
                      self.shard.max_seek - number)
Esempio n. 33
0
 def test_generate_slices_at_max_seek(self):
     gen = self.shard._generate_slices()
     self.shard.seek(SizeHelpers.mib_to_bytes(1))
     for slice in gen:
         self.assertIs(slice, None)
Esempio n. 34
0
 def test_calc_total_read(self):
     number = random.randint(1, SizeHelpers.mib_to_bytes(1))
     self.shard.seek(number)
     self.shard._calc_total_read()
     self.assertEqual(
         self.shard.total_read_bytes, self.shard.max_seek - number)