Esempio n. 1
0
def upload(args):
    """ Controls actions for uploading

    :param args: Parsed args namespace
    """
    shard_size = parse_shard_size(args.shard_size)

    try:
        filepath = Streamer.check_path(args.file)
    except FileError as e:
        sys.stderr.write('%s\n' % str(e))
        sys.exit(1)

    streamer = Streamer(args.server)
    shards = calculate_shards(args, shard_size, filepath)
    shard_info = []
    for idx, shard in enumerate(shards):
        i = idx + 1
        start = shard[0]
        callback = ProgressCallback()
        shard = streamer.upload(args.file,
                                start_pos=start,
                                shard_size=shard_size,
                                callback=callback.callback)
        callback.bar.finish()
        sys.stdout.flush()
        if args.verbose:
            print("\nShard %d - URI: %s\n" % (i, shard.uri))
        shard_info.append(shard.uri)

    print()
    print("Download this file by using the following command: ")
    print("upstream download --uri", " ".join(shard_info), "--dest <filename>")
Esempio n. 2
0
def upload(args):
    """ Controls actions for uploading

    :param args: Parsed args namespace
    """
    shard_size = parse_shard_size(args.shard_size)

    try:
        filepath = Streamer.check_path(args.file)
    except FileError as e:
        sys.stderr.write('%s\n' % str(e))
        sys.exit(1)

    streamer = Streamer(args.server)
    shards = calculate_shards(args, shard_size, filepath)
    shard_info = []
    for idx, shard in enumerate(shards):
        i = idx + 1
        start = shard[0]
        callback = ProgressCallback()
        shard = streamer.upload(
            args.file,
            start_pos=start,
            shard_size=shard_size,
            callback=callback.callback
        )
        callback.bar.finish()
        sys.stdout.flush()
        if args.verbose:
            print("\nShard %d - URI: %s\n" % (i, shard.uri))
        shard_info.append(shard.uri)

    print()
    print("Download this file by using the following command: ")
    print("upstream download --uri", " ".join(shard_info), "--dest <filename>")
Esempio n. 3
0
 def setUp(self):
     self.stream = Streamer("http://node1.metadisk.org")
     self.orig_hash = None
     self.uploadfile = "tests/1k.testfile"
     self.downloadfile = "download.testfile"
     self.shard = Shard(
         "2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
         "1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14")
Esempio n. 4
0
 def setUp(self):
     self.stream = Streamer("http://node1.metadisk.org")
     self.orig_hash = None
     self.uploadfile = "tests/1k.testfile"
     self.downloadfile = "download.testfile"
     self.shard = Shard(
         "2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
         "1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
     )
Esempio n. 5
0
def download(args):
    """ Controls actions for downloading

    :param args: Argparse namespace
    """
    shards = []
    for uri in args.uri:
        if args.verbose:
            print("Creating shard.")
        shard = Shard()
        shard.from_uri(uri)
        shards.append(shard)

    if args.verbose:
            print("There are %d shards to download." % len(shards))

    streamer = Streamer(args.server)
    if args.verbose:
        print("Connecting to %s..." % streamer.server)

    path, fname = check_and_get_dest(args.dest)
    savepath = os.path.join(path, fname)
    for i, shard in enumerate(shards):
        if args.verbose:
            print("Downloading file %s..." % shard.uri)
        else:
            print("Downloading file %d..." % (i + 1))
        sys.stdout.flush()
        try:
            r = streamer.download(shard, slicesize=8096)
        except ResponseError as e:
            sys.stderr.write("\nError!\n")
            sys.stderr.write("%s  %s\n" % (e.response.status_code,
                                           e.response.reason))
            sys.stderr.write("%s\n" % e.response.text)
            raise
        with open(savepath, 'ab') as f:
            if args.verbose:
                print("Writing shard.")
            for _bytes in r.iter_content():
                    f.write(_bytes)

    print("\nDownloaded to %s." % savepath)
    return fname
Esempio n. 6
0
def download(args):
    """ Controls actions for downloading

    :param args: Argparse namespace
    """
    shards = []
    for uri in args.uri:
        if args.verbose:
            print("Creating shard.")
        shard = Shard()
        shard.from_uri(uri)
        shards.append(shard)

    if args.verbose:
        print("There are %d shards to download." % len(shards))

    streamer = Streamer(args.server)
    if args.verbose:
        print("Connecting to %s..." % streamer.server)

    path, fname = check_and_get_dest(args.dest)
    savepath = os.path.join(path, fname)
    for i, shard in enumerate(shards):
        if args.verbose:
            print("Downloading file %s..." % shard.uri)
        else:
            print("Downloading file %d..." % (i + 1))
        sys.stdout.flush()

        r = streamer.download(shard, slicesize=8096)

        with open(savepath, 'ab') as f:
            if args.verbose:
                print("Writing shard.")
            for _bytes in r.iter_content():
                f.write(_bytes)

    print("\nDownloaded to %s." % savepath)
    return fname
Esempio n. 7
0
 def setUp(self):
     self.stream = Streamer("http://node1.metadisk.org")
     self.orig_hash = None
     self.uploadfile = "tests/1k.testfile"
     self.downloadfile = "download.testfile"
     self.shard = Shard(
         "2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
         "1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
     )
     self.args = mock.MagicMock()
     self.args.verbose = False
     self.args.server = 'http://node1.metadisk.org'
     self.args.uri = [self.shard.uri]
     self.args.file = self.uploadfile
     self.args.dest = self.downloadfile
     self.args.shard_size = SizeHelpers.mib_to_bytes(250)
Esempio n. 8
0
class TestStreamer(unittest.TestCase):
    def setUp(self):
        self.stream = Streamer("http://node1.metadisk.org")
        self.orig_hash = None
        self.uploadfile = "tests/1k.testfile"
        self.downloadfile = "download.testfile"
        self.shard = Shard(
            "2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
            "1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14"
        )

    def tearDown(self):
        del self.stream
        del self.orig_hash
        del self.uploadfile
        try:
            os.remove(self.downloadfile)
        except:
            pass
        try:
            os.remove(self.shard.filehash)
        except:
            pass
        del self.downloadfile
        del self.shard

    def test_initialization(self):
        self.assertEqual(self.stream.server, "http://node1.metadisk.org")

    def test_check_connectivity(self):
        def _failing_connection():
            Streamer("http://does.not.exist")

        self.assertRaises(ConnectError, _failing_connection)

    @mock.patch('requests.post')
    def test_upload_form_encoded(self, post):
        pass

    @mock.patch('requests.post')
    def test_upload_sharded_encoded(self, post):
        pass

    def test_upload(self):
        # Upload file and check file
        self.shard = self.stream.upload(self.uploadfile)
        self.assertEqual(
            self.shard.filehash,
            "2032e4fd19d4ab49a74ead0984a5f672"
            "c26e60da6e992eaf51f05dc874e94bd7")
        self.assertEqual(
            self.shard.decryptkey,
            "1b1f463cef1807a127af668f3a4fdcc7"
            "977c647bf2f357d9fa125f13548b1d14")

        # Try to upload wrong file
        def _failing_upload():
            self.stream.upload("not-a-real-file")
        self.assertRaises(FileError, _failing_upload)

    def test_upload_patched_404(self):
        self.stream._upload_form_encoded = mock.MagicMock()
        self.stream._upload_form_encoded.return_value()
        self.stream._upload_form_encoded.return_value.status_code = 404

        def _fourohfour():
            self.stream.upload(self.uploadfile)
        with self.assertRaises(ResponseError) as ex:
            _fourohfour()
            self.assertEqual(ex.message, "API call not found.")

    def test_upload_patched_402(self):
        self.stream._upload_form_encoded = mock.MagicMock()
        self.stream._upload_form_encoded.return_value()
        self.stream._upload_form_encoded.return_value.status_code = 402

        def _fourohtwo():
            self.stream.upload(self.uploadfile)
        with self.assertRaises(ResponseError) as ex:
            _fourohtwo()

    def test_upload_patched_500(self):
        self.stream._upload_form_encoded = mock.MagicMock()
        self.stream._upload_form_encoded.return_value()
        self.stream._upload_form_encoded.return_value.status_code = 500

        def _fivehundred():
            self.stream.upload(self.uploadfile)
        with self.assertRaises(ResponseError) as ex:
            _fivehundred()
            self.assertEqual(ex.message, "Server error.")

    def test_upload_patched_501(self):
        self.stream._upload_form_encoded = mock.MagicMock()
        self.stream._upload_form_encoded.return_value()
        self.stream._upload_form_encoded.return_value.status_code = 501
        self.stream._upload_form_encoded.return_value.reason = "Not Implemented"

        def _fiveohone():
            self.stream.upload(self.uploadfile)
        with self.assertRaises(ResponseError) as ex:
            _fiveohone()
            self.assertEqual(ex.message,
                             "Received status code 501 Not Implemented")

    def test_upload_check_path(self):
        homedir = os.path.expanduser(self.uploadfile)
        result = self.stream.check_path(self.uploadfile)
        self.assertEqual(homedir, result)

        with self.assertRaises(FileError) as ex:
            self.stream.check_path('~/does-not-exist')
            self.assertEqual(
                ex.message, '~/does-not-exist not a file or not found')

    def test_download(self):
        r = self.stream.download(self.shard)
        self.assertTrue(r)
        self.assertEquals(r.status_code, 200)
        self.assertEqual(len(r.content), 1024)

    def test_download_exception(self):
        self.shard.filehash = self.shard.filehash[:-5]
        with self.assertRaises(ResponseError) as ex:
            r = self.stream.download(self.shard)
        self.assertEqual(ex.exception.response.status_code, 404)

    def test_download_empty_shard(self):
        shard = Shard()
        with self.assertRaises(ShardError) as e:
            self.stream.download(shard)
        self.assertEqual(str(e.exception), "Shard missing filehash.")
Esempio n. 9
0
 def _failing_connection():
     Streamer("http://does.not.exist")
Esempio n. 10
0
class TestStreamer(unittest.TestCase):
    def setUp(self):
        self.stream = Streamer("http://node1.metadisk.org")
        self.orig_hash = None
        self.uploadfile = "tests/1k.testfile"
        self.downloadfile = "download.testfile"
        self.shard = Shard(
            "2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7",
            "1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14")

    def tearDown(self):
        del self.stream
        del self.orig_hash
        del self.uploadfile
        try:
            os.remove(self.downloadfile)
        except:
            pass
        try:
            os.remove(self.shard.filehash)
        except:
            pass
        del self.downloadfile
        del self.shard

    def test_initialization(self):
        self.assertEqual(self.stream.server, "http://node1.metadisk.org")

    def test_check_connectivity(self):
        def _failing_connection():
            Streamer("http://does.not.exist")

        self.assertRaises(ConnectError, _failing_connection)

    @mock.patch('requests.post')
    def test_upload_form_encoded(self, post):
        pass

    @mock.patch('requests.post')
    def test_upload_sharded_encoded(self, post):
        with self.assertRaises(NotImplementedError):
            self.stream._upload_sharded_encoded('http://fake.url', 'fake.path')

    @mock.patch('requests.post')
    def test_filestream(self, post):
        with self.assertRaises(NotImplementedError):
            self.stream._filestream('fake.path')

    def test_upload(self):
        # Upload file and check file
        self.shard = self.stream.upload(self.uploadfile)
        self.assertEqual(
            self.shard.filehash, "2032e4fd19d4ab49a74ead0984a5f672"
            "c26e60da6e992eaf51f05dc874e94bd7")
        self.assertEqual(
            self.shard.decryptkey, "1b1f463cef1807a127af668f3a4fdcc7"
            "977c647bf2f357d9fa125f13548b1d14")

        # Try to upload wrong file
        def _failing_upload():
            self.stream.upload("not-a-real-file")

        self.assertRaises(FileError, _failing_upload)

    def test_upload_patched_404(self):
        self.stream._upload_form_encoded = mock.MagicMock()
        self.stream._upload_form_encoded.return_value()
        self.stream._upload_form_encoded.return_value.status_code = 404

        def _fourohfour():
            self.stream.upload(self.uploadfile)

        with self.assertRaises(ResponseError) as ex:
            _fourohfour()
            self.assertEqual(ex.message, "API call not found.")

    def test_upload_patched_402(self):
        self.stream._upload_form_encoded = mock.MagicMock()
        self.stream._upload_form_encoded.return_value()
        self.stream._upload_form_encoded.return_value.status_code = 402

        def _fourohtwo():
            self.stream.upload(self.uploadfile)

        with self.assertRaises(ResponseError):
            _fourohtwo()

    def test_upload_patched_500(self):
        self.stream._upload_form_encoded = mock.MagicMock()
        self.stream._upload_form_encoded.return_value()
        self.stream._upload_form_encoded.return_value.status_code = 500

        def _fivehundred():
            self.stream.upload(self.uploadfile)

        with self.assertRaises(ResponseError) as ex:
            _fivehundred()
            self.assertEqual(ex.message, "Server error.")

    def test_upload_patched_501(self):
        self.stream._upload_form_encoded = mock.MagicMock()
        self.stream._upload_form_encoded.return_value()
        self.stream._upload_form_encoded.return_value.status_code = 501
        self.stream._upload_form_encoded.return_value.reason =\
            "Not Implemented"

        def _fiveohone():
            self.stream.upload(self.uploadfile)

        with self.assertRaises(ResponseError) as ex:
            _fiveohone()
            self.assertEqual(ex.message,
                             "Received status code 501 Not Implemented")

    def test_upload_check_path(self):
        homedir = os.path.expanduser(self.uploadfile)
        result = self.stream.check_path(self.uploadfile)
        self.assertEqual(homedir, result)

        with self.assertRaises(FileError) as ex:
            self.stream.check_path('~/does-not-exist')
            self.assertEqual(ex.message,
                             '~/does-not-exist not a file or not found')

    def test_download(self):
        r = self.stream.download(self.shard)
        self.assertTrue(r)
        self.assertEquals(r.status_code, 200)
        self.assertEqual(len(r.content), 1024)

    def test_download_exception(self):
        self.shard.filehash = self.shard.filehash[:-5]
        with self.assertRaises(ResponseError) as ex:
            self.stream.download(self.shard)
        self.assertEqual(ex.exception.response.status_code, 404)

    def test_download_empty_shard(self):
        shard = Shard()
        with self.assertRaises(ShardError) as e:
            self.stream.download(shard)
        self.assertEqual(str(e.exception), "Shard missing filehash.")