def upload(self, filepath, shard_size=0, start_pos=0, read_size=1024, callback=None): """ Uploads a shard via POST to the specified node to the web-core API. See API docs: https://github.com/Storj/web-core#api-documentation :param filepath: Path to file as a string :return: upstream.shard.Shard :raise LookupError: IF """ # Open the file and upload it via POST url = self.server + "/api/upload" # web-core API r = self._upload_form_encoded( url, filepath, shard_size=shard_size, start_pos=start_pos, read_size=read_size, callback=callback ) # Make sure that the API call is actually there if r.status_code == 404: raise ResponseError("API call not found.") elif r.status_code == 402: raise ResponseError("Payment required.") elif r.status_code == 500: raise ResponseError("Server error.") elif r.status_code == 201: shard = Shard() shard.from_json(r.text) return shard else: err = ResponseError("Received status code %s %s" % (r.status_code, r.reason)) err.response = r raise err
def setUp(self): self.cryptkey = ("2b77e64156f9f7eb16d74b98f70417e4" "d665d977d0ef00e793d41767acf13e8c") self.filehash = ("5547a152337de9ff6a97f6f099bb024e" "08af419cee613b18da76a33e581d49ac") self.raw_uri = ( "5547a152337de9ff6a97f6f099bb024e08af419cee613b18da76a33e581d49ac" "?key=2b77e64156f9f7eb16d74b98f70417e4d665d977d0ef00e793d41767acf" "13e8c") # Create empty object self.empty_shard = Shard() # Create half object self.half_shard = Shard(self.filehash) # Create full object self.full_shard = Shard(self.filehash, self.cryptkey) # Create new objects self.shard1 = Shard() self.shard2 = Shard() # Load content self.json_dict = { "key": "2b77e64156f9f7eb16d74b98f70417e4" "d665d977d0ef00e793d41767acf13e8c", "filehash": "5547a152337de9ff6a97f6f099bb024e" "08af419cee613b18da76a33e581d49ac" } self.shard1.from_json(json.dumps(self.json_dict)) self.shard2.from_uri(self.raw_uri)
def setUp(self): self.stream = Streamer("http://node1.metadisk.org") self.orig_hash = None self.uploadfile = "tests/1k.testfile" self.downloadfile = "download.testfile" self.shard = Shard( "2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7", "1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14")
def download(args): """ Controls actions for downloading :param args: Argparse namespace """ shards = [] for uri in args.uri: if args.verbose: print("Creating shard.") shard = Shard() shard.from_uri(uri) shards.append(shard) if args.verbose: print("There are %d shards to download." % len(shards)) streamer = Streamer(args.server) if args.verbose: print("Connecting to %s..." % streamer.server) path, fname = check_and_get_dest(args.dest) savepath = os.path.join(path, fname) for i, shard in enumerate(shards): if args.verbose: print("Downloading file %s..." % shard.uri) else: print("Downloading file %d..." % (i + 1)) sys.stdout.flush() try: r = streamer.download(shard, slicesize=8096) except ResponseError as e: sys.stderr.write("\nError!\n") sys.stderr.write("%s %s\n" % (e.response.status_code, e.response.reason)) sys.stderr.write("%s\n" % e.response.text) raise with open(savepath, 'ab') as f: if args.verbose: print("Writing shard.") for _bytes in r.iter_content(): f.write(_bytes) print("\nDownloaded to %s." % savepath) return fname
def download(args): """ Controls actions for downloading :param args: Argparse namespace """ shards = [] for uri in args.uri: if args.verbose: print("Creating shard.") shard = Shard() shard.from_uri(uri) shards.append(shard) if args.verbose: print("There are %d shards to download." % len(shards)) streamer = Streamer(args.server) if args.verbose: print("Connecting to %s..." % streamer.server) path, fname = check_and_get_dest(args.dest) savepath = os.path.join(path, fname) for i, shard in enumerate(shards): if args.verbose: print("Downloading file %s..." % shard.uri) else: print("Downloading file %d..." % (i + 1)) sys.stdout.flush() r = streamer.download(shard, slicesize=8096) with open(savepath, 'ab') as f: if args.verbose: print("Writing shard.") for _bytes in r.iter_content(): f.write(_bytes) print("\nDownloaded to %s." % savepath) return fname
def upload(self, filepath, shard_size=0, start_pos=0, read_size=1024, callback=None): """ Uploads a shard via POST to the specified node to the web-core API. See API docs: https://github.com/Storj/web-core#api-documentation :param filepath: Path to file as a string :return: upstream.shard.Shard :raise LookupError: IF """ # Open the file and upload it via POST url = self.server + "/api/upload" # web-core API r = self._upload_form_encoded(url, filepath, shard_size=shard_size, start_pos=start_pos, read_size=read_size, callback=callback) # Make sure that the API call is actually there if r.status_code == 404: raise ResponseError("API call not found.") elif r.status_code == 402: raise ResponseError("Payment required.") elif r.status_code == 500: raise ResponseError("Server error.") elif r.status_code == 201: shard = Shard() shard.from_json(r.text) return shard else: err = ResponseError("Received status code %s %s" % (r.status_code, r.reason)) err.response = r raise err
def setUp(self): self.stream = Streamer("http://node1.metadisk.org") self.orig_hash = None self.uploadfile = "tests/1k.testfile" self.downloadfile = "download.testfile" self.shard = Shard( "2032e4fd19d4ab49a74ead0984a5f672c26e60da6e992eaf51f05dc874e94bd7", "1b1f463cef1807a127af668f3a4fdcc7977c647bf2f357d9fa125f13548b1d14" ) self.args = mock.MagicMock() self.args.verbose = False self.args.server = 'http://node1.metadisk.org' self.args.uri = [self.shard.uri] self.args.file = self.uploadfile self.args.dest = self.downloadfile self.args.shard_size = SizeHelpers.mib_to_bytes(250)
class TestShard(unittest.TestCase): def setUp(self): self.cryptkey = ("2b77e64156f9f7eb16d74b98f70417e4" "d665d977d0ef00e793d41767acf13e8c") self.filehash = ("5547a152337de9ff6a97f6f099bb024e" "08af419cee613b18da76a33e581d49ac") self.raw_uri = ( "5547a152337de9ff6a97f6f099bb024e08af419cee613b18da76a33e581d49ac" "?key=2b77e64156f9f7eb16d74b98f70417e4d665d977d0ef00e793d41767acf" "13e8c") # Create empty object self.empty_shard = Shard() # Create half object self.half_shard = Shard(self.filehash) # Create full object self.full_shard = Shard(self.filehash, self.cryptkey) # Create new objects self.shard1 = Shard() self.shard2 = Shard() # Load content self.json_dict = { "key": "2b77e64156f9f7eb16d74b98f70417e4" "d665d977d0ef00e793d41767acf13e8c", "filehash": "5547a152337de9ff6a97f6f099bb024e" "08af419cee613b18da76a33e581d49ac" } self.shard1.from_json(json.dumps(self.json_dict)) self.shard2.from_uri(self.raw_uri) def tearDown(self): del self.empty_shard del self.half_shard del self.full_shard del self.shard1 del self.shard2 del self.json_dict def test_getters_empty_shard(self): def _callable(meth): meth = getattr(self.empty_shard, meth) meth() for method in ['uri', 'get_hashes', 'get_json']: self.assertRaises(ShardError, _callable, method) def test_getters_half_shard(self): def _callable(meth): meth = getattr(self.half_shard, meth) meth() for method in ['uri', 'get_hashes', 'get_json']: self.assertRaises(ShardError, _callable, method) def test_getters_full_shard(self): uri = self.full_shard.uri self.assertEqual(uri, self.raw_uri) hash, key = self.full_shard.get_hashes() self.assertEqual(hash, self.filehash) self.assertEqual(key, self.cryptkey) json_ = self.full_shard.get_json() self.assertEqual(json_, json.dumps(self.json_dict)) def test_shards(self): # Fit everything into one test case self.assertEqual(self.shard1.get_hashes(), self.shard2.get_hashes())
def test_download_empty_shard(self): shard = Shard() with self.assertRaises(ShardError) as e: self.stream.download(shard) self.assertEqual(str(e.exception), "Shard missing filehash.")
def test_shard_with_malformed_uri(self): shard = Shard() self.assertRaises(ShardError, shard.from_uri, 'funkymonkey')
class TestShard(unittest.TestCase): def setUp(self): self.cryptkey = ("2b77e64156f9f7eb16d74b98f70417e4" "d665d977d0ef00e793d41767acf13e8c") self.filehash = ("5547a152337de9ff6a97f6f099bb024e" "08af419cee613b18da76a33e581d49ac") self.raw_uri = ( "5547a152337de9ff6a97f6f099bb024e08af419cee613b18da76a33e581d49ac" "?key=2b77e64156f9f7eb16d74b98f70417e4d665d977d0ef00e793d41767acf" "13e8c") # Create empty object self.empty_shard = Shard() # Create half object self.half_shard = Shard(self.filehash) # Create full object self.full_shard = Shard(self.filehash, self.cryptkey) # Create new objects self.shard1 = Shard() self.shard2 = Shard() # Load content self.json_dict = { "key": "2b77e64156f9f7eb16d74b98f70417e4" "d665d977d0ef00e793d41767acf13e8c", "filehash": "5547a152337de9ff6a97f6f099bb024e" "08af419cee613b18da76a33e581d49ac" } self.shard1.from_json(json.dumps(self.json_dict)) self.shard2.from_uri(self.raw_uri) def tearDown(self): del self.empty_shard del self.half_shard del self.full_shard del self.shard1 del self.shard2 del self.json_dict def test_shard_with_malformed_uri(self): shard = Shard() self.assertRaises(ShardError, shard.from_uri, 'funkymonkey') def test_getters_empty_shard(self): def _callable(meth): meth = getattr(self.empty_shard, meth) meth() for method in ['uri', 'get_hashes', 'get_json']: self.assertRaises(ShardError, _callable, method) def test_getters_half_shard(self): def _callable(meth): meth = getattr(self.half_shard, meth) meth() for method in ['uri', 'get_hashes', 'get_json']: self.assertRaises(ShardError, _callable, method) def test_getters_full_shard(self): uri = self.full_shard.uri self.assertEqual(uri, self.raw_uri) hash, key = self.full_shard.get_hashes() self.assertEqual(hash, self.filehash) self.assertEqual(key, self.cryptkey) json_ = self.full_shard.get_json() self.assertEqual(json_, json.dumps(self.json_dict)) def test_shards(self): # Fit everything into one test case self.assertEqual(self.shard1.get_hashes(), self.shard2.get_hashes())