def test_upload_items(self): server_ref = isolate_storage.ServerRef('http://localhost:1', 'default') items = [ isolateserver.BufferItem('a'*12, server_ref.hash_algo), isolateserver.BufferItem('', server_ref.hash_algo), isolateserver.BufferItem('c'*1222, server_ref.hash_algo), isolateserver.BufferItem('d'*1223, server_ref.hash_algo), ] missing = { items[2]: 123, items[3]: 456, } storage_api = MockedStorageApi( server_ref, {item.digest: push_state for item, push_state in missing.items()}) storage = isolateserver.Storage(storage_api) # Intentionally pass a generator, to confirm it works. result = storage.upload_items((i for i in items)) self.assertEqual(sorted(missing), sorted(result)) self.assertEqual(4, len(items)) self.assertEqual(2, len(missing)) self.assertEqual([items], storage_api.contains_calls) self.assertEqual( sorted( ((items[2], 123, items[2].content()[0]), (items[3], 456, items[3].content()[0]))), sorted(storage_api.push_calls))
def zip_and_upload(self): """Zips up all the files necessary to run a shard and uploads to Swarming master. """ assert not self._isolate_item start_time = time.time() self._isolate_item = isolateserver.BufferItem( self.bundle.zip_into_buffer(), self._algo, is_isolated=True) print 'Zipping completed, time elapsed: %f' % (time.time() - start_time) try: start_time = time.time() uploaded = self.storage.upload_items([self._isolate_item]) elapsed = time.time() - start_time except (IOError, OSError) as exc: print >> sys.stderr, 'Failed to upload the zip file: %s' % exc return False if self._isolate_item in uploaded: print 'Upload complete, time elapsed: %f' % elapsed else: print 'Zip file already on server, time elapsed: %f' % elapsed return True
def run_push_and_fetch_test(self, namespace): storage = isolateserver.get_storage(self.server.url, namespace) # Upload items. items = [isolateserver.BufferItem('item %d' % i) for i in xrange(10)] uploaded = storage.upload_items(items) self.assertEqual(set(items), set(uploaded)) # Fetch them all back into local memory cache. cache = isolateserver.MemoryCache() queue = isolateserver.FetchQueue(storage, cache) # Start fetching. pending = set() for item in items: pending.add(item.digest) queue.add(item.digest) # Wait for fetch to complete. while pending: fetched = queue.wait(pending) pending.discard(fetched) # Ensure fetched same data as was pushed. self.assertEqual([i.buffer for i in items], [cache.read(i.digest) for i in items])
def upload_zip_bundle(isolate_server, bundle): """Uploads a zip package to isolate storage and returns raw fetch URL. Args: isolate_server: URL of an isolate server. bundle: instance of ZipPackage to upload. Returns: URL to get the file from on success. None on failure. """ # Swarming bot would need to be able to grab the file from the storage # using raw HTTP GET. Use 'default' namespace so that the raw data returned # to a bot is not zipped, since swarm_bot doesn't understand compressed # data yet. This namespace have nothing to do with |namespace| passed to # run_isolated.py that is used to store files for isolated task. logging.info('Zipping up and uploading files...') try: start_time = now() isolate_item = isolateserver.BufferItem(bundle.zip_into_buffer(), high_priority=True) with isolateserver.get_storage(isolate_server, 'default') as storage: uploaded = storage.upload_items([isolate_item]) bundle_url = storage.get_fetch_url(isolate_item) elapsed = now() - start_time except (IOError, OSError) as exc: tools.report_error('Failed to upload the zip file: %s' % exc) return None if isolate_item in uploaded: logging.info('Upload complete, time elapsed: %f', elapsed) else: logging.info('Zip file already on server, time elapsed: %f', elapsed) return bundle_url
def run_push_and_fetch_test(self, namespace): storage = isolateserver.get_storage( isolate_storage.ServerRef(self.server.url, namespace)) # Upload items. items = [ isolateserver.BufferItem('item %d' % i, storage.server_ref.hash_algo) for i in range(10) ] uploaded = storage.upload_items(items) self.assertEqual(set(items), set(uploaded)) # Fetch them all back into local memory cache. cache = local_caching.MemoryContentAddressedCache() queue = isolateserver.FetchQueue(storage, cache) # Start fetching. pending = set() for item in items: pending.add(item.digest) queue.add(item.digest) queue.wait_on(item.digest) # Wait for fetch to complete. while pending: fetched = queue.wait() pending.discard(fetched) # Ensure fetched same data as was pushed. actual = [] for i in items: with cache.getfileobj(i.digest) as f: actual.append(f.read()) self.assertEqual([''.join(i.content()) for i in items], actual)
def isolate_item(self): """Calling this property 'closes' the manifest and it can't be modified afterward. """ if self._isolate_item is None: self._isolate_item = isolateserver.BufferItem( self.bundle.zip_into_buffer(), high_priority=True) return self._isolate_item
def run_upload_items_test(self, namespace): storage = isolateserver.get_storage(self.server.url, namespace) # Items to upload. items = [isolateserver.BufferItem('item %d' % i) for i in xrange(10)] # Do it. uploaded = storage.upload_items(items) self.assertEqual(set(items), set(uploaded)) # All items are there now. self.assertFalse(dict(storage.get_missing_items(items))) # Now ensure upload_items skips existing items. more = [isolateserver.BufferItem('more item %d' % i) for i in xrange(10)] # Uploaded only |more|. uploaded = storage.upload_items(items + more) self.assertEqual(set(more), set(uploaded))
def run_upload_items_test(self, namespace): storage = isolateserver.get_storage( isolate_storage.ServerRef(self.server.url, namespace)) # Items to upload. items = [ isolateserver.BufferItem('item %d' % i, storage.server_ref.hash_algo) for i in xrange(10) ] # Do it. uploaded = storage.upload_items(items) self.assertEqual(set(items), set(uploaded)) # Now ensure upload_items skips existing items. more = [ isolateserver.BufferItem('more item %d' % i, storage.server_ref.hash_algo) for i in xrange(10) ] # Uploaded only |more|. uploaded = storage.upload_items(items + more) self.assertEqual(set(more), set(uploaded))
def run_synchronous_push_test(self, namespace): storage = isolateserver.get_storage(self.server.url, namespace) # Items to upload. items = [isolateserver.BufferItem('item %d' % i) for i in xrange(10)] # Storage is empty, all items are missing. missing = dict(storage.get_missing_items(items)) self.assertEqual(set(items), set(missing)) # Push, one by one. for item, push_state in missing.iteritems(): storage.push(item, push_state) # All items are there now. self.assertFalse(dict(storage.get_missing_items(items)))
def send_and_receive(random_pool, storage, progress, size): """Sends a random file and gets it back. # TODO(maruel): Add a batching argument of value [1, 500] to batch requests. Returns (delay, size) """ # Create a file out of the pool. start = time.time() batch = 1 items = [ isolateserver.BufferItem(random_pool.gen(size), False) for _ in xrange(batch) ] try: # len(_uploaded) may be < len(items) happen if the items is not random # enough or value of --mid-size is very low compared to --items. _uploaded = storage.upload_items(items) start = time.time() cache = isolateserver.MemoryCache() queue = isolateserver.FetchQueue(storage, cache) for i in items: queue.add(i.digest, i.size) waiting = [i.digest for i in items] while waiting: waiting.remove(queue.wait(waiting)) expected = {i.digest: ''.join(i.content()) for i in items} for d in cache.cached_set(): actual = cache.read(d) assert expected.pop(d) == actual assert not expected, expected duration = max(0, time.time() - start) except isolateserver.MappingError as e: duration = str(e) if isinstance(duration, float): progress.update_item('', index=1, data=size) else: progress.update_item('', index=1) return (duration, size)