def upload_hash_content_to_blobstore(generate_upload_url, data, hash_key, content): """Uploads the given hash contents directly to the blobsotre via a generated url. Arguments: generate_upload_url: The url to get the new upload url from. data: extra POST data. hash_key: sha1 of the uncompressed version of content. content: The contents to upload. Must fit in memory for now. """ logging.debug("Generating url to directly upload file to blobstore") assert isinstance(hash_key, str), hash_key assert isinstance(content, str), (hash_key, content) # TODO(maruel): Support large files. This would require streaming support. content_type, body = encode_multipart_formdata(data, [("content", hash_key, content)]) for attempt in xrange(run_isolated.URL_OPEN_MAX_ATTEMPTS): # Retry HTTP 50x here. response = run_isolated.url_open(generate_upload_url, data=data) if not response: raise run_isolated.MappingError("Unable to connect to server %s" % generate_upload_url) upload_url = response.read() # Do not retry this request on HTTP 50x. Regenerate an upload url each time # since uploading "consumes" the upload url. result = run_isolated.url_open(upload_url, data=body, content_type=content_type, retry_50x=False) if result: return result.read() if attempt != run_isolated.URL_OPEN_MAX_ATTEMPTS - 1: run_isolated.HttpService.sleep_before_retry(attempt, None) raise run_isolated.MappingError("Unable to connect to server %s" % generate_upload_url)
def upload_hash_content_to_blobstore(generate_upload_url, data, hash_key, content): """Uploads the given hash contents directly to the blobsotre via a generated url. Arguments: generate_upload_url: The url to get the new upload url from. data: extra POST data. hash_key: sha1 of the uncompressed version of content. content: The contents to upload. Must fit in memory for now. """ logging.debug('Generating url to directly upload file to blobstore') assert isinstance(hash_key, str), hash_key assert isinstance(content, str), (hash_key, content) # TODO(maruel): Support large files. This would require streaming support. content_type, body = encode_multipart_formdata( data, [('content', hash_key, content)]) for _ in range(run_isolated.MAX_URL_OPEN_ATTEMPTS): # Retry HTTP 50x here. response = run_isolated.url_open(generate_upload_url, data=data) if not response: raise run_isolated.MappingError('Unable to connect to server %s' % generate_upload_url) upload_url = response.read() # Do not retry this request on HTTP 50x. Regenerate an upload url each time # since uploading "consumes" the upload url. result = run_isolated.url_open(upload_url, data=body, content_type=content_type, retry_50x=False) if result: return result.read() raise run_isolated.MappingError('Unable to connect to server %s' % generate_upload_url)
def ProcessManifest(file_sha1, test_name, shards, test_filter, options): """Process the manifest file and send off the swarm test request.""" manifest = Manifest(file_sha1, test_name, shards, test_filter, options) # Zip up relevent files print "Zipping up files..." if not manifest.zip(): return 1 # Send test requests off to swarm. print 'Sending test requests to swarm' test_url = options.swarm_url.rstrip('/') + '/test' manifest_text = manifest.to_json() result = run_isolated.url_open(test_url, data={'request': manifest_text}) if not result: print >> sys.stderr, 'Failed to send test for %s\n%s' % ( test_name, test_url) return 1 try: json.load(result) except (ValueError, TypeError) as e: print >> sys.stderr, 'Failed to send test for %s' % test_name print >> sys.stderr, 'Manifest: %s' % manifest_text print >> sys.stderr, str(e) return 1 return 0
def ProcessManifest(file_sha1, test_name, shards, test_filter, options): """Process the manifest file and send off the swarm test request.""" try: manifest = Manifest(file_sha1, test_name, shards, test_filter, options) except ValueError as e: print >> sys.stderr, 'Unable to process %s: %s' % (test_name, e) return 1 # Zip up relevent files print "Zipping up files..." if not manifest.zip(): return 1 # Send test requests off to swarm. print('Sending test requests to swarm.') print('Server: %s' % options.swarm_url) print('Job name: %s' % test_name) test_url = options.swarm_url.rstrip('/') + '/test' manifest_text = manifest.to_json() result = run_isolated.url_open(test_url, data={'request': manifest_text}) if not result: print >> sys.stderr, 'Failed to send test for %s\n%s' % (test_name, test_url) return 1 try: json.load(result) except (ValueError, TypeError) as e: print >> sys.stderr, 'Failed to send test for %s' % test_name print >> sys.stderr, 'Manifest: %s' % manifest_text print >> sys.stderr, str(e) return 1 return 0
def retrieve_results(base_url, test_key, timeout, should_stop): """Retrieves results for a single test_key.""" assert isinstance(timeout, float) params = [('r', test_key)] result_url = '%s/get_result?%s' % (base_url, urllib.urlencode(params)) start = now() while True: if timeout and (now() - start) >= timeout: logging.warning('retrieve_results(%s) timed out', base_url) return {} # Do retries ourselve. response = run_isolated.url_open(result_url, retry_404=False, retry_50x=False) if response is None: # Aggressively poll for results. Do not use retry_404 so # should_stop is polled more often. remaining = min(5, timeout - (now() - start)) if timeout else 5 if remaining > 0: run_isolated.HttpService.sleep_before_retry(1, remaining) else: try: data = json.load(response) or {} except (ValueError, TypeError): logging.warning( 'Received corrupted data for test_key %s. Retrying.', test_key) else: if data['output']: return data if should_stop.get(): return {}
def get_test_keys(swarm_base_url, test_name, timeout): """Returns the Swarm test key for each shards of test_name.""" assert isinstance(timeout, float) key_data = urllib.urlencode([('name', test_name)]) url = '%s/get_matching_test_cases?%s' % (swarm_base_url, key_data) for i in range(run_isolated.URL_OPEN_MAX_ATTEMPTS): response = run_isolated.url_open(url, retry_404=True, timeout=timeout) if response is None: raise Failure( 'Error: Unable to find any tests with the name, %s, on swarm server' % test_name) result = response.read() # TODO(maruel): Compare exact string. if 'No matching' in result: logging.warning('Unable to find any tests with the name, %s, on swarm ' 'server' % test_name) if i != run_isolated.URL_OPEN_MAX_ATTEMPTS: run_isolated.HttpService.sleep_before_retry(i, None) continue return json.loads(result) raise Failure( 'Error: Unable to find any tests with the name, %s, on swarm server' % test_name)
def url_open(url, **kwargs): result = run_isolated.url_open(url, **kwargs) if not result: # If we get no response from the server, assume it is down and raise an # exception. raise run_isolated.MappingError("Unable to connect to server %s" % url) return result
def get_test_keys(swarm_base_url, test_name, timeout): """Returns the Swarm test key for each shards of test_name.""" assert isinstance(timeout, float) key_data = urllib.urlencode([('name', test_name)]) url = '%s/get_matching_test_cases?%s' % (swarm_base_url, key_data) for i in range(run_isolated.URL_OPEN_MAX_ATTEMPTS): response = run_isolated.url_open(url, retry_404=True, timeout=timeout) if response is None: raise Failure( 'Error: Unable to find any tests with the name, %s, on swarm server' % test_name) result = response.read() # TODO(maruel): Compare exact string. if 'No matching' in result: logging.warning( 'Unable to find any tests with the name, %s, on swarm ' 'server' % test_name) if i != run_isolated.URL_OPEN_MAX_ATTEMPTS: run_isolated.HttpService.sleep_before_retry(i, None) continue return json.loads(result) raise Failure( 'Error: Unable to find any tests with the name, %s, on swarm server' % test_name)
def retrieve_results(base_url, test_key, timeout, should_stop): """Retrieves results for a single test_key.""" assert isinstance(timeout, float) params = [('r', test_key)] result_url = '%s/get_result?%s' % (base_url, urllib.urlencode(params)) start = now() while True: if timeout and (now() - start) >= timeout: logging.warning('retrieve_results(%s) timed out', base_url) return {} # Do retries ourselve. response = run_isolated.url_open( result_url, retry_404=False, retry_50x=False) if response is None: # Aggressively poll for results. Do not use retry_404 so # should_stop is polled more often. remaining = min(5, timeout - (now() - start)) if timeout else 5 if remaining > 0: run_isolated.HttpService.sleep_before_retry(1, remaining) else: try: data = json.load(response) or {} except (ValueError, TypeError): logging.warning( 'Received corrupted data for test_key %s. Retrying.', test_key) else: if data['output']: return data if should_stop.get(): return {}
def url_open(url, **kwargs): result = run_isolated.url_open(url, **kwargs) if not result: # If we get no response from the server, assume it is down and raise an # exception. raise run_isolated.MappingError('Unable to connect to server %s' % url) return result
def zip_and_upload(self): """Zips up all the files necessary to run a shard and uploads to Swarming master. """ assert not self._zip_file_hash start_time = time.time() zip_memory_file = StringIO.StringIO() zip_file = zipfile.ZipFile(zip_memory_file, 'w') for source, relpath in self._files.iteritems(): zip_file.write(source, relpath) zip_file.close() print 'Zipping completed, time elapsed: %f' % (time.time() - start_time) zip_memory_file.flush() zip_contents = zip_memory_file.getvalue() zip_memory_file.close() self._zip_file_hash = hashlib.sha1(zip_contents).hexdigest() response = run_isolated.url_open( self._data_server_has + '?token=%s' % self._token(), data=self._zip_file_hash, content_type='application/octet-stream') if response is None: print >> sys.stderr, ( 'Unable to query server for zip file presence, aborting.') return False if response.read(1) == chr(1): print 'Zip file already on server, no need to reupload.' return True print 'Zip file not on server, starting uploading.' url = '%s%s?priority=0&token=%s' % ( self._data_server_storage, self._zip_file_hash, self._token()) response = run_isolated.url_open( url, data=zip_contents, content_type='application/octet-stream') if response is None: print >> sys.stderr, 'Failed to upload the zip file: %s' % url return False return True
def _token(self): if not self._token_cache: result = run_isolated.url_open(self._data_server_get_token) if not result: # TODO(maruel): Implement authentication. raise Failure('Failed to get token, need authentication') # Quote it right away, so creating the urls is simpler. self._token_cache = urllib.quote(result.read()) return self._token_cache
def token(self): if not self._token: result = run_isolated.url_open(self.base_url + '/content/get_token') if not result: # TODO(maruel): Implement authentication. raise Failure('Failed to get token, need authentication') # Quote it right away, so creating the urls is simpler. self._token = urllib.quote(result.read()) return self._token
def zip(self): """Zip up all the files necessary to run a shard.""" start_time = time.time() zip_memory_file = StringIO.StringIO() zip_file = zipfile.ZipFile(zip_memory_file, 'w') zip_file.write(RUN_TEST_PATH, RUN_TEST_NAME) zip_file.write(CLEANUP_SCRIPT_PATH, CLEANUP_SCRIPT_NAME) zip_file.close() print 'Zipping completed, time elapsed: %f' % (time.time() - start_time) zip_memory_file.flush() zip_contents = zip_memory_file.getvalue() zip_memory_file.close() self.zip_file_hash = hashlib.sha1(zip_contents).hexdigest() response = run_isolated.url_open( self.data_server_has + '?token=%s' % self.token(), data=self.zip_file_hash, content_type='application/octet-stream') if response is None: print >> sys.stderr, ( 'Unable to query server for zip file presence, aborting.') return False if response.read(1) == chr(1): print 'Zip file already on server, no need to reupload.' return True print 'Zip file not on server, starting uploading.' url = '%s%s?priority=0&token=%s' % (self.data_server_storage, self.zip_file_hash, self.token()) response = run_isolated.url_open( url, data=zip_contents, content_type='application/octet-stream') if response is None: print >> sys.stderr, 'Failed to upload the zip file: %s' % url return False return True
def zip(self): """Zip up all the files necessary to run a shard.""" start_time = time.time() zip_memory_file = StringIO.StringIO() zip_file = zipfile.ZipFile(zip_memory_file, 'w') zip_file.write(RUN_TEST_PATH, RUN_TEST_NAME) zip_file.write(CLEANUP_SCRIPT_PATH, CLEANUP_SCRIPT_NAME) zip_file.close() print 'Zipping completed, time elapsed: %f' % (time.time() - start_time) zip_memory_file.flush() zip_contents = zip_memory_file.getvalue() zip_memory_file.close() self.zip_file_hash = hashlib.sha1(zip_contents).hexdigest() response = run_isolated.url_open( self.data_server_has + '?token=%s' % self.token(), data=self.zip_file_hash, content_type='application/octet-stream') if response is None: print >> sys.stderr, ( 'Unable to query server for zip file presence, aborting.') return False if response.read(1) == chr(1): print 'Zip file already on server, no need to reupload.' return True print 'Zip file not on server, starting uploading.' url = '%s%s?priority=0&token=%s' % ( self.data_server_storage, self.zip_file_hash, self.token()) response = run_isolated.url_open( url, data=zip_contents, content_type='application/octet-stream') if response is None: print >> sys.stderr, 'Failed to upload the zip file: %s' % url return False return True
def get_test_keys(swarm_base_url, test_name): key_data = urllib.urlencode([('name', test_name)]) test_keys_url = '%s/get_matching_test_cases?%s' % (swarm_base_url, key_data) result = run_isolated.url_open(test_keys_url) if result is None: return [] result_str = result.read() if 'No matching' in result_str: print ('Error: Unable to find any tests with the name, %s, on swarm server' % test_name) return [] # TODO(csharp): return in a proper format (like json) return result_str.split()
def get_test_keys(swarm_base_url, test_name): key_data = urllib.urlencode([('name', test_name)]) test_keys_url = '%s/get_matching_test_cases?%s' % (swarm_base_url, key_data) result = run_isolated.url_open(test_keys_url) if result is None: return [] result_str = result.read() if 'No matching' in result_str: print( 'Error: Unable to find any tests with the name, %s, on swarm server' % test_name) return [] # TODO(csharp): return in a proper format (like json) return result_str.split()
def swarm_get_results(swarm_base_url, test_keys, wait): """Retrieves the given swarm test results from the swarm server and print it to stdout. """ outputs = [] for test in test_keys: result_url = '%s/get_result?r=%s' % (swarm_base_url, test) while True: result = run_isolated.url_open(result_url) if result is None: continue data = json.load(result) if data['output']: outputs.append(data) break if not wait: break return outputs
def process_manifest( file_sha1, test_name, shards, test_filter, os_image, working_dir, data_server, swarm_url, verbose, profile, priority): """Process the manifest file and send off the swarm test request.""" try: manifest = Manifest( file_sha1, test_name, shards, test_filter, os_image, working_dir, data_server, verbose, profile, priority) except ValueError as e: print >> sys.stderr, 'Unable to process %s: %s' % (test_name, e) return 1 chromium_setup(manifest) # Zip up relevent files print "Zipping up files..." if not manifest.zip_and_upload(): return 1 # Send test requests off to swarm. print('Sending test requests to swarm.') print('Server: %s' % swarm_url) print('Job name: %s' % test_name) test_url = swarm_url.rstrip('/') + '/test' manifest_text = manifest.to_json() result = run_isolated.url_open(test_url, data={'request': manifest_text}) if not result: print >> sys.stderr, 'Failed to send test for %s\n%s' % ( test_name, test_url) return 1 try: json.load(result) except (ValueError, TypeError) as e: print >> sys.stderr, 'Failed to send test for %s' % test_name print >> sys.stderr, 'Manifest: %s' % manifest_text print >> sys.stderr, str(e) return 1 return 0
def call(self, mode, sleep_duration, **kwargs): url = self.server.url + '/%s/%f' % (mode, sleep_duration) kwargs['max_attempts'] = 2 return run_isolated.url_open(url, **kwargs)