def test_cache_create_error(self): _, filename = tempfile.mkstemp() home_was = os.environ['HOME'] os.environ['HOME'] = filename try: c = arvados.http_cache('test') self.assertEqual(None, c) finally: os.environ['HOME'] = home_was os.unlink(filename)
def run(leave_running_atexit=False): """Ensure an API server is running, and ARVADOS_API_* env vars have admin credentials for it. If ARVADOS_TEST_API_HOST is set, a parent process has started a test server for us to use: we just need to reset() it using the admin token fixture. If a previous call to run() started a new server process, and it is still running, we just need to reset() it to fixture state and return. If neither of those options work out, we'll really start a new server. """ global my_api_host # Delete cached discovery documents. # # This will clear cached docs that belong to other processes (like # concurrent test suites) even if they're still running. They should # be able to tolerate that. for fn in glob.glob(os.path.join( str(arvados.http_cache('discovery')), '*,arvados,v1,rest,*')): os.unlink(fn) pid_file = _pidfile('api') pid_file_ok = find_server_pid(pid_file, 0) existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host) if existing_api_host and pid_file_ok: if existing_api_host == my_api_host: try: return reset() except: # Fall through to shutdown-and-start case. pass else: # Server was provided by parent. Can't recover if it's # unresettable. return reset() # Before trying to start up our own server, call stop() to avoid # "Phusion Passenger Standalone is already running on PID 12345". # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so # we know the server is ours to kill.) stop(force=True) restore_cwd = os.getcwd() api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api') os.chdir(api_src_dir) # Either we haven't started a server of our own yet, or it has # died, or we have lost our credentials, or something else is # preventing us from calling reset(). Start a new one. if not os.path.exists('tmp'): os.makedirs('tmp') if not os.path.exists('tmp/api'): os.makedirs('tmp/api') if not os.path.exists('tmp/logs'): os.makedirs('tmp/logs') # Install the git repository fixtures. gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git') gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar') if not os.path.isdir(gitdir): os.makedirs(gitdir) subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball]) # The nginx proxy isn't listening here yet, but we need to choose # the wss:// port now so we can write the API server config file. wss_port = find_available_port() _setport('wss', wss_port) port = find_available_port() env = os.environ.copy() env['RAILS_ENV'] = 'test' env['ARVADOS_TEST_WSS_PORT'] = str(wss_port) env.pop('ARVADOS_WEBSOCKETS', None) env.pop('ARVADOS_TEST_API_HOST', None) env.pop('ARVADOS_API_HOST', None) env.pop('ARVADOS_API_HOST_INSECURE', None) env.pop('ARVADOS_API_TOKEN', None) start_msg = subprocess.check_output( ['bundle', 'exec', 'passenger', 'start', '-d', '-p{}'.format(port), '--pid-file', pid_file, '--log-file', os.path.join(os.getcwd(), 'log/test.log'), '--ssl', '--ssl-certificate', 'tmp/self-signed.pem', '--ssl-certificate-key', 'tmp/self-signed.key'], env=env) if not leave_running_atexit: atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir) match = re.search(r'Accessible via: https://(.*?)/', start_msg) if not match: raise Exception( "Passenger did not report endpoint: {}".format(start_msg)) my_api_host = match.group(1) os.environ['ARVADOS_API_HOST'] = my_api_host # Make sure the server has written its pid file and started # listening on its TCP port find_server_pid(pid_file) _wait_until_port_listens(port) reset() os.chdir(restore_cwd)
def run(leave_running_atexit=False): """Ensure an API server is running, and ARVADOS_API_* env vars have admin credentials for it. If ARVADOS_TEST_API_HOST is set, a parent process has started a test server for us to use: we just need to reset() it using the admin token fixture. If a previous call to run() started a new server process, and it is still running, we just need to reset() it to fixture state and return. If neither of those options work out, we'll really start a new server. """ global my_api_host # Delete cached discovery documents. # # This will clear cached docs that belong to other processes (like # concurrent test suites) even if they're still running. They should # be able to tolerate that. for fn in glob.glob( os.path.join(str(arvados.http_cache('discovery')), '*,arvados,v1,rest,*')): os.unlink(fn) pid_file = _pidfile('api') pid_file_ok = find_server_pid(pid_file, 0) existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host) if existing_api_host and pid_file_ok: if existing_api_host == my_api_host: try: return reset() except: # Fall through to shutdown-and-start case. pass else: # Server was provided by parent. Can't recover if it's # unresettable. return reset() # Before trying to start up our own server, call stop() to avoid # "Phusion Passenger Standalone is already running on PID 12345". # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so # we know the server is ours to kill.) stop(force=True) restore_cwd = os.getcwd() api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api') os.chdir(api_src_dir) # Either we haven't started a server of our own yet, or it has # died, or we have lost our credentials, or something else is # preventing us from calling reset(). Start a new one. if not os.path.exists('tmp'): os.makedirs('tmp') if not os.path.exists('tmp/api'): os.makedirs('tmp/api') if not os.path.exists('tmp/logs'): os.makedirs('tmp/logs') # Install the git repository fixtures. gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git') gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar') if not os.path.isdir(gitdir): os.makedirs(gitdir) subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball]) port = internal_port_from_config("RailsAPI") env = os.environ.copy() env['RAILS_ENV'] = 'test' env['ARVADOS_RAILS_LOG_TO_STDOUT'] = '1' env.pop('ARVADOS_WEBSOCKETS', None) env.pop('ARVADOS_TEST_API_HOST', None) env.pop('ARVADOS_API_HOST', None) env.pop('ARVADOS_API_HOST_INSECURE', None) env.pop('ARVADOS_API_TOKEN', None) logf = open(_logfilename('railsapi'), WRITE_MODE) railsapi = subprocess.Popen([ 'bundle', 'exec', 'passenger', 'start', '-p{}'.format(port), '--pid-file', pid_file, '--log-file', '/dev/stdout', '--ssl', '--ssl-certificate', 'tmp/self-signed.pem', '--ssl-certificate-key', 'tmp/self-signed.key' ], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf) if not leave_running_atexit: atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir) my_api_host = "127.0.0.1:" + str(port) os.environ['ARVADOS_API_HOST'] = my_api_host # Make sure the server has written its pid file and started # listening on its TCP port _wait_until_port_listens(port) find_server_pid(pid_file) reset() os.chdir(restore_cwd)
def run(leave_running_atexit=False): """Ensure an API server is running, and ARVADOS_API_* env vars have admin credentials for it. If ARVADOS_TEST_API_HOST is set, a parent process has started a test server for us to use: we just need to reset() it using the admin token fixture. If a previous call to run() started a new server process, and it is still running, we just need to reset() it to fixture state and return. If neither of those options work out, we'll really start a new server. """ global my_api_host # Delete cached discovery document. shutil.rmtree(arvados.http_cache('discovery')) pid_file = _pidfile('api') pid_file_ok = find_server_pid(pid_file, 0) existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host) if existing_api_host and pid_file_ok: if existing_api_host == my_api_host: try: return reset() except: # Fall through to shutdown-and-start case. pass else: # Server was provided by parent. Can't recover if it's # unresettable. return reset() # Before trying to start up our own server, call stop() to avoid # "Phusion Passenger Standalone is already running on PID 12345". # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so # we know the server is ours to kill.) stop(force=True) restore_cwd = os.getcwd() api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api') os.chdir(api_src_dir) # Either we haven't started a server of our own yet, or it has # died, or we have lost our credentials, or something else is # preventing us from calling reset(). Start a new one. if not os.path.exists('tmp'): os.makedirs('tmp') if not os.path.exists('tmp/api'): os.makedirs('tmp/api') if not os.path.exists('tmp/logs'): os.makedirs('tmp/logs') if not os.path.exists('tmp/self-signed.pem'): # We assume here that either passenger reports its listening # address as https:/0.0.0.0:port/. If it reports "127.0.0.1" # then the certificate won't match the host and reset() will # fail certificate verification. If it reports "localhost", # clients (notably Python SDK's websocket client) might # resolve localhost as ::1 and then fail to connect. subprocess.check_call([ 'openssl', 'req', '-new', '-x509', '-nodes', '-out', 'tmp/self-signed.pem', '-keyout', 'tmp/self-signed.key', '-days', '3650', '-subj', '/CN=0.0.0.0' ], stdout=sys.stderr) # Install the git repository fixtures. gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git') gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar') if not os.path.isdir(gitdir): os.makedirs(gitdir) subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball]) port = find_available_port() env = os.environ.copy() env['RAILS_ENV'] = 'test' env['ARVADOS_WEBSOCKETS'] = 'yes' env.pop('ARVADOS_TEST_API_HOST', None) env.pop('ARVADOS_API_HOST', None) env.pop('ARVADOS_API_HOST_INSECURE', None) env.pop('ARVADOS_API_TOKEN', None) start_msg = subprocess.check_output([ 'bundle', 'exec', 'passenger', 'start', '-d', '-p{}'.format(port), '--pid-file', pid_file, '--log-file', os.path.join(os.getcwd(), 'log/test.log'), '--ssl', '--ssl-certificate', 'tmp/self-signed.pem', '--ssl-certificate-key', 'tmp/self-signed.key' ], env=env) if not leave_running_atexit: atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir) match = re.search(r'Accessible via: https://(.*?)/', start_msg) if not match: raise Exception( "Passenger did not report endpoint: {}".format(start_msg)) my_api_host = match.group(1) os.environ['ARVADOS_API_HOST'] = my_api_host # Make sure the server has written its pid file and started # listening on its TCP port find_server_pid(pid_file) _wait_until_port_listens(port) reset() os.chdir(restore_cwd)
def run(leave_running_atexit=False): """Ensure an API server is running, and ARVADOS_API_* env vars have admin credentials for it. If ARVADOS_TEST_API_HOST is set, a parent process has started a test server for us to use: we just need to reset() it using the admin token fixture. If a previous call to run() started a new server process, and it is still running, we just need to reset() it to fixture state and return. If neither of those options work out, we'll really start a new server. """ global my_api_host # Delete cached discovery document. shutil.rmtree(arvados.http_cache('discovery')) pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH) pid_file_ok = find_server_pid(pid_file, 0) existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host) if existing_api_host and pid_file_ok: if existing_api_host == my_api_host: try: return reset() except: # Fall through to shutdown-and-start case. pass else: # Server was provided by parent. Can't recover if it's # unresettable. return reset() # Before trying to start up our own server, call stop() to avoid # "Phusion Passenger Standalone is already running on PID 12345". # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so # we know the server is ours to kill.) stop(force=True) restore_cwd = os.getcwd() api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api') os.chdir(api_src_dir) # Either we haven't started a server of our own yet, or it has # died, or we have lost our credentials, or something else is # preventing us from calling reset(). Start a new one. if not os.path.exists('tmp/self-signed.pem'): # We assume here that either passenger reports its listening # address as https:/0.0.0.0:port/. If it reports "127.0.0.1" # then the certificate won't match the host and reset() will # fail certificate verification. If it reports "localhost", # clients (notably Python SDK's websocket client) might # resolve localhost as ::1 and then fail to connect. subprocess.check_call([ 'openssl', 'req', '-new', '-x509', '-nodes', '-out', 'tmp/self-signed.pem', '-keyout', 'tmp/self-signed.key', '-days', '3650', '-subj', '/CN=0.0.0.0'], stdout=sys.stderr) port = find_available_port() env = os.environ.copy() env['RAILS_ENV'] = 'test' env['ARVADOS_WEBSOCKETS'] = 'yes' env.pop('ARVADOS_TEST_API_HOST', None) env.pop('ARVADOS_API_HOST', None) env.pop('ARVADOS_API_HOST_INSECURE', None) env.pop('ARVADOS_API_TOKEN', None) start_msg = subprocess.check_output( ['bundle', 'exec', 'passenger', 'start', '-d', '-p{}'.format(port), '--pid-file', os.path.join(os.getcwd(), pid_file), '--log-file', os.path.join(os.getcwd(), 'log/test.log'), '--ssl', '--ssl-certificate', 'tmp/self-signed.pem', '--ssl-certificate-key', 'tmp/self-signed.key'], env=env) if not leave_running_atexit: atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir) match = re.search(r'Accessible via: https://(.*?)/', start_msg) if not match: raise Exception( "Passenger did not report endpoint: {}".format(start_msg)) my_api_host = match.group(1) os.environ['ARVADOS_API_HOST'] = my_api_host # Make sure the server has written its pid file before continuing find_server_pid(pid_file) reset() os.chdir(restore_cwd)
def run(leave_running_atexit=False): """Ensure an API server is running, and ARVADOS_API_* env vars have admin credentials for it. If ARVADOS_TEST_API_HOST is set, a parent process has started a test server for us to use: we just need to reset() it using the admin token fixture. If a previous call to run() started a new server process, and it is still running, we just need to reset() it to fixture state and return. If neither of those options work out, we'll really start a new server. """ global my_api_host # Delete cached discovery documents. # # This will clear cached docs that belong to other processes (like # concurrent test suites) even if they're still running. They should # be able to tolerate that. for fn in glob.glob( os.path.join(str(arvados.http_cache('discovery')), '*,arvados,v1,rest,*')): os.unlink(fn) pid_file = _pidfile('api') pid_file_ok = find_server_pid(pid_file, 0) existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host) if existing_api_host and pid_file_ok: if existing_api_host == my_api_host: try: return reset() except: # Fall through to shutdown-and-start case. pass else: # Server was provided by parent. Can't recover if it's # unresettable. return reset() # Before trying to start up our own server, call stop() to avoid # "Phusion Passenger Standalone is already running on PID 12345". # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so # we know the server is ours to kill.) stop(force=True) restore_cwd = os.getcwd() api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api') os.chdir(api_src_dir) # Either we haven't started a server of our own yet, or it has # died, or we have lost our credentials, or something else is # preventing us from calling reset(). Start a new one. if not os.path.exists('tmp'): os.makedirs('tmp') if not os.path.exists('tmp/api'): os.makedirs('tmp/api') if not os.path.exists('tmp/logs'): os.makedirs('tmp/logs') # Install the git repository fixtures. gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git') gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar') if not os.path.isdir(gitdir): os.makedirs(gitdir) subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball]) # The nginx proxy isn't listening here yet, but we need to choose # the wss:// port now so we can write the API server config file. wss_port = find_available_port() _setport('wss', wss_port) port = find_available_port() env = os.environ.copy() env['RAILS_ENV'] = 'test' env['ARVADOS_TEST_WSS_PORT'] = str(wss_port) env.pop('ARVADOS_WEBSOCKETS', None) env.pop('ARVADOS_TEST_API_HOST', None) env.pop('ARVADOS_API_HOST', None) env.pop('ARVADOS_API_HOST_INSECURE', None) env.pop('ARVADOS_API_TOKEN', None) start_msg = subprocess.check_output([ 'bundle', 'exec', 'passenger', 'start', '-d', '-p{}'.format(port), '--pid-file', pid_file, '--log-file', os.path.join(os.getcwd(), 'log/test.log'), '--ssl', '--ssl-certificate', 'tmp/self-signed.pem', '--ssl-certificate-key', 'tmp/self-signed.key' ], env=env) if not leave_running_atexit: atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir) match = re.search(r'Accessible via: https://(.*?)/', start_msg) if not match: raise Exception( "Passenger did not report endpoint: {}".format(start_msg)) my_api_host = match.group(1) os.environ['ARVADOS_API_HOST'] = my_api_host # Make sure the server has written its pid file and started # listening on its TCP port find_server_pid(pid_file) _wait_until_port_listens(port) reset() os.chdir(restore_cwd)
def run(leave_running_atexit=False): """Ensure an API server is running, and ARVADOS_API_* env vars have admin credentials for it. If ARVADOS_TEST_API_HOST is set, a parent process has started a test server for us to use: we just need to reset() it using the admin token fixture. If a previous call to run() started a new server process, and it is still running, we just need to reset() it to fixture state and return. If neither of those options work out, we'll really start a new server. """ global my_api_host # Delete cached discovery document. shutil.rmtree(arvados.http_cache("discovery")) pid_file = _pidfile("api") pid_file_ok = find_server_pid(pid_file, 0) existing_api_host = os.environ.get("ARVADOS_TEST_API_HOST", my_api_host) if existing_api_host and pid_file_ok: if existing_api_host == my_api_host: try: return reset() except: # Fall through to shutdown-and-start case. pass else: # Server was provided by parent. Can't recover if it's # unresettable. return reset() # Before trying to start up our own server, call stop() to avoid # "Phusion Passenger Standalone is already running on PID 12345". # (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so # we know the server is ours to kill.) stop(force=True) restore_cwd = os.getcwd() api_src_dir = os.path.join(SERVICES_SRC_DIR, "api") os.chdir(api_src_dir) # Either we haven't started a server of our own yet, or it has # died, or we have lost our credentials, or something else is # preventing us from calling reset(). Start a new one. if not os.path.exists("tmp"): os.makedirs("tmp") if not os.path.exists("tmp/api"): os.makedirs("tmp/api") if not os.path.exists("tmp/logs"): os.makedirs("tmp/logs") if not os.path.exists("tmp/self-signed.pem"): # We assume here that either passenger reports its listening # address as https:/0.0.0.0:port/. If it reports "127.0.0.1" # then the certificate won't match the host and reset() will # fail certificate verification. If it reports "localhost", # clients (notably Python SDK's websocket client) might # resolve localhost as ::1 and then fail to connect. subprocess.check_call( [ "openssl", "req", "-new", "-x509", "-nodes", "-out", "tmp/self-signed.pem", "-keyout", "tmp/self-signed.key", "-days", "3650", "-subj", "/CN=0.0.0.0", ], stdout=sys.stderr, ) # Install the git repository fixtures. gitdir = os.path.join(SERVICES_SRC_DIR, "api", "tmp", "git") gittarball = os.path.join(SERVICES_SRC_DIR, "api", "test", "test.git.tar") if not os.path.isdir(gitdir): os.makedirs(gitdir) subprocess.check_output(["tar", "-xC", gitdir, "-f", gittarball]) port = find_available_port() env = os.environ.copy() env["RAILS_ENV"] = "test" env["ARVADOS_WEBSOCKETS"] = "yes" env.pop("ARVADOS_TEST_API_HOST", None) env.pop("ARVADOS_API_HOST", None) env.pop("ARVADOS_API_HOST_INSECURE", None) env.pop("ARVADOS_API_TOKEN", None) start_msg = subprocess.check_output( [ "bundle", "exec", "passenger", "start", "-d", "-p{}".format(port), "--pid-file", pid_file, "--log-file", os.path.join(os.getcwd(), "log/test.log"), "--ssl", "--ssl-certificate", "tmp/self-signed.pem", "--ssl-certificate-key", "tmp/self-signed.key", ], env=env, ) if not leave_running_atexit: atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir) match = re.search(r"Accessible via: https://(.*?)/", start_msg) if not match: raise Exception("Passenger did not report endpoint: {}".format(start_msg)) my_api_host = match.group(1) os.environ["ARVADOS_API_HOST"] = my_api_host # Make sure the server has written its pid file and started # listening on its TCP port find_server_pid(pid_file) _wait_until_port_listens(port) reset() os.chdir(restore_cwd)