def test_cache_create_error(self):
_, filename = tempfile.mkstemp()
home_was = os.environ['HOME']
os.environ['HOME'] = filename
try:
c = arvados.http_cache('test')
self.assertEqual(None, c)
finally:
os.environ['HOME'] = home_was
os.unlink(filename)
def test_cache_create_error(self):
_, filename = tempfile.mkstemp()
home_was = os.environ['HOME']
os.environ['HOME'] = filename
try:
c = arvados.http_cache('test')
self.assertEqual(None, c)
finally:
os.environ['HOME'] = home_was
os.unlink(filename)
def run(leave_running_atexit=False):
"""Ensure an API server is running, and ARVADOS_API_* env vars have
admin credentials for it.
If ARVADOS_TEST_API_HOST is set, a parent process has started a
test server for us to use: we just need to reset() it using the
admin token fixture.
If a previous call to run() started a new server process, and it
is still running, we just need to reset() it to fixture state and
return.
If neither of those options work out, we'll really start a new
server.
"""
global my_api_host
# Delete cached discovery documents.
#
# This will clear cached docs that belong to other processes (like
# concurrent test suites) even if they're still running. They should
# be able to tolerate that.
for fn in glob.glob(os.path.join(
str(arvados.http_cache('discovery')),
'*,arvados,v1,rest,*')):
os.unlink(fn)
pid_file = _pidfile('api')
pid_file_ok = find_server_pid(pid_file, 0)
existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
if existing_api_host and pid_file_ok:
if existing_api_host == my_api_host:
try:
return reset()
except:
# Fall through to shutdown-and-start case.
pass
else:
# Server was provided by parent. Can't recover if it's
# unresettable.
return reset()
# Before trying to start up our own server, call stop() to avoid
# "Phusion Passenger Standalone is already running on PID 12345".
# (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
# we know the server is ours to kill.)
stop(force=True)
restore_cwd = os.getcwd()
api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
os.chdir(api_src_dir)
# Either we haven't started a server of our own yet, or it has
# died, or we have lost our credentials, or something else is
# preventing us from calling reset(). Start a new one.
if not os.path.exists('tmp'):
os.makedirs('tmp')
if not os.path.exists('tmp/api'):
os.makedirs('tmp/api')
if not os.path.exists('tmp/logs'):
os.makedirs('tmp/logs')
# Install the git repository fixtures.
gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
if not os.path.isdir(gitdir):
os.makedirs(gitdir)
subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
# The nginx proxy isn't listening here yet, but we need to choose
# the wss:// port now so we can write the API server config file.
wss_port = find_available_port()
_setport('wss', wss_port)
port = find_available_port()
env = os.environ.copy()
env['RAILS_ENV'] = 'test'
env['ARVADOS_TEST_WSS_PORT'] = str(wss_port)
env.pop('ARVADOS_WEBSOCKETS', None)
env.pop('ARVADOS_TEST_API_HOST', None)
env.pop('ARVADOS_API_HOST', None)
env.pop('ARVADOS_API_HOST_INSECURE', None)
env.pop('ARVADOS_API_TOKEN', None)
start_msg = subprocess.check_output(
['bundle', 'exec',
'passenger', 'start', '-d', '-p{}'.format(port),
'--pid-file', pid_file,
'--log-file', os.path.join(os.getcwd(), 'log/test.log'),
'--ssl',
'--ssl-certificate', 'tmp/self-signed.pem',
'--ssl-certificate-key', 'tmp/self-signed.key'],
env=env)
if not leave_running_atexit:
atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
match = re.search(r'Accessible via: https://(.*?)/', start_msg)
if not match:
raise Exception(
"Passenger did not report endpoint: {}".format(start_msg))
my_api_host = match.group(1)
os.environ['ARVADOS_API_HOST'] = my_api_host
# Make sure the server has written its pid file and started
# listening on its TCP port
find_server_pid(pid_file)
_wait_until_port_listens(port)
reset()
os.chdir(restore_cwd)
def run(leave_running_atexit=False):
"""Ensure an API server is running, and ARVADOS_API_* env vars have
admin credentials for it.
If ARVADOS_TEST_API_HOST is set, a parent process has started a
test server for us to use: we just need to reset() it using the
admin token fixture.
If a previous call to run() started a new server process, and it
is still running, we just need to reset() it to fixture state and
return.
If neither of those options work out, we'll really start a new
server.
"""
global my_api_host
# Delete cached discovery documents.
#
# This will clear cached docs that belong to other processes (like
# concurrent test suites) even if they're still running. They should
# be able to tolerate that.
for fn in glob.glob(
os.path.join(str(arvados.http_cache('discovery')),
'*,arvados,v1,rest,*')):
os.unlink(fn)
pid_file = _pidfile('api')
pid_file_ok = find_server_pid(pid_file, 0)
existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
if existing_api_host and pid_file_ok:
if existing_api_host == my_api_host:
try:
return reset()
except:
# Fall through to shutdown-and-start case.
pass
else:
# Server was provided by parent. Can't recover if it's
# unresettable.
return reset()
# Before trying to start up our own server, call stop() to avoid
# "Phusion Passenger Standalone is already running on PID 12345".
# (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
# we know the server is ours to kill.)
stop(force=True)
restore_cwd = os.getcwd()
api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
os.chdir(api_src_dir)
# Either we haven't started a server of our own yet, or it has
# died, or we have lost our credentials, or something else is
# preventing us from calling reset(). Start a new one.
if not os.path.exists('tmp'):
os.makedirs('tmp')
if not os.path.exists('tmp/api'):
os.makedirs('tmp/api')
if not os.path.exists('tmp/logs'):
os.makedirs('tmp/logs')
# Install the git repository fixtures.
gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
if not os.path.isdir(gitdir):
os.makedirs(gitdir)
subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
port = internal_port_from_config("RailsAPI")
env = os.environ.copy()
env['RAILS_ENV'] = 'test'
env['ARVADOS_RAILS_LOG_TO_STDOUT'] = '1'
env.pop('ARVADOS_WEBSOCKETS', None)
env.pop('ARVADOS_TEST_API_HOST', None)
env.pop('ARVADOS_API_HOST', None)
env.pop('ARVADOS_API_HOST_INSECURE', None)
env.pop('ARVADOS_API_TOKEN', None)
logf = open(_logfilename('railsapi'), WRITE_MODE)
railsapi = subprocess.Popen([
'bundle', 'exec', 'passenger', 'start', '-p{}'.format(port),
'--pid-file', pid_file, '--log-file', '/dev/stdout', '--ssl',
'--ssl-certificate', 'tmp/self-signed.pem', '--ssl-certificate-key',
'tmp/self-signed.key'
],
env=env,
stdin=open('/dev/null'),
stdout=logf,
stderr=logf)
if not leave_running_atexit:
atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
my_api_host = "127.0.0.1:" + str(port)
os.environ['ARVADOS_API_HOST'] = my_api_host
# Make sure the server has written its pid file and started
# listening on its TCP port
_wait_until_port_listens(port)
find_server_pid(pid_file)
reset()
os.chdir(restore_cwd)
def run(leave_running_atexit=False):
"""Ensure an API server is running, and ARVADOS_API_* env vars have
admin credentials for it.
If ARVADOS_TEST_API_HOST is set, a parent process has started a
test server for us to use: we just need to reset() it using the
admin token fixture.
If a previous call to run() started a new server process, and it
is still running, we just need to reset() it to fixture state and
return.
If neither of those options work out, we'll really start a new
server.
"""
global my_api_host
# Delete cached discovery document.
shutil.rmtree(arvados.http_cache('discovery'))
pid_file = _pidfile('api')
pid_file_ok = find_server_pid(pid_file, 0)
existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
if existing_api_host and pid_file_ok:
if existing_api_host == my_api_host:
try:
return reset()
except:
# Fall through to shutdown-and-start case.
pass
else:
# Server was provided by parent. Can't recover if it's
# unresettable.
return reset()
# Before trying to start up our own server, call stop() to avoid
# "Phusion Passenger Standalone is already running on PID 12345".
# (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
# we know the server is ours to kill.)
stop(force=True)
restore_cwd = os.getcwd()
api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
os.chdir(api_src_dir)
# Either we haven't started a server of our own yet, or it has
# died, or we have lost our credentials, or something else is
# preventing us from calling reset(). Start a new one.
if not os.path.exists('tmp'):
os.makedirs('tmp')
if not os.path.exists('tmp/api'):
os.makedirs('tmp/api')
if not os.path.exists('tmp/logs'):
os.makedirs('tmp/logs')
if not os.path.exists('tmp/self-signed.pem'):
# We assume here that either passenger reports its listening
# address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
# then the certificate won't match the host and reset() will
# fail certificate verification. If it reports "localhost",
# clients (notably Python SDK's websocket client) might
# resolve localhost as ::1 and then fail to connect.
subprocess.check_call([
'openssl', 'req', '-new', '-x509', '-nodes', '-out',
'tmp/self-signed.pem', '-keyout', 'tmp/self-signed.key', '-days',
'3650', '-subj', '/CN=0.0.0.0'
],
stdout=sys.stderr)
# Install the git repository fixtures.
gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
if not os.path.isdir(gitdir):
os.makedirs(gitdir)
subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
port = find_available_port()
env = os.environ.copy()
env['RAILS_ENV'] = 'test'
env['ARVADOS_WEBSOCKETS'] = 'yes'
env.pop('ARVADOS_TEST_API_HOST', None)
env.pop('ARVADOS_API_HOST', None)
env.pop('ARVADOS_API_HOST_INSECURE', None)
env.pop('ARVADOS_API_TOKEN', None)
start_msg = subprocess.check_output([
'bundle', 'exec', 'passenger', 'start', '-d', '-p{}'.format(port),
'--pid-file', pid_file, '--log-file',
os.path.join(os.getcwd(),
'log/test.log'), '--ssl', '--ssl-certificate',
'tmp/self-signed.pem', '--ssl-certificate-key', 'tmp/self-signed.key'
],
env=env)
if not leave_running_atexit:
atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
match = re.search(r'Accessible via: https://(.*?)/', start_msg)
if not match:
raise Exception(
"Passenger did not report endpoint: {}".format(start_msg))
my_api_host = match.group(1)
os.environ['ARVADOS_API_HOST'] = my_api_host
# Make sure the server has written its pid file and started
# listening on its TCP port
find_server_pid(pid_file)
_wait_until_port_listens(port)
reset()
os.chdir(restore_cwd)
def run(leave_running_atexit=False):
"""Ensure an API server is running, and ARVADOS_API_* env vars have
admin credentials for it.
If ARVADOS_TEST_API_HOST is set, a parent process has started a
test server for us to use: we just need to reset() it using the
admin token fixture.
If a previous call to run() started a new server process, and it
is still running, we just need to reset() it to fixture state and
return.
If neither of those options work out, we'll really start a new
server.
"""
global my_api_host
# Delete cached discovery document.
shutil.rmtree(arvados.http_cache('discovery'))
pid_file = os.path.join(SERVICES_SRC_DIR, 'api', SERVER_PID_PATH)
pid_file_ok = find_server_pid(pid_file, 0)
existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
if existing_api_host and pid_file_ok:
if existing_api_host == my_api_host:
try:
return reset()
except:
# Fall through to shutdown-and-start case.
pass
else:
# Server was provided by parent. Can't recover if it's
# unresettable.
return reset()
# Before trying to start up our own server, call stop() to avoid
# "Phusion Passenger Standalone is already running on PID 12345".
# (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
# we know the server is ours to kill.)
stop(force=True)
restore_cwd = os.getcwd()
api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
os.chdir(api_src_dir)
# Either we haven't started a server of our own yet, or it has
# died, or we have lost our credentials, or something else is
# preventing us from calling reset(). Start a new one.
if not os.path.exists('tmp/self-signed.pem'):
# We assume here that either passenger reports its listening
# address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
# then the certificate won't match the host and reset() will
# fail certificate verification. If it reports "localhost",
# clients (notably Python SDK's websocket client) might
# resolve localhost as ::1 and then fail to connect.
subprocess.check_call([
'openssl', 'req', '-new', '-x509', '-nodes',
'-out', 'tmp/self-signed.pem',
'-keyout', 'tmp/self-signed.key',
'-days', '3650',
'-subj', '/CN=0.0.0.0'],
stdout=sys.stderr)
port = find_available_port()
env = os.environ.copy()
env['RAILS_ENV'] = 'test'
env['ARVADOS_WEBSOCKETS'] = 'yes'
env.pop('ARVADOS_TEST_API_HOST', None)
env.pop('ARVADOS_API_HOST', None)
env.pop('ARVADOS_API_HOST_INSECURE', None)
env.pop('ARVADOS_API_TOKEN', None)
start_msg = subprocess.check_output(
['bundle', 'exec',
'passenger', 'start', '-d', '-p{}'.format(port),
'--pid-file', os.path.join(os.getcwd(), pid_file),
'--log-file', os.path.join(os.getcwd(), 'log/test.log'),
'--ssl',
'--ssl-certificate', 'tmp/self-signed.pem',
'--ssl-certificate-key', 'tmp/self-signed.key'],
env=env)
if not leave_running_atexit:
atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
match = re.search(r'Accessible via: https://(.*?)/', start_msg)
if not match:
raise Exception(
"Passenger did not report endpoint: {}".format(start_msg))
my_api_host = match.group(1)
os.environ['ARVADOS_API_HOST'] = my_api_host
# Make sure the server has written its pid file before continuing
find_server_pid(pid_file)
reset()
os.chdir(restore_cwd)
def run(leave_running_atexit=False):
"""Ensure an API server is running, and ARVADOS_API_* env vars have
admin credentials for it.
If ARVADOS_TEST_API_HOST is set, a parent process has started a
test server for us to use: we just need to reset() it using the
admin token fixture.
If a previous call to run() started a new server process, and it
is still running, we just need to reset() it to fixture state and
return.
If neither of those options work out, we'll really start a new
server.
"""
global my_api_host
# Delete cached discovery documents.
#
# This will clear cached docs that belong to other processes (like
# concurrent test suites) even if they're still running. They should
# be able to tolerate that.
for fn in glob.glob(
os.path.join(str(arvados.http_cache('discovery')),
'*,arvados,v1,rest,*')):
os.unlink(fn)
pid_file = _pidfile('api')
pid_file_ok = find_server_pid(pid_file, 0)
existing_api_host = os.environ.get('ARVADOS_TEST_API_HOST', my_api_host)
if existing_api_host and pid_file_ok:
if existing_api_host == my_api_host:
try:
return reset()
except:
# Fall through to shutdown-and-start case.
pass
else:
# Server was provided by parent. Can't recover if it's
# unresettable.
return reset()
# Before trying to start up our own server, call stop() to avoid
# "Phusion Passenger Standalone is already running on PID 12345".
# (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
# we know the server is ours to kill.)
stop(force=True)
restore_cwd = os.getcwd()
api_src_dir = os.path.join(SERVICES_SRC_DIR, 'api')
os.chdir(api_src_dir)
# Either we haven't started a server of our own yet, or it has
# died, or we have lost our credentials, or something else is
# preventing us from calling reset(). Start a new one.
if not os.path.exists('tmp'):
os.makedirs('tmp')
if not os.path.exists('tmp/api'):
os.makedirs('tmp/api')
if not os.path.exists('tmp/logs'):
os.makedirs('tmp/logs')
# Install the git repository fixtures.
gitdir = os.path.join(SERVICES_SRC_DIR, 'api', 'tmp', 'git')
gittarball = os.path.join(SERVICES_SRC_DIR, 'api', 'test', 'test.git.tar')
if not os.path.isdir(gitdir):
os.makedirs(gitdir)
subprocess.check_output(['tar', '-xC', gitdir, '-f', gittarball])
# The nginx proxy isn't listening here yet, but we need to choose
# the wss:// port now so we can write the API server config file.
wss_port = find_available_port()
_setport('wss', wss_port)
port = find_available_port()
env = os.environ.copy()
env['RAILS_ENV'] = 'test'
env['ARVADOS_TEST_WSS_PORT'] = str(wss_port)
env.pop('ARVADOS_WEBSOCKETS', None)
env.pop('ARVADOS_TEST_API_HOST', None)
env.pop('ARVADOS_API_HOST', None)
env.pop('ARVADOS_API_HOST_INSECURE', None)
env.pop('ARVADOS_API_TOKEN', None)
start_msg = subprocess.check_output([
'bundle', 'exec', 'passenger', 'start', '-d', '-p{}'.format(port),
'--pid-file', pid_file, '--log-file',
os.path.join(os.getcwd(),
'log/test.log'), '--ssl', '--ssl-certificate',
'tmp/self-signed.pem', '--ssl-certificate-key', 'tmp/self-signed.key'
],
env=env)
if not leave_running_atexit:
atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
match = re.search(r'Accessible via: https://(.*?)/', start_msg)
if not match:
raise Exception(
"Passenger did not report endpoint: {}".format(start_msg))
my_api_host = match.group(1)
os.environ['ARVADOS_API_HOST'] = my_api_host
# Make sure the server has written its pid file and started
# listening on its TCP port
find_server_pid(pid_file)
_wait_until_port_listens(port)
reset()
os.chdir(restore_cwd)
def run(leave_running_atexit=False):
"""Ensure an API server is running, and ARVADOS_API_* env vars have
admin credentials for it.
If ARVADOS_TEST_API_HOST is set, a parent process has started a
test server for us to use: we just need to reset() it using the
admin token fixture.
If a previous call to run() started a new server process, and it
is still running, we just need to reset() it to fixture state and
return.
If neither of those options work out, we'll really start a new
server.
"""
global my_api_host
# Delete cached discovery document.
shutil.rmtree(arvados.http_cache("discovery"))
pid_file = _pidfile("api")
pid_file_ok = find_server_pid(pid_file, 0)
existing_api_host = os.environ.get("ARVADOS_TEST_API_HOST", my_api_host)
if existing_api_host and pid_file_ok:
if existing_api_host == my_api_host:
try:
return reset()
except:
# Fall through to shutdown-and-start case.
pass
else:
# Server was provided by parent. Can't recover if it's
# unresettable.
return reset()
# Before trying to start up our own server, call stop() to avoid
# "Phusion Passenger Standalone is already running on PID 12345".
# (If we've gotten this far, ARVADOS_TEST_API_HOST isn't set, so
# we know the server is ours to kill.)
stop(force=True)
restore_cwd = os.getcwd()
api_src_dir = os.path.join(SERVICES_SRC_DIR, "api")
os.chdir(api_src_dir)
# Either we haven't started a server of our own yet, or it has
# died, or we have lost our credentials, or something else is
# preventing us from calling reset(). Start a new one.
if not os.path.exists("tmp"):
os.makedirs("tmp")
if not os.path.exists("tmp/api"):
os.makedirs("tmp/api")
if not os.path.exists("tmp/logs"):
os.makedirs("tmp/logs")
if not os.path.exists("tmp/self-signed.pem"):
# We assume here that either passenger reports its listening
# address as https:/0.0.0.0:port/. If it reports "127.0.0.1"
# then the certificate won't match the host and reset() will
# fail certificate verification. If it reports "localhost",
# clients (notably Python SDK's websocket client) might
# resolve localhost as ::1 and then fail to connect.
subprocess.check_call(
[
"openssl",
"req",
"-new",
"-x509",
"-nodes",
"-out",
"tmp/self-signed.pem",
"-keyout",
"tmp/self-signed.key",
"-days",
"3650",
"-subj",
"/CN=0.0.0.0",
],
stdout=sys.stderr,
)
# Install the git repository fixtures.
gitdir = os.path.join(SERVICES_SRC_DIR, "api", "tmp", "git")
gittarball = os.path.join(SERVICES_SRC_DIR, "api", "test", "test.git.tar")
if not os.path.isdir(gitdir):
os.makedirs(gitdir)
subprocess.check_output(["tar", "-xC", gitdir, "-f", gittarball])
port = find_available_port()
env = os.environ.copy()
env["RAILS_ENV"] = "test"
env["ARVADOS_WEBSOCKETS"] = "yes"
env.pop("ARVADOS_TEST_API_HOST", None)
env.pop("ARVADOS_API_HOST", None)
env.pop("ARVADOS_API_HOST_INSECURE", None)
env.pop("ARVADOS_API_TOKEN", None)
start_msg = subprocess.check_output(
[
"bundle",
"exec",
"passenger",
"start",
"-d",
"-p{}".format(port),
"--pid-file",
pid_file,
"--log-file",
os.path.join(os.getcwd(), "log/test.log"),
"--ssl",
"--ssl-certificate",
"tmp/self-signed.pem",
"--ssl-certificate-key",
"tmp/self-signed.key",
],
env=env,
)
if not leave_running_atexit:
atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
match = re.search(r"Accessible via: https://(.*?)/", start_msg)
if not match:
raise Exception("Passenger did not report endpoint: {}".format(start_msg))
my_api_host = match.group(1)
os.environ["ARVADOS_API_HOST"] = my_api_host
# Make sure the server has written its pid file and started
# listening on its TCP port
find_server_pid(pid_file)
_wait_until_port_listens(port)
reset()
os.chdir(restore_cwd)
