def start_server(self, config=None, name='default'):
'''Run the Bottle server as a thread'''
from ambry.client.siesta import API
import ambry.server.main
from threading import Thread
import time
from functools import partial
from ambry.client.rest import RemoteLibrary
if not config:
config = self.server_library_config(name)
self.server_url = "http://{}".format(config['host'])
global_logger.info("Checking server at: {}".format(self.server_url))
a = RemoteLibrary(self.server_url)
#
# Test to see of the server is already running.
#
try:
# An echo request to see if the server is running.
r = a.get_is_debug()
if r.object:
global_logger.info( 'Already running a debug server')
else:
global_logger.info( 'Already running a non-debug server')
# We already have a server, so carry on
return config
except:
# We'll get an exception refused eception if there is not server
global_logger.info( 'No server, starting a local debug server')
server = Thread(target = partial(ambry.server.main.test_run, config) )
server.setDaemon(True)
server.start()
#ambry.server.bottle.debug()
# Wait for the server to start
for i in range(1,10): #@UnusedVariable
try:
# An echo request to see if the server is running.
r = a.get_test_echo('start_server')
break
except:
global_logger.info( 'Server not started yet, waiting')
time.sleep(1)
r = a.get_test_echo('start_server')
return config
def start_server(self, config=None, name='default'):
'''Run the Bottle server as a thread'''
from ambry.client.siesta import API
import ambry.server.main
from threading import Thread
import time
from functools import partial
from ambry.client.rest import RemoteLibrary
if not config:
config = self.server_library_config(name)
self.server_url = "http://{}".format(config['host'])
global_logger.info("Checking server at: {}".format(self.server_url))
a = RemoteLibrary(self.server_url)
#
# Test to see of the server is already running.
#
try:
# An echo request to see if the server is running.
r = a.get_is_debug()
if r.object:
global_logger.info('Already running a debug server')
else:
global_logger.info('Already running a non-debug server')
# We already have a server, so carry on
return config
except:
# We'll get an exception refused eception if there is not server
global_logger.info('No server, starting a local debug server')
server = Thread(target=partial(ambry.server.main.test_run, config))
server.setDaemon(True)
server.start()
#ambry.server.bottle.debug()
# Wait for the server to start
for i in range(1, 10): #@UnusedVariable
try:
# An echo request to see if the server is running.
r = a.get_test_echo('start_server')
break
except:
global_logger.info('Server not started yet, waiting')
time.sleep(1)
r = a.get_test_echo('start_server')
return config
def test_resolve(self):
from ambry.client.rest import RemoteLibrary
l = self.get_library()
l.purge()
print l.info
#
# Check that the library can list datasets that are inserted externally
#
l.put_bundle(self.bundle)
ident = self.bundle.identity
# Local Library
self.assertEquals(ident.vid, l.resolve(ident.vid).vid)
self.assertEquals(ident.vid, l.resolve(ident.vname).vid)
self.assertEquals(ident.vid, l.resolve(ident.cache_key).vid)
self.assertEquals(ident.vid, l.resolve(ident.sname).vid)
for p in self.bundle.partitions:
print '--', p.identity.cache_key
dsid = l.resolve(p.identity.vid)
self.assertEquals(ident.vid, dsid.vid)
self.assertEquals(p.identity.vid, dsid.partition.vid)
dsid = l.resolve(p.identity.cache_key)
if not dsid:
ck = p.identity.cache_key
l.resolve(ck)
self.assertIsNotNone(dsid)
self.assertEquals(ident.vid, dsid.vid)
self.assertEquals(p.identity.vid, dsid.partition.vid)
# Remote Library
self.start_server()
rl = RemoteLibrary(self.server_url)
self.assertEquals(ident.vid, rl.resolve(ident.vid).vid)
self.assertEquals(ident.vid, rl.resolve(ident.vname).vid)
self.assertEquals(ident.vid, rl.resolve(ident.cache_key).vid)
self.assertEquals(ident.vid, (rl.resolve(ident.sname).vid))
for p in self.bundle.partitions:
print '--',p.identity.cache_key
dsid = rl.resolve(p.identity.vid)
self.assertEquals(ident.vid, dsid.vid)
self.assertEquals(p.identity.vid, dsid.partition.vid)
dsid = rl.resolve(p.identity.cache_key)
self.assertEquals(ident.vid, dsid.vid)
self.assertEquals(p.identity.vid, dsid.partition.vid)
print rl.resolve('source/dataset-subset-variation-0.0.1/geot1.geodb')
def test_simple_install(self):
from ambry.client.rest import RemoteLibrary
from ambry.cache.remote import RestReadCache
config = self.start_server()
# Create the library so we can get the same remote config
l = new_library(config)
s3 = l.upstream.last_upstream()
s3.clean()
print "S3 cache ", str(s3)
if not s3.has(self.bundle.identity.cache_key):
print 'Uploading: ', self.bundle.identity.cache_key
s3.put(self.bundle.database.path,self.bundle.identity.cache_key)
self.web_exists(s3,self.bundle.identity.cache_key)
for p in self.bundle.partitions:
if not s3.has(p.identity.cache_key):
print 'Uploading: ', p.identity.cache_key
s3.put(p.database.path,p.identity.cache_key)
self.web_exists(s3,p.identity.cache_key)
else:
print 'Has : ', p.identity.cache_key
#
# Kick the remote library to load the dataset
#
rl = RemoteLibrary(self.server_url)
ident = self.bundle.identity
ident.add_md5(file=self.bundle.database.path)
rl.load_dataset(ident)
self.assertIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',
set([i.fqname for i in rl.list().values()]))
return
# Try variants of find.
r = api.find(self.bundle.identity.name)
self.assertEquals(self.bundle.identity.name, r[0].name)
r = api.find(QueryCommand().identity(name = self.bundle.identity.name))
self.assertEquals(self.bundle.identity.name, r[0].name)
for partition in self.bundle.partitions:
r = api.find((QueryCommand().partition(name = partition.identity.name)).to_dict())
self.assertEquals(partition.identity.name, r[0].name)
def stop_server(self):
'''Shutdown the server process by calling the close() API, then waiting for it
to stop serving requests '''
import socket
import time
import ambry.client.exceptions as exc
from requests.exceptions import ConnectionError
from ambry.client.rest import RemoteLibrary
if not self.server_url:
return
a = RemoteLibrary(self.server_url)
try:
is_debug = a.get_is_debug()
except ConnectionError:
# Already closed:
return
if not is_debug:
global_logger.info("Server is not debug, won't stop")
return
else:
global_logger.info("Server at {} is debug, stopping".format(
self.server_url))
# Wait for the server to shutdown
for i in range(1, 10): #@UnusedVariable
try:
a.post_close()
global_logger.info('Teardown: server still running, waiting')
time.sleep(1)
except socket.error:
pass # Just means that the socket is already closed
except IOError:
pass # Probably just means that the socket is already closed
except ConnectionError:
pass # Another way the socket can be closed. Thrown by requests library.
except Exception as e:
global_logger.error(
"Got an exception while stopping: {} {}".format(
type(e), e))
break
time.sleep(2) # Let the socket clear
def resolve_ref_one(self, ref, location = Dataset.LOCATION.LIBRARY):
from requests.exceptions import ConnectionError
from ambry.client.rest import RemoteLibrary
import semantic_version
from ..identity import Identity
if self.local_resolver:
ip,ident = self.local_resolver.resolve_ref_one(ref, location)
if ident:
idents = [ident]
else:
idents = []
else:
ip = Identity.classify(ref)
idents = []
# If the local returned a result, we only need to go to the
# remote if this is a semantic version request, to possible
# get a newer version
if len(idents) == 0 or isinstance(ip.version, semantic_version.Spec):
if self.urls:
for url in self.urls:
rl = RemoteLibrary(url)
try:
ident = rl.resolve(ref, location)
except ConnectionError:
continue
if ident:
ident.locations.set(Dataset.LOCATION.REMOTE)
ident.url = url
idents.append(ident)
if not idents:
return ip, None
idents = sorted(idents, reverse=True, key=lambda x: x.on.revision )
# Since we sorted by revision, and the internal resolutions take care of semantic versioning,
# if this is a semantic version request, the idents array should be sorted with the highest revision number
# for the spec at the top
return ip, idents.pop(0)
def stop_server(self):
'''Shutdown the server process by calling the close() API, then waiting for it
to stop serving requests '''
import socket
import time
import ambry.client.exceptions as exc
from requests.exceptions import ConnectionError
from ambry.client.rest import RemoteLibrary
if not self.server_url:
return
a = RemoteLibrary(self.server_url)
try:
is_debug = a.get_is_debug()
except ConnectionError:
# Already closed:
return
if not is_debug:
global_logger.info("Server is not debug, won't stop")
return
else:
global_logger.info("Server at {} is debug, stopping".format(self.server_url))
# Wait for the server to shutdown
for i in range(1,10): #@UnusedVariable
try:
a.post_close()
global_logger.info('Teardown: server still running, waiting')
time.sleep(1)
except socket.error:
pass # Just means that the socket is already closed
except IOError:
pass # Probably just means that the socket is already closed
except ConnectionError:
pass # Another way the socket can be closed. Thrown by requests library.
except Exception as e:
global_logger.error("Got an exception while stopping: {} {}".format(type(e), e))
break
time.sleep(2) # Let the socket clear
def sync_remotes(self, clean=False):
from ambry.client.rest import RemoteLibrary
if clean:
self.database.session.query(Dataset).filter(Dataset.location == Dataset.LOCATION.REMOTE).delete()
self.files.query.type(Dataset.LOCATION.REMOTE).delete()
if not self.remotes:
return
for url in self.remotes:
self.logger.info("Remote sync: {}".format(url))
rl = RemoteLibrary(url)
for ident in rl.list().values():
if self.files.query.type(Dataset.LOCATION.REMOTE).ref(ident.vid).one_maybe:
continue
self.sync_remote_dataset(url, ident)
self.logger.info("Remote {} sync: {}".format(url, ident.fqname))
def test_connection(self):
'''
Test some of the server's test functions
:return:
'''
from ambry.client.rest import RemoteLibrary
self.start_server()
a = RemoteLibrary(self.server_url)
self.assertEquals('foobar', a.get_test_echo('foobar'))
self.assertEquals('foobar', a.put_test_echo('foobar'))
with self.assertRaises(Exception):
a.get_test_exception()
r = a.get_root()
self.assertIn('/tmp/server/remote', r['remotes'])
def test_connection(self):
'''
Test some of the server's test functions
:return:
'''
from ambry.client.rest import RemoteLibrary
self.start_server()
a = RemoteLibrary(self.server_url)
self.assertEquals('foobar', a.get_test_echo('foobar'))
self.assertEquals('foobar', a.put_test_echo('foobar'))
with self.assertRaises(Exception):
a.get_test_exception()
r = a.get_root()
self.assertEquals('devtest.sandiegodata.org', r['upstream']['bucket'])
self.assertEquals('library-test', r['upstream']['prefix'])
def test_load(self):
from ambry.run import get_runconfig, RunConfig
from ambry.client.rest import RemoteLibrary
from ambry.cache import new_cache
from ambry.util import md5_for_file
from ambry.identity import Identity
config = self.start_server()
l = new_library(config)
rl = RemoteLibrary(self.server_url)
#
# Check that the library can list datasets that are inserted externally
#
l.put_bundle(self.bundle)
s = set([i.fqname for i in rl.list().values()])
self.assertIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001', s)
dsident = rl.dataset('diEGPXmDC8001')
s = set([i.fqname for i in dsident.partitions.values()])
self.assertEquals(4, len(s))
self.assertIn('source-dataset-subset-variation-tthree-0.0.1~piEGPXmDC8003001', s)
self.assertIn('source-dataset-subset-variation-geot1-geo-0.0.1~piEGPXmDC8001001', s)
self.assertIn('source-dataset-subset-variation-geot2-geo-0.0.1~piEGPXmDC8002001', s)
#
# Upload the dataset to S3, clear the library, then load it back in
#
rc = get_runconfig((os.path.join(self.bundle_dir,'test-run-config.yaml'),RunConfig.USER_ACCOUNTS))
cache = new_cache(rc.filesystem('cached-compressed-s3'))
fn = self.bundle.database.path
identity = self.bundle.identity
relpath = identity.cache_key
r = cache.put(fn, relpath, identity.to_meta(file=fn))
self.assertTrue(bool(cache.has(relpath)))
# clear the library.
l.purge()
self.assertNotIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',
set([i.fqname for i in rl.list()]))
# Load from S3, directly in to the local library
identity.add_md5(md5_for_file(fn))
l.load(identity.cache_key, identity.md5)
self.assertIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',
set([i.fqname for i in rl.list().values()]))
# Do it one more time, using the remote library
l.purge()
self.assertNotIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',
set([i.fqname for i in rl.list().values()]))
rl.load_dataset(identity)
self.assertIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',
set([i.fqname for i in rl.list().values()]))
# Check that we can get the record from the library
self.assertEquals(identity.vid, rl.resolve(identity.vid).vid)
self.assertEquals(identity.vid, rl.resolve(identity.vname).vid)
self.assertEquals(identity.vid, rl.resolve(identity.cache_key).vid)
self.assertEquals(identity.vid, rl.resolve(identity.sname).vid)
def test_joint_resolve(self):
'''Test resolving from either a remote or local library, from the local interface '''
from ambry.identity import Identity
from ambry.client.rest import RemoteLibrary
from ambry.library.query import RemoteResolver
self.start_server()
config = self.server_library_config()
# Create the library so we can get the same remote config
l = new_library(config)
s3 = l.upstream.last_upstream()
print l.info
db = l.database
db.enable_delete = True
db.clean()
l.put_bundle(self.bundle)
# This might not do anything if the files already are in s3
def push_cb(action, metadata, time):
print action, metadata['fqname']
l.push(cb=push_cb)
# Check they are on the web
self.web_exists(s3,self.bundle.identity.cache_key)
for p in self.bundle.partitions:
self.web_exists(s3,p.identity.cache_key)
# Check the basic resolvers
ident = self.bundle.identity
self.assertEquals(ident.vid, l.resolve(ident.vid).vid)
rl = RemoteLibrary(self.server_url)
self.assertEquals(ident.vid, rl.resolve(ident.vname).vid)
# That's the basics, now test the primary use case with the remote resolver.
# Remote resolver only
rr = RemoteResolver(local_resolver=None, remote_urls=[self.server_url])
self.assertEquals(ident.vid, rr.resolve_ref_one(ident.vid)[1].vid)
self.assertEquals('http://localhost:7979', rr.resolve_ref_one(ident.vid)[1].url)
# Local Resolver only
rr = RemoteResolver(local_resolver=l.database.resolver, remote_urls=None)
self.assertEquals(ident.vid, rr.resolve_ref_one(ident.vid)[1].vid)
self.assertIsNone(rr.resolve_ref_one(ident.vid)[1].url)
self.stop_server()
# Remote resolver only
rr = RemoteResolver(local_resolver=None, remote_urls=[self.server_url])
self.assertIsNone(rr.resolve_ref_one(ident.vid)[1])
# Combined
rr = RemoteResolver(local_resolver=l.database.resolver, remote_urls=[self.server_url])
self.assertEquals(ident.vid, rr.resolve_ref_one(ident.vid)[1].vid)
self.assertIsNone(rr.resolve_ref_one(ident.vid)[1].url)
