Example #1
0
def config_value(args, rc):

    def sub_value(value, subs):

        if isinstance(value, (list, tuple)):
            return [i.format(**subs) for i in value]
        else:
            try:
                return value.format(**subs)
            except AttributeError:
                return str(value)

    def dump_key(key, subs):
        for path, value in rc.config.flatten():
            dot_path = '.'.join(path)
            if key:
                if key == dot_path:
                    print sub_value(value, subs)
                    return
            else:
                print dot_path, '=', sub_value(value, subs)

    subs = dict(root=rc.filesystem_path('root'))

    if not args.key:
        if args.yaml:
            print rc.dump()
        else:
            dump_key(None, subs)
    else:
        dump_key(args.key[0], subs)
Example #2
0
    def setUp(self):

        self.copy_or_build_bundle()

        self.bundle_dir = os.path.join(
            os.path.dirname(os.path.abspath(__file__)), 'testbundle')
        self.rc = RunConfig([
            os.path.join(self.bundle_dir, 'client-test-config.yaml'),
            os.path.join(self.bundle_dir, 'bundle.yaml'), RunConfig.USER_CONFIG
        ])

        self.server_rc = RunConfig([
            os.path.join(self.bundle_dir, 'server-test-config.yaml'),
            RunConfig.USER_CONFIG
        ])

        self.bundle = Bundle()
        self.bundle_dir = self.bundle.bundle_dir
Example #3
0
    def setUp(self):

        import shutil, os

        self.copy_or_build_bundle()
        self.bundle_dir = os.path.join(
            os.path.dirname(os.path.abspath(__file__)), '../testbundle')

        self.bundle = Bundle()
        self.bundle_dir = self.bundle.bundle_dir

        self.server_rc = RunConfig(
            [os.path.join(self.bundle_dir, 'server-test-config.yaml')])
        self.client_rc = RunConfig(
            [os.path.join(self.bundle_dir, 'client-test-config.yaml')])

        root = os.path.join(self.client_rc.filesystem.root_dir, 'test')

        shutil.rmtree(root)
Example #4
0
    def setUp(self):
        
        rm_rf('/tmp/server')

        self.copy_or_build_bundle()
        self.bundle_dir =  os.path.join(os.path.dirname(os.path.abspath(__file__)),'testbundle')    
        self.rc = RunConfig([os.path.join(self.bundle_dir,'client-test-config.yaml'),
                             os.path.join(self.bundle_dir,'bundle.yaml'),
                             RunConfig.USER_ACCOUNTS])
         
        self.server_rc = RunConfig([os.path.join(self.bundle_dir,'server-test-config.yaml'),
                                    RunConfig.USER_ACCOUNTS])
       
        self.bundle = Bundle()  
        self.bundle_dir = self.bundle.bundle_dir
Example #5
0
    def build(bundle_dir):
        from ambry.library import new_library



        # Import the bundle file from the directory

        bundle_class = load_bundle(bundle_dir)
        bundle = bundle_class(bundle_dir)

        l = new_library(rc.library(args.library_name))

        if l.get(bundle.identity.vid)  and not args.force:
            prt("{} Bundle is already in library", bundle.identity.name)
            return
        elif bundle.is_built and not args.force and not args.clean:
            prt("{} Bundle is already built",bundle.identity.name)
            return
        else:

            if args.dryrun:
                prt("{} Would build but in dry run ", bundle.identity.name)
                return

            repo.bundle = bundle
             
            if args.clean: 
                bundle.clean()
                
            # Re-create after cleaning is important for something ... 

            bundle = bundle_class(bundle_dir)
                

            prt("{} Building ", bundle.identity.name)

            if not bundle.run_prepare():
                fatal("{} Prepare failed", bundle.identity.name)
            
            if not bundle.run_build():
                fatal("{} Build failed", bundle.identity.name)
            
        if args.install and not args.dryrun:
            if not bundle.run_install(force=True):
                fatal('{} Install failed', bundle.identity.name)
Example #6
0
    def build(bundle_dir):
        from ambry.library import new_library

        # Import the bundle file from the directory

        bundle_class = load_bundle(bundle_dir)
        bundle = bundle_class(bundle_dir)

        l = new_library(rc.library(args.library_name))

        if l.get(bundle.identity.vid) and not args.force:
            prt("{} Bundle is already in library", bundle.identity.name)
            return
        elif bundle.is_built and not args.force and not args.clean:
            prt("{} Bundle is already built", bundle.identity.name)
            return
        else:

            if args.dryrun:
                prt("{} Would build but in dry run ", bundle.identity.name)
                return

            repo.bundle = bundle

            if args.clean:
                bundle.clean()

            # Re-create after cleaning is important for something ...

            bundle = bundle_class(bundle_dir)

            prt("{} Building ", bundle.identity.name)

            if not bundle.run_prepare():
                fatal("{} Prepare failed", bundle.identity.name)

            if not bundle.run_build():
                fatal("{} Build failed", bundle.identity.name)

        if args.install and not args.dryrun:
            if not bundle.run_install(force=True):
                fatal('{} Install failed', bundle.identity.name)
Example #7
0
def source_init(args, l, st, rc):
    from ..source.repository import new_repository

    dir_ = args.dir

    if not dir_:
        dir_ = os.getcwd()

    repo = new_repository(rc.sourcerepo(args.name))
    repo.bundle_dir = dir_

    repo.delete_remote()
    import time
    time.sleep(3)
    repo.init_descriptor()
    repo.init_remote()

    repo.push()

    st.sync_bundle(dir_)
Example #8
0
def source_init(args, l, st, rc):
    from ..source.repository import new_repository

    dir_ = args.dir

    if not dir_:
        dir_ = os.getcwd()

    repo = new_repository(rc.sourcerepo(args.name))
    repo.bundle_dir = dir_

    repo.delete_remote()
    import time
    time.sleep(3)
    repo.init_descriptor()
    repo.init_remote()

    repo.push()

    st.sync_bundle(dir_)
Example #9
0
def source_build(args, l, st, rc):
    """Build a single bundle, or a set of bundles in a directory.

    The build process will build all dependencies for each bundle before
    buildng the bundle.

    """

    from ambry.identity import Identity
    from ..source.repository import new_repository

    repo = new_repository(rc.sourcerepo(args.name))

    dir_ = None
    name = None

    if args.dir:
        if os.path.exists(args.dir):
            dir_ = args.dir
            name = None
        else:
            name = args.dir
            try:
                Identity.parse_name(name)
            except:
                fatal(
                    "Argument '{}' must be either a bundle name or a directory".format(name))
                return

    if not dir_:
        dir_ = rc.sourcerepo.dir

    def build(bundle_dir):
        from ambry.library import new_library

        # Import the bundle file from the directory

        bundle_class = load_bundle(bundle_dir)
        bundle = bundle_class(bundle_dir)

        l = new_library(rc.library(args.library_name))

        if l.get(bundle.identity.vid) and not args.force:
            prt("{} Bundle is already in library", bundle.identity.name)
            return
        elif bundle.is_built and not args.force and not args.clean:
            prt("{} Bundle is already built", bundle.identity.name)
            return
        else:

            if args.dryrun:
                prt("{} Would build but in dry run ", bundle.identity.name)
                return

            repo.bundle = bundle

            if args.clean:
                bundle.clean()

            # Re-create after cleaning is important for something ...

            bundle = bundle_class(bundle_dir)

            prt("{} Building ", bundle.identity.name)

            if not bundle.run_prepare():
                fatal("{} Prepare failed", bundle.identity.name)

            if not bundle.run_build():
                fatal("{} Build failed", bundle.identity.name)

        if args.install and not args.dryrun:
            if not bundle.run_install(force=True):
                fatal('{} Install failed', bundle.identity.name)

    build_dirs = {}

    # Find all of the dependencies for the named bundle, and make those first.
    for root, _, files in os.walk(rc.sourcerepo.dir):
        if 'bundle.yaml' in files:
            bundle_class = load_bundle(root)
            bundle = bundle_class(root)
            build_dirs[bundle.identity.name] = root

    if name:
        deps = repo.bundle_deps(name)
        deps.append(name)

    else:

        deps = []

        # Walk the subdirectory for the files to build, and
        # add all of their dependencies
        for root, _, files in os.walk(dir_):
            if 'bundle.yaml' in files:

                bundle_class = load_bundle(root)
                bundle = bundle_class(root)

                for dep in repo.bundle_deps(bundle.identity.name):
                    if dep not in deps:
                        deps.append(dep)

                deps.append(bundle.identity.name)

    for n in deps:
        try:
            dir_ = build_dirs[n]
        except KeyError:
            fatal("Failed to find directory for bundle {}".format(n))

        prt('')
        prt("{} Building in {}".format(n, dir_))
        build(dir_)
Example #10
0
def config_install(args, rc):
    import yaml
    import pkgutil
    import os
    from ambry.run import RunConfig as rc
    import getpass

    edit_args = ' '.join(args.args)

    user = getpass.getuser()

    if user == 'root':
        install_file = rc.ROOT_CONFIG
        default_root = '/ambry'
    elif os.getenv('VIRTUAL_ENV'):  # Special case for python virtual environments
        install_file = os.path.join(os.getenv('VIRTUAL_ENV'), '.ambry.yaml')
        default_root = os.path.join(os.getenv('VIRTUAL_ENV'), 'data')
    else:
        install_file = rc.USER_CONFIG
        warn(("Installing as non-root, to '{}'\n" +
              "Run as root to install for all users.").format(install_file))
        default_root = os.path.join(os.path.expanduser('~'), 'ambry')

    if os.path.exists(install_file):
        if args.edit:
            prt("File output file exists, editing: {}".format(install_file))
            with open(install_file) as f:
                contents = f.read()
        elif args.force:
            prt("File output file exists, overwriting: {}".format(
                install_file))
            contents = pkgutil.get_data(
                "ambry.support", 'ambry-{}.yaml'.format(args.template))
        else:
            fatal(
                "Output file {} exists. Use -e to edit, or -f to overwrite".format(install_file))
    else:
        contents = pkgutil.get_data(
            "ambry.support", 'ambry-{}.yaml'.format(args.template))

    d = yaml.load(contents)

    # Set the key-value entries.
    if edit_args:
        key, value = edit_args.split('=')

        value = value.strip()
        key = key.strip()
        key_parts = key.split('.')
        e = d
        for k in key_parts:
            k = k.strip()
            print k, str(key_parts[-1])
            if str(k) == str(key_parts[-1]):
                e[k] = value
            else:
                
                e = e[k]
                

    if args.root:
        d['filesystem']['root'] = args.root
    elif default_root:
        d['filesystem']['root'] = default_root

    if args.remote:
        try:
            d['library']['default']['remotes'] = [args.remote]
        except Exception as e:
            err("Failed to set remote: {} ".format(e))

    s = yaml.dump(d, indent=4, default_flow_style=False)

    if args.prt:
        prt(s.replace("{", "{{").replace("}", "}}"))

    else:

        dirname = os.path.dirname(install_file)

        if not os.path.isdir(dirname):
            os.makedirs(dirname)

        with open(install_file, 'w') as f:
            prt('Writing config file: {}'.format(install_file))
            f.write(s)

    if not os.path.exists(rc.USER_ACCOUNTS):
        with open(rc.USER_ACCOUNTS, 'w') as f:

            d = dict(accounts=dict(ambry=dict(name=None, email=None)))

            prt('Writing config file: {}'.format(rc.USER_ACCOUNTS))
            f.write(yaml.dump(d, indent=4, default_flow_style=False))

    # Make the directories.

    from ..run import get_runconfig
    rc = get_runconfig(install_file)

    for name in rc.group('filesystem').keys():
        fs = rc.filesystem(name)

        try:
            dr = fs['dir']
            if not os.path.exists(dr):
                prt("Making directory: {}".format(dr))
                os.makedirs(dr)
        except KeyError:
            pass
Example #11
0
class Test(TestBase):
 
    def setUp(self):
        
        rm_rf('/tmp/server')

        self.copy_or_build_bundle()
        self.bundle_dir =  os.path.join(os.path.dirname(os.path.abspath(__file__)),'testbundle')    
        self.rc = RunConfig([os.path.join(self.bundle_dir,'client-test-config.yaml'),
                             os.path.join(self.bundle_dir,'bundle.yaml'),
                             RunConfig.USER_ACCOUNTS])
         
        self.server_rc = RunConfig([os.path.join(self.bundle_dir,'server-test-config.yaml'),
                                    RunConfig.USER_ACCOUNTS])
       
        self.bundle = Bundle()  
        self.bundle_dir = self.bundle.bundle_dir

    def tearDown(self):
        from ambry.library import clear_libraries

        # new_library() caches the library, and the test runner uses different threads
        # which together with the sqlite driver's inability to use threads, can cause some problems.

        clear_libraries()

        self.stop_server()

    def get_library(self, name = 'default'):
        """Return the same library that the server uses. """
        from ambry.library import new_library

        config = self.server_rc.library(name)

        l =  new_library(config, reset = True)

        return l

    def web_exists(self,s3, rel_path):
    
        import requests
        import urlparse
        
        url  = s3.path(rel_path, method='HEAD')
      
        parts = list(urlparse.urlparse(url))
        qs = urlparse.parse_qs(parts[4])
        parts[4] = None
        
        url = urlparse.urlunparse(parts)
        r = requests.head(urlparse.urlunparse(parts), params=qs)

        self.assertEquals(200, r.status_code)
    
        return True

    def test_connection(self):
        '''
        Test some of the server's test functions
        :return:
        '''
        from ambry.client.rest import RemoteLibrary

        self.start_server()

        a = RemoteLibrary(self.server_url)

        self.assertEquals('foobar', a.get_test_echo('foobar'))
        self.assertEquals('foobar', a.put_test_echo('foobar'))

        with self.assertRaises(Exception):
            a.get_test_exception()

        r = a.get_root()

        self.assertEquals('devtest.sandiegodata.org', r['upstream']['bucket'])
        self.assertEquals('library-test', r['upstream']['prefix'])

    def test_resolve(self):
        from ambry.client.rest import RemoteLibrary

        l = self.get_library()
        l.purge()
        print l.info
        #
        # Check that the library can list datasets that are inserted externally
        #

        r = l.put(self.bundle)

        ident = self.bundle.identity

        # Local Library
        self.assertEquals(ident.vid, l.resolve(ident.vid).vid)
        self.assertEquals(ident.vid, l.resolve(ident.vname).vid)
        self.assertEquals(ident.vid, l.resolve(ident.cache_key).vid)
        self.assertEquals(ident.vid, l.resolve(ident.sname).vid)

        for p in self.bundle.partitions:
            print '--', p.identity.cache_key
            dsid = l.resolve(p.identity.vid)
            self.assertEquals(ident.vid, dsid.vid)
            self.assertEquals(p.identity.vid, dsid.partition.vid)

            dsid = l.resolve(p.identity.cache_key)

            if not dsid:
                ck = p.identity.cache_key
                l.resolve(ck)

            self.assertIsNotNone(dsid)
            self.assertEquals(ident.vid, dsid.vid)
            self.assertEquals(p.identity.vid, dsid.partition.vid)

        # Remote Library

        self.start_server()

        rl = RemoteLibrary(self.server_url)

        self.assertEquals(ident.vid, rl.resolve(ident.vid).vid)
        self.assertEquals(ident.vid, rl.resolve(ident.vname).vid)
        self.assertEquals(ident.vid, rl.resolve(ident.cache_key).vid)
        self.assertEquals(ident.vid, (rl.resolve(ident.sname).vid))

        for p in self.bundle.partitions:
            print '--',p.identity.cache_key
            dsid = rl.resolve(p.identity.vid)
            self.assertEquals(ident.vid, dsid.vid)
            self.assertEquals(p.identity.vid, dsid.partition.vid)

            dsid = rl.resolve(p.identity.cache_key)
            self.assertEquals(ident.vid, dsid.vid)
            self.assertEquals(p.identity.vid, dsid.partition.vid)

        print rl.resolve('source/dataset-subset-variation-0.0.1/geot1.geodb')

    def test_load(self):

        from ambry.run import  get_runconfig, RunConfig
        from ambry.client.rest import RemoteLibrary
        from ambry.cache import new_cache
        from ambry.util import md5_for_file
        from ambry.identity import Identity

        config = self.start_server()
        l = new_library(config)

        rl = RemoteLibrary(self.server_url)


        #
        # Check that the library can list datasets that are inserted externally
        #

        r = l.put(self.bundle)

        s = set([i.fqname for i in rl.list().values()])

        self.assertIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001', s)

        dsident = rl.dataset('diEGPXmDC8001')

        s = set([i.fqname for i in dsident.partitions.values()])

        self.assertEquals(11, len(s))

        self.assertIn('source-dataset-subset-variation-hdf5-hdf-0.0.1~piEGPXmDC800f001', s)
        self.assertIn('source-dataset-subset-variation-tthree-3-0.0.1~piEGPXmDC800d001', s)
        self.assertIn('source-dataset-subset-variation-geot1-geo-0.0.1~piEGPXmDC8002001', s)

        #
        # Upload the dataset to S3, clear the library, then load it back in
        #

        rc = get_runconfig((os.path.join(self.bundle_dir,'test-run-config.yaml'),RunConfig.USER_ACCOUNTS))
        cache = new_cache(rc.filesystem('cached-compressed-s3'))

        fn = self.bundle.database.path
        identity = self.bundle.identity
        relpath = identity.cache_key

        r = cache.put(fn, relpath, identity.to_meta(file=fn))


        self.assertTrue(bool(cache.has(relpath)))

        # clear the library.

        l.purge()
        self.assertNotIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',
                         set([i.fqname for i in rl.list()]))

        # Load from  S3, directly in to the local library

        identity.add_md5(md5_for_file(fn))

        l.load(identity.cache_key, identity.md5)

        self.assertIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',
                      set([i.fqname for i in rl.list().values()]))

        # Do it one more time, using the remote library

        l.purge()
        self.assertNotIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',
                         set([i.fqname for i in rl.list().values()]))

        rl.load_dataset(identity)

        self.assertIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',
                      set([i.fqname for i in rl.list().values()]))

        # Check that we can get the record from the library

        self.assertEquals(identity.vid, rl.resolve(identity.vid).vid)
        self.assertEquals(identity.vid, rl.resolve(identity.vname).vid)
        self.assertEquals(identity.vid, rl.resolve(identity.cache_key).vid)
        self.assertEquals(identity.vid, rl.resolve(identity.sname).vid)


    def test_push(self):
        from ambry.identity import Identity
        from functools import partial

        config = self.server_library_config()

        # Create the library so we can get the same remote config
        l = new_library(config)
        s3 = l.upstream.last_upstream()

        print l.info

        db = l.database
        db.enable_delete = True
        try:
            db.drop()
            db.create()
        except:
            pass

        s3 = l.upstream.last_upstream()
        s3.clean()

        l.put_bundle(self.bundle)

        def push_cb(expect, action, metadata, time):
            import json

            self.assertIn(action, expect)

            identity = Identity.from_dict(json.loads(metadata['identity']))
            print action, identity.cache_key

        def throw_cb(action, metadata, time):
            raise Exception("Push shonld not run")

        l.push(cb=partial(push_cb, ('Pushed', 'Pushing')))

        # ALl should be pushed, so should not run
        l.push(cb=throw_cb)

        # Resetting library, but not s3, should already have all
        # records
        db = l.database
        db.enable_delete = True
        db.drop()
        db.create()
        l.put_bundle(self.bundle)

        l.push(cb=partial(push_cb, ('Has')))

        self.web_exists(s3, self.bundle.identity.cache_key)

        for p in self.bundle.partitions:
            self.web_exists(s3, p.identity.cache_key)

        l.sync_upstream()



    def test_caches(self):
        '''Basic test of put(), get() and has() for all cache types'''
        from functools import partial
        from ambry.run import  get_runconfig, RunConfig
        from ambry.filesystem import Filesystem
        from ambry.cache import new_cache
        from ambry.util import md5_for_file
        from ambry.bundle import DbBundle

        #self.start_server() # For the rest-cache

        #fn = '/tmp/1mbfile'
        #with open(fn, 'wb') as f:
        #    f.write('.'*(1024))

        fn = self.bundle.database.path

        # Opening the file might run the database updates in
        # database.sqlite._on_connect_update_schema, which can affect the md5.
        b = DbBundle(fn)

        md5 = md5_for_file(fn)

        print "MD5 {}  = {}".format(fn, md5)

        rc = get_runconfig((os.path.join(self.bundle_dir,'test-run-config.yaml'),
                            RunConfig.USER_ACCOUNTS))

        for i, fsname in enumerate(['fscache', 'limitedcache', 'compressioncache',
                                    'cached-s3', 'cached-compressed-s3']):

            config = rc.filesystem(fsname)
            cache = new_cache(config)
            print '---', fsname, cache
            identity = self.bundle.identity

            relpath = identity.cache_key

            r = cache.put(fn, relpath,identity.to_meta(md5=md5))

            r = cache.get(relpath)

            if not r.startswith('http'):
                self.assertTrue(os.path.exists(r), 'Not a url: {}: {}'.format(r,str(cache)))

            self.assertTrue(cache.has(relpath, md5=md5))

            cache.remove(relpath, propagate=True)

            self.assertFalse(os.path.exists(r), str(cache))
            self.assertFalse(cache.has(relpath))


        cache = new_cache(rc.filesystem('s3cache-noupstream'))
        r = cache.put(fn, 'a')


    def test_simple_install(self):
        from ambry.client.rest import RemoteLibrary
        from ambry.cache.remote import RestReadCache
        
        config = self.start_server()

        # Create the library so we can get the same remote config
        l = new_library(config)


        s3 = l.upstream.last_upstream()

        s3.clean()

        print "S3 cache ", str(s3)

        if not s3.has(self.bundle.identity.cache_key):
            print 'Uploading: ', self.bundle.identity.cache_key
            s3.put(self.bundle.database.path,self.bundle.identity.cache_key)
            self.web_exists(s3,self.bundle.identity.cache_key)

        for p in self.bundle.partitions:
            if not s3.has(p.identity.cache_key):
                print 'Uploading: ', p.identity.cache_key
                s3.put(p.database.path,p.identity.cache_key)
                self.web_exists(s3,p.identity.cache_key)
            else:
                print 'Has      : ', p.identity.cache_key

        #
        # Kick the remote library to load the dataset
        #
        rl = RemoteLibrary(self.server_url)
        ident = self.bundle.identity
        ident.add_md5(file=self.bundle.database.path)
        rl.load_dataset(ident)
        self.assertIn('source-dataset-subset-variation-0.0.1~diEGPXmDC8001',
                      set([i.fqname for i in rl.list().values()]))


        return

        # Try variants of find. 
        r = api.find(self.bundle.identity.name)
        self.assertEquals(self.bundle.identity.name, r[0].name)
        
        r = api.find(QueryCommand().identity(name = self.bundle.identity.name))
        self.assertEquals(self.bundle.identity.name, r[0].name)

        for partition in self.bundle.partitions:
            r = api.find((QueryCommand().partition(name = partition.identity.name)).to_dict())
            self.assertEquals(partition.identity.name, r[0].name)
    def test_joint_resolve(self):
        '''Test resolving from either a remote or local library, from the local interface '''

        from ambry.identity import Identity
        from ambry.client.rest import RemoteLibrary
        from ambry.library.query import RemoteResolver

        self.start_server()

        config = self.server_library_config()

        # Create the library so we can get the same remote config
        l = new_library(config)

        s3 = l.upstream.last_upstream()
        print l.info
        db = l.database
        db.enable_delete = True
        db.clean()

        l.put_bundle(self.bundle)

        # This might not do anything if the files already are in s3
        def push_cb(action, metadata, time):
            print action, metadata['fqname']

        l.push(cb=push_cb)

        # Check they are on the web
        self.web_exists(s3,self.bundle.identity.cache_key)
        for p in self.bundle.partitions:
            self.web_exists(s3,p.identity.cache_key)

        # Check the basic resolvers
        ident = self.bundle.identity
        self.assertEquals(ident.vid, l.resolve(ident.vid).vid)

        rl = RemoteLibrary(self.server_url)
        self.assertEquals(ident.vid, rl.resolve(ident.vname).vid)

        # That's the basics, now test the primary use case with the remote resolver.

        # Remote resolver only
        rr = RemoteResolver(local_resolver=None, remote_urls=[self.server_url])
        self.assertEquals(ident.vid, rr.resolve_ref_one(ident.vid)[1].vid)
        self.assertEquals('http://localhost:7979', rr.resolve_ref_one(ident.vid)[1].url)

        # Local Resolver only
        rr = RemoteResolver(local_resolver=l.database.resolver, remote_urls=None)
        self.assertEquals(ident.vid, rr.resolve_ref_one(ident.vid)[1].vid)
        self.assertIsNone(rr.resolve_ref_one(ident.vid)[1].url)

        self.stop_server()

        # Remote resolver only
        rr = RemoteResolver(local_resolver=None, remote_urls=[self.server_url])
        self.assertIsNone(rr.resolve_ref_one(ident.vid)[1])

        # Combined
        rr = RemoteResolver(local_resolver=l.database.resolver, remote_urls=[self.server_url])
        self.assertEquals(ident.vid, rr.resolve_ref_one(ident.vid)[1].vid)
        self.assertIsNone(rr.resolve_ref_one(ident.vid)[1].url)

    def test_files(self):
        '''
        Test some of the server's file functions
        :return:
        '''

        from ambry.cache import new_cache
        from ambry.bundle import DbBundle

        fs = new_cache(self.server_rc.filesystem('rrc-fs'))
        fs.clean()
        remote = new_cache(self.server_rc.filesystem('rrc'))

        config = self.start_server()

        l = new_library(config)

        l.put_bundle(self.bundle)
        l.push()

        ident = self.bundle.identity
        ck = ident.cache_key

        # The remote is tied to the REST server, so it has the
        # bundle, but the new filesystem cache does not.

        self.assertFalse(fs.has(ck))
        self.assertTrue(remote.has(ck))

        # But if we tie them together, the FS cache should have it

        fs.upstream = remote
        self.assertTrue(fs.has(ck))

        path = fs.get(ck)

        b = DbBundle(path)
        self.assertEquals(ck, b.identity.cache_key)

        # It should have been copied, so the fs should still have
        # it after disconnecting.

        fs.upstream = None
        self.assertTrue(fs.has(ck))

    def test_library_get(self):
        from ambry.library import new_library

        vid = self.bundle.identity.vid

        config = self.server_library_config()

        # Create the library so we can get the same remote config
        server_l = new_library(config)

        server_l.put_bundle(self.bundle)

        # Local only; no connection to server
        local_l  = new_library(self.server_rc.library("local"))

        # A library that connects to the server
        remote_l = new_library(self.server_rc.library("reader"))
        remote_l.purge()

        self.assertTrue(len(remote_l.list()) == 0)

        self.assertEquals(vid, server_l.resolve(vid).vid)
        self.assertIsNone(local_l.resolve(vid))
        self.assertIsNone(remote_l.resolve(vid))

        self.start_server()

        self.assertEquals(vid, remote_l.resolve(vid).vid)

        b = remote_l.get(vid)

        print b.identity.fqname

        for p in self.bundle.partitions:
            b = remote_l.get(p.identity.vid)
            self.assertTrue(p.identity.fqname, b.partition.identity.fqname)

        self.assertEqual(1, len(remote_l.list()))

        # Test out syncing.


        remote_l.sync_remotes()


    # =======================

    def x_test_remote_library(self):
   
        # This test does not work with the threaded test server. 
        
        # It does work with an external server, but you have to delete 
        # All of the files on the remote library between runs. 
   
        #
        # First store the files in the local library
        #
        
        self.start_server()
        
        self.get_library('server').purge()
        self.get_library('clean').purge()

        l = self.get_library()
     
        r = l.put(self.bundle)

        r = l.get(self.bundle.identity.name)
        self.assertEquals(self.bundle.identity.name, r.identity.name)

        for partition in self.bundle.partitions:
            r = l.put(partition)

            # Get the partition with a name
            r = l.get(partition.identity.name)
            self.assertTrue(bool(r))
            self.assertEquals(partition.identity.name, r.partition.identity.name)
            self.assertEquals(self.bundle.identity.name, r.identity.name)
            
            # Get the partition with an id
            r = l.get(partition.identity.id_)
            self.assertTrue(bool(r))
            self.assertEquals(partition.identity.name, r.partition.identity.name)
            self.assertEquals(self.bundle.identity.name, r.identity.name)            

        #
        # Now start with a different, clean library with the same remote
        #

        # haven't pushed yet, so should fail. 
        l2 = self.get_library('clean')
        b = l2.get(self.bundle.identity.name)
        self.assertTrue(not b)
        
        # Copy all of the newly added files to the server. 
        l.push()
   
        l2 = self.get_library('clean')

        r = l2.get(self.bundle.identity.name)

        self.assertTrue(bool(r))

        r = l2.get(r.partitions.all[0].identity.id_)

        self.assertTrue(bool(r))
        self.assertTrue(os.path.exists(r.partition.database.path))

    def x_test_remote_library_partitions(self):

        self.start_server()

        l = self.get_library()
     
        r = l.put(self.bundle)

        r = l.get(self.bundle.identity.name)
        self.assertEquals(self.bundle.identity.name, r.identity.name)

        for partition in self.bundle.partitions:
            r = l.put(partition)

            # Get the partition with a name
            r = l.get(partition.identity.name)
            self.assertTrue(r is not False)
            self.assertEquals(partition.identity.name, r.partition.identity.name)
            self.assertEquals(self.bundle.identity.name, r.identity.name)

        # Copy all of the newly added files to the server. 
        l.push()
            
        l2 = new_library('clean')
        l2.purge()
        
        r = l2.get('b1DxuZ001')
     
        self.assertTrue(r is not None and r is not False)
        
        print r
        
        self.assertTrue(r.partition is not None and r.partition is not False)
        self.assertEquals(r.partition.identity.id_,'b1DxuZ001' )
        
        self.assertTrue(os.path.exists(r.partition.database.path))
   
    def x_test_test(self):
        from ambry.client.siesta import  API
        
        self.start_server()
        
        a = API(self.server_url)
        
        # Test echo for get. 
        r = a.test.echo('foobar').get(bar='baz')
        
        self.assertEquals(200,r.status)
        self.assertIsNone(r.exception)
        
        self.assertEquals('foobar',r.object[0])
        self.assertEquals('baz',r.object[1]['bar'])
        
        # Test echo for put. 
        r = a.test.echo().put(['foobar'],bar='baz')
        
        self.assertEquals(200,r.status)
        self.assertIsNone(r.exception)

        self.assertEquals('foobar',r.object[0][0])
        self.assertEquals('baz',r.object[1]['bar'])
      
        
        with self.assertRaises(Exception):
            r = a.test.exception.put('foo')
        
        with self.assertRaises(Exception):
            r = a.test.exception.get()

    def _test_put_bundle(self, name, remote_config=None):
        from ambry.bundle import DbBundle
        from ambry.library.query import QueryCommand
        
        rm_rf('/tmp/server')
        
        self.start_server(remote_config)
        
        r = None #Rest(self.server_url, remote_config)
        
        bf = self.bundle.database.path

        # With an FLO
        response =  r.put(open(bf), self.bundle.identity)
        self.assertEquals(self.bundle.identity.id_, response.object.get('id'))
      
        # with a path
        response =  r.put( bf, self.bundle.identity)
        self.assertEquals(self.bundle.identity.id_, response.object.get('id'))

        for p in self.bundle.partitions.all:
            response =  r.put( open(p.database.path), p.identity)
            self.assertEquals(p.identity.id_, response.object.get('id'))

        # Now get the bundles
        bundle_file = r.get(self.bundle.identity,'/tmp/foo.db')
        bundle = DbBundle(bundle_file)

        self.assertIsNot(bundle, None)
        self.assertEquals('a1DxuZ',bundle.identity.id_)

        # Should show up in datasets list. 
        
        o = r.list()
   
        self.assertTrue('a1DxuZ' in o.keys() )
    
        o = r.find(QueryCommand().table(name='tone').partition(any=True))
      
        self.assertTrue( 'b1DxuZ001' in [i.id_ for i in o])
        self.assertTrue( 'a1DxuZ' in [i.as_dataset.id_ for i in o])

    def x_test_put_bundle_noremote(self):
        return self._test_put_bundle('default')

    def x_test_put_bundle_remote(self):
        return self._test_put_bundle('default-remote', self.rc.accounts)

    def x_test_remote_cache(self):
        self.start_server(name='default-remote')
    
    def x_test_put_redirect(self):
        from ambry.bundle import DbBundle
        from ambry.library.query import QueryCommand
        from ambry.util import md5_for_file, rm_rf, bundle_file_type

        #
        # Simple out and retrieve
        # 
        cache = self.bundle.filesystem._get_cache(self.server_rc.filesystem, 'direct-remote')
        cache2 = self.bundle.filesystem._get_cache(self.server_rc.filesystem, 'direct-remote-2')

        rm_rf(os.path.dirname(cache.cache_dir))
        rm_rf(os.path.dirname(cache2.cache_dir))
        
        cache.put( self.bundle.database.path, 'direct')

        path = cache2.get('direct')

        self.assertEquals('sqlite',bundle_file_type(path))

        cache.remove('direct', propagate = True)

        #
        #  Connect through server. 
        #
        rm_rf('/tmp/server')
        self.start_server(name='default-remote')
        
        api = None # Rest(self.server_url, self.rc.accounts)  

        # Upload directly, then download via the cache. 
        
        cache.remove(self.bundle.identity.cache_key, propagate = True)
        
        r = api.upload_file(self.bundle.identity, self.bundle.database.path, force=True )

        path = cache.get(self.bundle.identity.cache_key)
        
        b = DbBundle(path)

        self.assertEquals("source-dataset-subset-variation-ca0d",b.identity.name )
      
        #
        # Full service
        #

        p  = self.bundle.partitions.all[0]

        cache.remove(self.bundle.identity.cache_key, propagate = True)
        cache.remove(p.identity.cache_key, propagate = True)
        
        r = api.put( self.bundle.database.path, self.bundle.identity )
        print "Put {}".format(r.object)
        r = api.put(p.database.path, p.identity )
        print "Put {}".format(r.object)
        
        r = api.put(p.database.path, p.identity )
        
        r = api.get(p.identity,'/tmp/foo.db')
        print "Get {}".format(r)        

        b = DbBundle(r)

        self.assertEquals("source-dataset-subset-variation-ca0d",b.identity.name )

    def x_test_dump(self):
        import time
        import logging 
     
       
        l = new_library(self.server_rc.library('default-remote'), reset = True)
        l.clean()

        self.start_server()
        
        l.run_dumper_thread()
        l.run_dumper_thread()
       
        self.assertFalse(l.database.needs_dump())
        l.put(self.bundle)
        self.assertTrue(l.database.needs_dump()) 
        l.run_dumper_thread()
        time.sleep(6)
        self.assertFalse(l.database.needs_dump())
            
        l.run_dumper_thread()
        l.put(self.bundle)
        l.run_dumper_thread()
        time.sleep(7)
        print l.database.needs_dump()
        self.assertFalse(l.database.needs_dump())
        
        self.assertEquals(self.bundle.identity.name,  l.get(self.bundle.identity.name).identity.name)
        
        l.clean()
        
        self.assertEqual(None, l.get(self.bundle.identity.name))
        
        l.restore()
        
        self.assertEquals(self.bundle.identity.name,  l.get(self.bundle.identity.name).identity.name)
Example #12
0
def source_build(args, l, st, rc):
    """Build a single bundle, or a set of bundles in a directory.

    The build process will build all dependencies for each bundle before
    buildng the bundle.

    """

    from ambry.identity import Identity
    from ..source.repository import new_repository

    repo = new_repository(rc.sourcerepo(args.name))

    dir_ = None
    name = None

    if args.dir:
        if os.path.exists(args.dir):
            dir_ = args.dir
            name = None
        else:
            name = args.dir
            try:
                Identity.parse_name(name)
            except:
                fatal(
                    "Argument '{}' must be either a bundle name or a directory"
                    .format(name))
                return

    if not dir_:
        dir_ = rc.sourcerepo.dir

    def build(bundle_dir):
        from ambry.library import new_library

        # Import the bundle file from the directory

        bundle_class = load_bundle(bundle_dir)
        bundle = bundle_class(bundle_dir)

        l = new_library(rc.library(args.library_name))

        if l.get(bundle.identity.vid) and not args.force:
            prt("{} Bundle is already in library", bundle.identity.name)
            return
        elif bundle.is_built and not args.force and not args.clean:
            prt("{} Bundle is already built", bundle.identity.name)
            return
        else:

            if args.dryrun:
                prt("{} Would build but in dry run ", bundle.identity.name)
                return

            repo.bundle = bundle

            if args.clean:
                bundle.clean()

            # Re-create after cleaning is important for something ...

            bundle = bundle_class(bundle_dir)

            prt("{} Building ", bundle.identity.name)

            if not bundle.run_prepare():
                fatal("{} Prepare failed", bundle.identity.name)

            if not bundle.run_build():
                fatal("{} Build failed", bundle.identity.name)

        if args.install and not args.dryrun:
            if not bundle.run_install(force=True):
                fatal('{} Install failed', bundle.identity.name)

    build_dirs = {}

    # Find all of the dependencies for the named bundle, and make those first.
    for root, _, files in os.walk(rc.sourcerepo.dir):
        if 'bundle.yaml' in files:
            bundle_class = load_bundle(root)
            bundle = bundle_class(root)
            build_dirs[bundle.identity.name] = root

    if name:
        deps = repo.bundle_deps(name)
        deps.append(name)

    else:

        deps = []

        # Walk the subdirectory for the files to build, and
        # add all of their dependencies
        for root, _, files in os.walk(dir_):
            if 'bundle.yaml' in files:

                bundle_class = load_bundle(root)
                bundle = bundle_class(root)

                for dep in repo.bundle_deps(bundle.identity.name):
                    if dep not in deps:
                        deps.append(dep)

                deps.append(bundle.identity.name)

    for n in deps:
        try:
            dir_ = build_dirs[n]
        except KeyError:
            fatal("Failed to find directory for bundle {}".format(n))

        prt('')
        prt("{} Building in {}".format(n, dir_))
        build(dir_)