Example #1
0
    def x_test_rewrite(self):
        from  testbundle.bundle import Bundle
        from sqlalchemy.exc import IntegrityError
        import json
        from ambry.run import get_runconfig

        # Prepare to rewrite the bundle.yaml file.
        bundle = Bundle()
        orig = os.path.join(bundle.bundle_dir,'bundle.yaml')
        save = os.path.join(bundle.bundle_dir,'bundle.yaml.save')

        try:
            os.rename(orig,save)

            print 'Write to ', orig
            with open(orig,'w') as f:
                f.write(json.dumps(
                    {
                        "identity":{
                            "dataset": "dataset1",
                            "id": "dfoo",
                            "revision": 100,
                            "source": "source1",
                            "subset": "subset1",
                            "variation": "variation1",
                            "version": "1.0.1",
                            "vid": "dfob001",
                        },
                        "about": {
                            "author": "*****@*****.**"
                        }
                    }
                ))

            get_runconfig.clear() # clear config cache.
            bundle = Bundle()
            bundle.clean()
            bundle.pre_prepare()
            bundle.prepare()
            bundle.post_prepare() # Does the rewrite, adding the 'names'

            # Need to clear and reload one more time for the 'names' to appear
            get_runconfig.clear() # clear config cache.
            bundle = Bundle()
            bundle.exit_on_fatal = False

            self.assertEquals('dataset1', bundle.config.identity.dataset)
            self.assertEquals('dfoo', bundle.config.identity.id)
            self.assertEquals(100, bundle.config.identity.revision)

            self.assertEquals("source1-dataset1-subset1-variation1-1.0.100~dfoo01C", bundle.config.names.fqname)

            self.assertEquals("*****@*****.**", bundle.config.about.author)

        finally:
            os.rename(save, orig)
            self.delete_bundle()
Example #2
0
    def test_versions(self):

        idnt = self.bundle.identity

        l = self.get_library()

        orig = os.path.join(self.bundle.bundle_dir, 'bundle.yaml')
        save = os.path.join(self.bundle.bundle_dir, 'bundle.yaml.save')
        shutil.copyfile(orig, save)

        datasets = {}

        try:
            for i in [1, 2, 3]:
                idnt._on.revision = i
                idnt.name.version_major = i
                idnt.name.version_minor = i * 10

                bundle = Bundle()
                get_runconfig.clear()  # clear runconfig cache

                bundle.metadata.load_all()

                bundle.metadata.identity = idnt.ident_dict
                bundle.metadata.names = idnt.names_dict

                bundle.metadata.write_to_dir(write_all=True)

                bundle = Bundle()

                bundle.clean()
                bundle.pre_prepare()
                bundle.prepare()
                bundle.post_prepare()
                bundle.pre_build()

                bundle.build_small()
                # bundle.build()
                bundle.post_build()

                bundle = Bundle()

                l.put_bundle(bundle)

        finally:
            os.rename(save, orig)

        #
        # Save the list of datasets for version analysis in other
        # tests
        #

        db = l.database

        for d in db.list(with_partitions=True).values():
            datasets[d.vid] = d.dict
            datasets[d.vid]['partitions'] = {}

            for p_vid, p in d.partitions.items():
                datasets[d.vid]['partitions'][p_vid] = p.dict

        with open(self.bundle.filesystem.path('meta', 'version_datasets.json'), 'w') as f:
            f.write(json.dumps(datasets))

        r = Resolver(db.session)

        # ref = idnt.id_

        ref = 'source-dataset-subset-variation-=2.20'

        ip, results = r.resolve_ref_all(ref)
Example #3
0
    def test_versions(self):
        import testbundle.bundle
        from ambry.run import get_runconfig
        from ambry.library.query import Resolver
        import shutil
        idnt = self.bundle.identity

        l = self.get_library()

        l.purge()

        orig = os.path.join(self.bundle.bundle_dir,'bundle.yaml')
        save = os.path.join(self.bundle.bundle_dir,'bundle.yaml.save')
        shutil.copyfile(orig,save)

        datasets = {}

        try:
            for i in [1,2,3]:
                idnt._on.revision = i
                idnt.name.version_major = i
                idnt.name.version_minor = i*10

                bundle = Bundle()
                bundle.config.rewrite(identity=idnt.ident_dict,
                                      names=idnt.names_dict)
                get_runconfig.clear() #clear runconfig cache

                print 'Building version {}'.format(i)

                bundle = Bundle()

                bundle.clean()
                bundle.pre_prepare()
                bundle.prepare()
                bundle.post_prepare()
                bundle.pre_build()
                bundle.build_small()
                #bundle.build()
                bundle.post_build()

                bundle = Bundle()

                print "Installing ", bundle.identity.vname
                l.put(bundle)

        finally:
            pass
            os.rename(save, orig)

        #
        # Save the list of datasets for version analysis in other
        # tests
        #

        db = l.database

        for d in db.list().values():
            datasets[d.vid] = d.dict
            datasets[d.vid]['partitions'] = {}

            for p_vid, p in d.partitions.items():
                datasets[d.vid]['partitions'][p_vid] = p.dict


        with open(self.bundle.filesystem.path('meta','version_datasets.json'),'w') as f:
            import json
            f.write(json.dumps(datasets))


        r = Resolver(db.session)

        ref = idnt.id_

        ref = "source-dataset-subset-variation-=2.20"

        ip, results = r.resolve_ref_all(ref)

        for row in results:
            print row
Example #4
0
    def test_versions(self):
        from ambry.run import get_runconfig
        from ambry.library.query import Resolver
        import shutil
        idnt = self.bundle.identity

        l = self.get_library()

        l.purge()

        orig = os.path.join(self.bundle.bundle_dir, 'bundle.yaml')
        save = os.path.join(self.bundle.bundle_dir, 'bundle.yaml.save')
        shutil.copyfile(orig, save)

        datasets = {}

        try:
            for i in [1, 2, 3]:
                idnt._on.revision = i
                idnt.name.version_major = i
                idnt.name.version_minor = i * 10

                bundle = Bundle()
                get_runconfig.clear()  #clear runconfig cache

                bundle.metadata.load_all()

                bundle.metadata.identity = idnt.ident_dict
                bundle.metadata.names = idnt.names_dict

                bundle.metadata.write_to_dir(write_all=True)

                bundle = Bundle()

                bundle.clean()
                bundle.pre_prepare()
                bundle.prepare()
                bundle.post_prepare()
                bundle.pre_build()
                bundle.build_small()
                #bundle.build()
                bundle.post_build()

                bundle = Bundle()

                l.put_bundle(bundle)

        finally:
            pass
            os.rename(save, orig)

        #
        # Save the list of datasets for version analysis in other
        # tests
        #

        db = l.database

        for d in db.list(with_partitions=True).values():
            datasets[d.vid] = d.dict
            datasets[d.vid]['partitions'] = {}

            for p_vid, p in d.partitions.items():
                datasets[d.vid]['partitions'][p_vid] = p.dict

        with open(self.bundle.filesystem.path('meta', 'version_datasets.json'),
                  'w') as f:
            import json
            f.write(json.dumps(datasets))

        r = Resolver(db.session)

        ref = idnt.id_

        ref = "source-dataset-subset-variation-=2.20"

        ip, results = r.resolve_ref_all(ref)

        for row in results:
            print row