Esempio n. 1
0
    def do_run(self, args):
        from api_etl import Config

        self.config = Config()

        parts = args.split('.')
        services = []
        service_entries = []

        manifest_filename = "{}.yml".format(parts[0])
        manifest = Manifest(manifest_filename)

        if len(parts) == 1:
            services = svcs.service(parts[0])
            service_entries = manifest.get_services()
        else:
            services = [svcs.named_subservice(*parts)]
            service_entries = [manifest.get_service(parts[1])]

        # The ordering of entries in the manifest and entrypoints is
        # important, they must match otherwise this zip call will
        # pair up the wrong things.
        entry = zip(services, service_entries)
        for e in entry:
            ep, sm = e
            self.run_etl(parts[0], ep, sm)
Esempio n. 2
0
    def do_run(self, args):
        from api_etl import Config

        self.config = Config()

        parts = args.split('.')
        services = []
        service_entries = []

        manifest_filename = "{}.yml".format(parts[0])
        manifest = Manifest(manifest_filename)

        if len(parts) == 1:
            services = svcs.service(parts[0])
            service_entries = manifest.get_services()
        else:
            services = [svcs.named_subservice(*parts)]
            service_entries = [manifest.get_service(parts[1])]

        # The ordering of entries in the manifest and entrypoints is
        # important, they must match otherwise this zip call will
        # pair up the wrong things.
        entry = zip(services, service_entries)
        for e in entry:
            ep, sm = e
            self.run_etl(parts[0], ep, sm)
Esempio n. 3
0
    def do_manifest_convert(self, args):
        from api_etl import Config
        self.config = Config()
        owner = self.config.database('owner')
        self.conn = psycopg2.connect("dbname=apiserver user={}".format(owner))

        self.lookup = {
            "character varying": "string",
            "text": "string"
        }

        root = self.config.config.get('manifest', 'location')
        for f in glob.glob(os.path.join(root, '*.yml')):
            m = Manifest(f)
            theme = m.data['title'].lower()

            for service in m.get_services():
                out = {
                    "theme": theme,
                    "id": service.name,
                    "title": service.description,
                    "tablename": service.name,
                }

                q = """
                  SELECT column_name, data_type
                  FROM information_schema.columns
                  WHERE table_schema = 'public'
                    AND table_name   = '{table}';
                """.format(table=service.name)

                cur = self.conn.cursor()
                cur.execute(q)
                result = cur.fetchall()
                cur.close()

                out['fields'] = []
                for (n,t) in result:
                    d = {"name": n, "type": self.lookup.get(t, t)}
                    out['fields'].append(d)

                if isinstance(service.table_settings, list):
                    settings = service.table_settings[0]
                else:
                    settings = service.table_settings
                out['choice_fields'] = settings.get('choice_fields', [])

                if hasattr(service, "searchables"):
                    out['queries'] = service.searchables

                outfile = os.path.join(root, "manifests", service.name + ".json")
                print outfile
                with open(outfile, "w") as fout:
                    json.dump(out, fout, indent=2)

        self.conn.close()
Esempio n. 4
0
    def do_manifest_convert(self, args):
        from api_etl import Config
        self.config = Config()
        owner = self.config.database('owner')
        self.conn = psycopg2.connect("dbname=apiserver user={}".format(owner))

        self.lookup = {"character varying": "string", "text": "string"}

        root = self.config.config.get('manifest', 'location')
        for f in glob.glob(os.path.join(root, '*.yml')):
            m = Manifest(f)
            theme = m.data['title'].lower()

            for service in m.get_services():
                out = {
                    "theme": theme,
                    "id": service.name,
                    "title": service.description,
                    "tablename": service.name,
                }

                q = """
                  SELECT column_name, data_type
                  FROM information_schema.columns
                  WHERE table_schema = 'public'
                    AND table_name   = '{table}';
                """.format(table=service.name)

                cur = self.conn.cursor()
                cur.execute(q)
                result = cur.fetchall()
                cur.close()

                out['fields'] = []
                for (n, t) in result:
                    d = {"name": n, "type": self.lookup.get(t, t)}
                    out['fields'].append(d)

                if isinstance(service.table_settings, list):
                    settings = service.table_settings[0]
                else:
                    settings = service.table_settings
                out['choice_fields'] = settings.get('choice_fields', [])

                if hasattr(service, "searchables"):
                    out['queries'] = service.searchables

                outfile = os.path.join(root, "manifests",
                                       service.name + ".json")
                print outfile
                with open(outfile, "w") as fout:
                    json.dump(out, fout, indent=2)

        self.conn.close()