def do_run(self, args):
from api_etl import Config
self.config = Config()
parts = args.split('.')
services = []
service_entries = []
manifest_filename = "{}.yml".format(parts[0])
manifest = Manifest(manifest_filename)
if len(parts) == 1:
services = svcs.service(parts[0])
service_entries = manifest.get_services()
else:
services = [svcs.named_subservice(*parts)]
service_entries = [manifest.get_service(parts[1])]
# The ordering of entries in the manifest and entrypoints is
# important, they must match otherwise this zip call will
# pair up the wrong things.
entry = zip(services, service_entries)
for e in entry:
ep, sm = e
self.run_etl(parts[0], ep, sm)
def do_run(self, args):
from api_etl import Config
self.config = Config()
parts = args.split('.')
services = []
service_entries = []
manifest_filename = "{}.yml".format(parts[0])
manifest = Manifest(manifest_filename)
if len(parts) == 1:
services = svcs.service(parts[0])
service_entries = manifest.get_services()
else:
services = [svcs.named_subservice(*parts)]
service_entries = [manifest.get_service(parts[1])]
# The ordering of entries in the manifest and entrypoints is
# important, they must match otherwise this zip call will
# pair up the wrong things.
entry = zip(services, service_entries)
for e in entry:
ep, sm = e
self.run_etl(parts[0], ep, sm)
def do_manifest_convert(self, args):
from api_etl import Config
self.config = Config()
owner = self.config.database('owner')
self.conn = psycopg2.connect("dbname=apiserver user={}".format(owner))
self.lookup = {
"character varying": "string",
"text": "string"
}
root = self.config.config.get('manifest', 'location')
for f in glob.glob(os.path.join(root, '*.yml')):
m = Manifest(f)
theme = m.data['title'].lower()
for service in m.get_services():
out = {
"theme": theme,
"id": service.name,
"title": service.description,
"tablename": service.name,
}
q = """
SELECT column_name, data_type
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = '{table}';
""".format(table=service.name)
cur = self.conn.cursor()
cur.execute(q)
result = cur.fetchall()
cur.close()
out['fields'] = []
for (n,t) in result:
d = {"name": n, "type": self.lookup.get(t, t)}
out['fields'].append(d)
if isinstance(service.table_settings, list):
settings = service.table_settings[0]
else:
settings = service.table_settings
out['choice_fields'] = settings.get('choice_fields', [])
if hasattr(service, "searchables"):
out['queries'] = service.searchables
outfile = os.path.join(root, "manifests", service.name + ".json")
print outfile
with open(outfile, "w") as fout:
json.dump(out, fout, indent=2)
self.conn.close()
def do_manifest_convert(self, args):
from api_etl import Config
self.config = Config()
owner = self.config.database('owner')
self.conn = psycopg2.connect("dbname=apiserver user={}".format(owner))
self.lookup = {"character varying": "string", "text": "string"}
root = self.config.config.get('manifest', 'location')
for f in glob.glob(os.path.join(root, '*.yml')):
m = Manifest(f)
theme = m.data['title'].lower()
for service in m.get_services():
out = {
"theme": theme,
"id": service.name,
"title": service.description,
"tablename": service.name,
}
q = """
SELECT column_name, data_type
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = '{table}';
""".format(table=service.name)
cur = self.conn.cursor()
cur.execute(q)
result = cur.fetchall()
cur.close()
out['fields'] = []
for (n, t) in result:
d = {"name": n, "type": self.lookup.get(t, t)}
out['fields'].append(d)
if isinstance(service.table_settings, list):
settings = service.table_settings[0]
else:
settings = service.table_settings
out['choice_fields'] = settings.get('choice_fields', [])
if hasattr(service, "searchables"):
out['queries'] = service.searchables
outfile = os.path.join(root, "manifests",
service.name + ".json")
print outfile
with open(outfile, "w") as fout:
json.dump(out, fout, indent=2)
self.conn.close()